Automatic Servo with Manual Override (delay problems)

#include <Servo.h> 
 
Servo myservo;
const int leftbutton = 2;  //variable to represent the pin that the left button is on
const int rightbutton = 4;  //variable to represent the pin that the right button is on
int casenum = 0;  //case number to tell the servo which case to run
int pos = 0;  //position
int delaytime = 20;  // increasing this slows down the servo movement
unsigned long LastTimeButtonWasPressed; // last time update
long interval = 3000UL; // interval at which to do something (milliseconds)
 
void setup()
{   
  
  
  myservo.attach(9);  // attaches the servo on pin 9 to the servo object 
  
  pinMode(leftbutton, INPUT);  // the buttons are the inputs
  pinMode(rightbutton, INPUT);

} 

void loop()
{
  unsigned long StartVoidLoopTime = millis();
  if (digitalRead(leftbutton))  //checks to see if each button is being pressed
        {
          casenum = 2;  //if so it switches the casenum
        }
        if (digitalRead(rightbutton))
        {
          casenum = 3;
        }
  
  switch (casenum)  //switch switches the case based on the value of casenum
  {
    case 0:
    for (pos = pos; pos < 180; pos += 1)  // pan right
      {
        
        myservo.write(pos);  // Move to next position
        delay(delaytime);  // Short pause to allow it to move
        if (digitalRead(leftbutton))  //checks to see if each button is being pressed
        {
          casenum = 2;  //if so it switches the casenum
          break;
        }
        if (digitalRead(rightbutton))
        {
          casenum = 3;
          break;
        }
      }

    casenum = 1;  //if none of the buttons are pressed then it goes to pan left
    break;
   
    case 1:
    for(pos = pos; pos >= 0; pos -= 1)  // pan left
      {
     
        myservo.write(pos);  // Move to next position
        delay(delaytime);  // Short pause to allow it to move
        if (digitalRead(leftbutton))  //checks to see if each button is being pressed
        {
          casenum = 2;  //if so it switches the casenum
          break;
        }
        if (digitalRead(rightbutton))
        {
          casenum = 3;
         break;
        }
        
      }
    casenum = 0;
    break;

    case 2:
    while (digitalRead(leftbutton))  //while the button is being pressed
    {
        myservo.write(pos);  //rotates servo to its current positon
        delay(delaytime);
        pos-=1;  //position becomes one less
        LastTimeButtonWasPressed = millis();
    }
        myservo.write(pos);
        if(StartVoidLoopTime - LastTimeButtonWasPressed >= interval)  //if the total time - the start time is greater than 3 sec 
        {         
          //casenum = 1;  //this is where the code goes wrong, it seems like it reads this if statement 
                        //and immedietly say thinks its true
          //LastTimeButtonWasPressed = millis();
          //LastTimeButtonWasPressed = StartVoidLoopTime;
          break;
        }
      
      
    case 3: 
    while (digitalRead(4))
        {
          myservo.write(pos);
          delay(delaytime);
          pos +=1;
          LastTimeButtonWasPressed = millis();
        }
        myservo.write(pos);
        if (StartVoidLoopTime - LastTimeButtonWasPressed >= interval);  //if the total time - the start time is greater than 3 sec 
           {       
          casenum = 1;  //this is where the code goes wrong, it seems like it reads this if statement 
                       //and immedietly say thinks its true
    
           }
            break;
          
        
            
                
       default:
       casenum = 0;
       break;
  }  
}