Automatic Servo with Manual Override (delay problems)

Here is a modified version of your code that I reformatted and made some significant changes to. I have not tested it, but it should be significantly easier to read, understand, and troubleshoot:

#include <Servo.h> 
 
Servo myservo;
const int leftButton  = 2;     //variable to represent the pin that the left button is on
const int rightButton = 4;     //variable to represent the pin that the right button is on
int caseNum = 0;               //case number to tell the servo which case to run
int pos = 0;                   //position
int delayTime = 20;            // increasing this slows down the servo movement
unsigned long lastPress = 0;   // last time update
long interval = 3000UL;        // interval at which to do something (milliseconds)
 
void setup() {   
  myservo.attach(9);           // attaches the servo on pin 9 to the servo object 
  pinMode(leftButton, INPUT ); // the buttons are the inputs
  pinMode(rightButton, INPUT);

} 

void loop() {
  if (digitalRead(leftButton)) {         //check if left button pressed
    caseNum = 2;                         //if so it switches the caseNum
  } else if (digitalRead(rightButton)) { //checks right button (else skips if left is pressed)
    caseNum = 3;
  }
  
  switch (caseNum) {
    case 0:
      while( pos < 180 ) {   // pan right until 180 degrees
        myservo.write(pos);  // Move to next position
        delay(delayTime);    // Short pause to allow it to move
        pos++;               // increment
        if (digitalRead(leftButton) || digitalRead(rightButton))  // checks to see if either button is down
          break;             // break out of subloop to check inputs
      }
      caseNum = 1;  // if no buttons pressed, default to pan left
      break;
    case 1:
      while(pos > 0) {       // pan right until 0 degrees
        myservo.write(pos);  // Move to next position
        delay(delayTime);    // Short pause to allow it to move
        if (digitalRead(leftButton) || digitalRead(rightButton))  // checks to see if either button is down
          break;             // break out of subloop to check inputs
      }
      caseNum = 0;  // if no buttons pressed, default to pan right
      break;
    case 2:
      while (digitalRead(leftButton)) { //while the button is being pressed
        pos-=1;                         //position becomes one less
        if( pos < 0 ) pos = 0;          // only necessary if you want to restrict motion to 180
        myservo.write(pos);             //rotates servo to its current positon
        delay(delayTime);
        lastPress = millis();
      }
      if(millis() - lastPress >= interval)  // if it has been interval since last press
        caseNum = 0;                        // return to expected pan sequence
      break;      
    case 3: 
      while (digitalRead(4)) {
          pos+=1;                         // position becomes one greater
          if( pos > 180 ) pos = 180;      // only necessary if you want to restrict motion to 180
          myservo.write(pos);
          delay(delayTime);
          lastPress = millis();
      }
      if ( millis() - lastPress >= interval)  // if it has been interval since last press 
        caseNum = 1;
      break;
    default:
      caseNum = 0;
  }
}