4 digit 7-segment LED Display is Showing each digit sequentially not all at once

So I have a code that uses both the SevSeg.h and IRremote.h libraries. It's basically a infrared remote controlled piano, that displays the note currently played on the 7-seg LED display. With only a 1 digit display it worked flawlessly. But now that I have added the 4 digit display(part #5461AS, pinout attached as image) it flashes through all the digits sequentially instead of just showing the character being used consistently. I think it only happens in the switch/case structure. If i write characters to the display outside the case structure they have no trouble displaying all 4 characters at once. I have a suspicion. There is only 1 connection for each respective line segment of the character, and 4 digit connections. Does this mean that it cannot write each digit simultaneously and has to go 1 at a time? But if so, I feel like it should happen at a speed much faster than my eye could see, because it's quite slow. Could it also be that I'm using my digit pins to be the analog pins? Do i need to specifically declare them digital? I'm at a loss for why it wont just show all 4 characters at the same time. Instead of digit 1 on, all else off. Digit 2 on, all else off. Digit 3 on, all else off. Digit 4 on, all else off. And that cycle repeats while a note is being played.

Here is my code, I use comments for mostly all of it, if you need any clarifications I'll gladly answer whatever question you have. (i left out a majority of "imperial march" song so it would fit character limit)

#include <boarddefs.h>
#include <IRremote.h>
#include <IRremoteInt.h>
#include <SevSeg.h>

// Define hex values for IR Remote buttons 1-9
#define Button_1 0xFF30CF
#define Button_2 0xFF18E7
#define Button_3 0xFF7A85
#define Button_4 0xFF10EF
#define Button_5 0xFF38C7
#define Button_6 0xFF5AA5
#define Button_7 0xFF42BD
#define Button_8 0xFF4AB5
#define Button_9 0xFF52AD
#define Button_FUNCTION 0xFFE21D
#define Button_UP 0xFF906F
#define Button_DOWN 0xFFE01F
#define Button_VOL_UP 0xFF629D
#define Button_VOL_DOWN 0xFFA857
#define Button_PLAY_PAUSE 0xFF02FD
#define Button_FAST_FORW 0xFFC23D
#define Button_REVERSE 0xFF22DD
#define Button_EQ 0xFF9867
#define Button_ST/REPT 0xFFB04F
#define Button_POWER 0xFFA25D     

#define DVD_Remote_Enter 0x8E715AA5


// Define tone Hz values for different buttons
#define NOTE_C4  262
#define NOTE_CS4 277
#define NOTE_D4  294
#define NOTE_DS4 311
#define NOTE_E4  330
#define NOTE_F4  349
#define NOTE_FS4 370
#define NOTE_G4  392
#define NOTE_GS4 415
#define NOTE_A4  440
#define NOTE_AS4 466
#define NOTE_B4  494
#define NOTE_C5  523
#define NOTE_CS5 554
#define NOTE_D5  587
#define NOTE_DS5 622
#define NOTE_E5  659
#define NOTE_F5  698
#define NOTE_FS5 740
#define NOTE_G5  784
#define NOTE_GS5 831 
#define NOTE_A5  880
#define NOTE_AS5 932
#define NOTE_B5  988
#define NOTE_C6  1047

SevSeg sevseg;             // Initialize our seven segment display variable
//int volume = 10;           // Set an analog pin that will control voltage to amp (0-5V)
int receiver = 13;         // initialize pin 11 as our IR Receiver pin
int speaker  = 11;         // initialize pin 11 as speaker;
uint32_t Previous;         // default/previous value for IR recv when 0xFFFFFFFF (button being held down)
//int bright = 90;

IRrecv irrecv(receiver);   // creates new instance of receiver
decode_results results;    


void setup() {
  Serial.begin(9600);  // begin serial data
  irrecv.enableIRIn(); // start the infrared receiver

  byte numDigits = 4;                                 // only one digit display
  byte digitPins[] = {A0, A1, A2, A3};                // specify digit pins
  byte segmentPins[] = {2, 3, 4, 5, 6, 7, 8, 9};      // set segment pins in order A,B,C,D,E,F,G etc
  bool resistorsOnSegments = true;                    // resistor is in series with segment pins
  byte hardwareConfig = COMMON_CATHODE;               // segments share common cathode (-) terminal

  //pinMode(volume, OUTPUT);    // Configure volume pin as output
  //analogWrite(volume, 255);   // Set volume pin to 5V (on a 0-255 0-5V scale)

  sevseg.begin(hardwareConfig, numDigits, digitPins, segmentPins, resistorsOnSegments);
  sevseg.setBrightness(90);

}

void loop() {
  if (irrecv.decode(&results)) {        // if we receive an IR signal
    if (results.value==0xFFFFFFFF) {    // if button is still being held down
      results.value = Previous;         // default to the initial HEX value of button being held down
    }
    switch(results.value) {
      case Button_1: tone(speaker, NOTE_C4, 300); sevseg.setChars("C4"); sevseg.refreshDisplay(); break;
      case Button_2: tone(speaker, NOTE_D4, 300); sevseg.setChars("D4"); sevseg.refreshDisplay(); break;
      case Button_3: tone(speaker, NOTE_E4, 300); sevseg.setChars("E4"); sevseg.refreshDisplay(); break;
      case Button_4: tone(speaker, NOTE_F4, 300); sevseg.setChars("F4"); sevseg.refreshDisplay(); break;
      case Button_5: tone(speaker, NOTE_G4, 300); sevseg.setChars("G4"); sevseg.refreshDisplay(); break;
      case Button_6: tone(speaker, NOTE_A4, 300); sevseg.setChars("A4"); sevseg.refreshDisplay(); break;
      case Button_7: tone(speaker, NOTE_B4, 300); sevseg.setChars("B4"); sevseg.refreshDisplay(); break;
      case Button_8: tone(speaker, NOTE_C5, 300); sevseg.setChars("C5"); sevseg.refreshDisplay(); break;
      case Button_9: tone(speaker, NOTE_D5, 300); sevseg.setChars("D5"); sevseg.refreshDisplay(); break;
      case Button_FUNCTION: ImperialMarch(); break; 
      case Button_EQ: sevseg.blank(); sevseg.refreshDisplay(); break;
      //case Button_VOL_UP: break; 
      //case Button_VOL_DOWN: break;
    }
  Serial.println(results.value, HEX);   // display hex results
  irrecv.resume();                      // next value
  }
  Previous=results.value;
}

void ImperialMarch()
{
  // 1st Measure
  tone(speaker, NOTE_G4, 650);sevseg.setChars("G"); sevseg.refreshDisplay();
  delay(750);
  tone(speaker, NOTE_G4, 650);sevseg.setChars("G"); sevseg.refreshDisplay();
  delay(750);
  tone(speaker, NOTE_G4, 650);sevseg.setChars("G"); sevseg.refreshDisplay();
  delay(750);
  tone(speaker, NOTE_DS4, 500);sevseg.setChars("D"); sevseg.refreshDisplay();
  delay(600);
  tone(speaker, NOTE_AS4, 100);sevseg.setChars("A"); sevseg.refreshDisplay();
  delay(150);

4 digit 7 segment pinout.png

I would try removing all of the sevseg.refreshDisplay() functions from the case statements and put one sevseg.refreshDisplay() in loop() so that sevseg.refreshDisplay() is executed every time loop() runs.

There is only 1 connection for each respective line segment of the character, and 4 digit connections. Does this mean that it cannot write each digit simultaneously and has to go 1 at a time?

most multi-digit 7-segment displays are driven as you suggest. only one digit is active at a time. the segment values and the active digit are changed and maintained for a relatively short time (20msec) before moving to the next digit.

I'm surpised that sevseg doesn't do this

groundFungus:
I would try removing all of the sevseg.refreshDisplay() functions from the case statements and put one sevseg.refreshDisplay() in loop() so that sevseg.refreshDisplay() is executed every time loop() runs.

Oh my gosh! Man, that was a good idea and it worked. That's just bad programming practice on my part in the first place haha. Good, simple tip that ended up fixing everything. Have a good one!