Pressing Buttons simultaniously - LED toggle (74HC165, InfectedBytes Library)

Hi everyone!

I've got the following problem:

I am building a MIDI-Controller. Right now I am trying to get the all the buttons to work.
The first 4 buttons have LEDs that toggle on and off with them.
The main functionality is working. When I press the buttons I can toggle LEDs on and off.
The only problem is, when i press two buttons at the same time, only one LED lights up.

I am guessing this is a problem with the debouncing? Or is there any mistake in my code?
Couldn't find anything about this specific question.

Thank you!

My Setup:

and my current code:

#include <ShiftIn.h>

// Init ShiftIn instance with one chip.
// The number in brackets defines  the number of daisy-chained 74HC165 chips
// So if you are using two chips, you would write: ShiftIn<2> shift;
ShiftIn<1> shift;

int button[] = {LOW, LOW, LOW, LOW, LOW, LOW, LOW, LOW};
int buttonOld[] = {LOW, LOW, LOW, LOW, LOW, LOW, LOW, LOW};

int state[] = {LOW, LOW, LOW, LOW};   // the current state of the output pin
int reading[4];           // the current reading from the input pin
int previous[] = {LOW, LOW, LOW, LOW};

long time = 0;         // the last time the output pin was toggled
long debounce = 100;


void setup() {
  Serial.begin(9600);
  // declare pins: pLoadPin, clockEnablePin, dataPin, clockPin
  shift.begin(8, 9, 11, 12);
  pinMode(4, OUTPUT);
  pinMode(5, OUTPUT);
  pinMode(6, OUTPUT);
  pinMode(7, OUTPUT);

}

void displayValues() {
  for (int i = 0; i < shift.getDataWidth(); i++) {
    button[i] = shift.state(i);
    // Get the current state of the button

    //Serial.println(shift.state(1));
    if (button[i] == HIGH && buttonOld[i] == LOW) {
      Serial.write(176);
      Serial.write(i + 1);
      Serial.write(button[i]);
    }
    buttonOld[i] = button[i];
  }
}

void ledlights(){
    for (int j = 0; j < 4; j++) {
      
      reading[j] = digitalRead(shift.state(j));
      if (reading[j] == HIGH && previous[j] == LOW && millis() - time > debounce) {
        if (state[j] == HIGH)
          state[j] = LOW;
        else
          state[j] = HIGH;

        time = millis();
      }

      digitalWrite(j + 4, state[j]);

      previous[j] = reading[j];
    }
}

void loop() {
  shift.update(); // read in all values. returns true if any button has changed
  displayValues();
  ledlights();

}