Looks like waiting for serial was the problem
This fixed it:
void setup() {
Serial.begin(9600);
Serial.println("32-Button Custom Gamepad Controller");
// Initialize custom HID FIRST
static HIDSubDescriptor node(_hidReportDescriptor, sizeof(_hidReportDescriptor));
HID().AppendDescriptor(&node);
// Add small delay for HID to initialize
delay(100);
// Initialize gamepad report
gamepadReport.reportId = 1;
gamepadReport.buttons = 0;