speeding up reading from SD

Hi, im building an LED Matrix that laods animations from an SD card (basically a diy gameframe clone)
The animations are stored as .bmp files on the sd card.
I have written working code, but i have some problems with inconsistant frame timing.

Im measuring the time it takes to laod the frame and depending on the image it takes about 30-80ms for one 16x16 bmp file of about 800bytes size.

because i want to play my animations with at least 15 frames per second any time over 70ms is bad.

Is there anything i can do to speed up my reading speed?

im using an esp32 microcontroller with the included SD library (sadly sd fat doesnt work on esp)

Below is my function for parsing and reading a bmp file from the SD card:

int readBMP(cLEDMatrixBase &matrix, String path)
{
  //see http://www.ece.ualberta.ca/~elliott/ee552/studentAppNotes/2003_w/misc/bmp_file_format/bmp_file_format.htm for reference
  struct bmp_file_header {
    uint16_t signature;
    uint32_t file_size;
    uint32_t reserved[4];
    uint32_t image_offset;
  };
  struct bmp_info_header {
    uint32_t header_size;
    uint32_t image_width;
    uint32_t image_height;
    uint16_t color_planes;
    uint16_t bits_per_pixel;
    uint32_t compression_method;
    uint32_t image_size;
    uint32_t horizontal_resolution;
    uint32_t vertical_resolution;
    uint32_t colors_in_palette;
    uint32_t important_colors;
  };
  unsigned long start_time = millis();
  File bmpFile = SD.open(path);
  if (!bmpFile) {
    Serial.println("Error: Failed to open");
    return -1;
  }
  
  //read file and info header
  bmp_file_header file_header;
  bmp_info_header info_header;
  file_header.signature = readNbytes16(bmpFile);
  bmpFile.seek(bmpFile.position() + 8);
  file_header.image_offset = readNbytes32(bmpFile);
  bmpFile.seek(bmpFile.position() + 4);
  info_header.image_width = readNbytes32(bmpFile);
  info_header.image_height = readNbytes32(bmpFile);
  bmpFile.seek(bmpFile.position() + 2);
  info_header.bits_per_pixel = readNbytes16(bmpFile);
  info_header.compression_method = readNbytes32(bmpFile);


  //Check for supported BMP file
  if (file_header.signature != 0x4D42) //BMP signature check
  {
    Serial.print("Error: Invalid file signature!");
    Serial.println(file_header.signature);
    return -1;
  }
  if (info_header.compression_method != 0) //check for no compression
  {
    Serial.print("Error: Invalid compression!");
    Serial.println(info_header.compression_method);
    return -1;
  }
  if (info_header.image_width != 16 || info_header.image_height != 16 ) //check that image is 16x16
  {
    Serial.print("Error: Invalid resolution!");
    Serial.println(info_header.image_width + "x" + info_header.image_height);
    return -1;
  }
  if (info_header.bits_per_pixel != 24) //check for 24 bit BMP
  {
    Serial.println("Error: Only 24 bit files are supported!");
    return -1;
  }
  
  //skip the whole header
  bmpFile.seek(file_header.image_offset);   
  byte r, g, b;
  for (byte column = 0; column <  16; column++)
  {
    for (byte row = 0; row < 16; row++)
    {
      b = bmpFile.read();
      g = bmpFile.read();
      r = bmpFile.read();
      matrix(row, column) = CRGB(r, g, b);
    }
  }
  unsigned long current_time = millis();
  if (DEBUG)Serial.println("Time taken: " + String(current_time - start_time) + "ms");
  return 0;
}

and this are my functions for reading either a 32 bit or 16 bit block:

//read up to 4 Bytes form the File p_file and store it as  int32_t
int32_t readNbytes32(File& p_file, int pos, byte nBytes)
{
  if (pos == -1)
    pos = p_file.position();
  if (nBytes > 4)
    return 0;
  if (p_file.position() != pos)
    p_file.seek(pos);

  int32_t weight = 1;
  int32_t result = 0;
  for (; nBytes; nBytes--)
  {
    result += weight * p_file.read();
    weight <<= 8;
  }
  return result;
}

//read up to 2 Bytes form the File p_file and store it as  int16_t
int16_t readNbytes16(File& p_file, int pos, byte nBytes)
{
  if (pos == -1)
    pos = p_file.position();
  if (nBytes > 4)
    return 0;
  if (p_file.position() != pos)
    p_file.seek(pos);

  int16_t weight = 1;
  int16_t result = 0;
  for (; nBytes; nBytes--)
  {
    result += weight * p_file.read();
    weight <<= 8;
  }
  return result;
}