SD Card bitmap read error/bug?

Greetings fellow tinkerers!

I'm having a doozie of an issue with a Dumilenove w/328p -- on Arduino IDE 1.0.1, using default SD library. Also included is the FAST_SPI library for outputting the data to another bus once its been read from the card. The result is the SAME with and without the FAST_SPI library/code.

EDIT the problem is mostly solved, but evolved into a completely different glitch! See below for new version/problem!

Power: I've tried with/without external power, same result.

Sketch: Loading from SD card a bitmap file, 64 pixels wide by 100 pixels tall. Trick is I only load 1 line of it at a time because I have so little ram to work with. During testing I'm spitting that array out the Serial.print.

I think I may be running out of RAM and having a pointer go haywire somewhere in the background, but I can't debug to that level to understand.

The attached 'output' from the serial is a short version. If I let it run long enough it either stops (fully crashes) or reads, but the 'for' loop in the main loop exceeds the original height (<100) and starts asking for row ++ forever.

Attached is the bitmap file, the code, and the output.

#include <FastSPI_LED.h>
#include <SD.h>

File myFile;
const int chipSelect = 4;
String fileNam = "3.bmp";
unsigned int height = 1;
unsigned long bitmapOffset = 0x36;
unsigned long filePosition = 0;

#define NUM_LEDS 64


int frameDelay = 1; //number of millis between animated frames


struct CRGB { 
  unsigned char r; 
  unsigned char g; 
  unsigned char b; 
}; 
struct CRGB *leds;  //I don't know what this does.


unsigned long CurTime = millis();

void setup()
{

  randomSeed(analogRead(0));
  Serial.begin(9600);
  //Serial.println("Start");

  //Serial.print(CurTime);
  fastSPIsetup();  //make the fastSPI library do its magic
  Serial.print("Initializing SD card...");
  // make sure that the default chip select pin is set to
  // output, even if you don't use it:
  pinMode(10, OUTPUT);
  if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    // don't do anything more:
    return;
  }
  Serial.println("card initialized.");
}

void loop() { 
  myFile = SD.open("3.bmp", FILE_READ);
  Serial.print("BitmapOffsetStart:");
  Serial.println(bitmapOffset,HEX);
  myFile.seek(0x16);
  height = myFile.read();
  myFile.seek(0xA);
  bitmapOffset = myFile.read();
  myFile.close();
  Serial.print("BitmapOffset:");
  Serial.println(bitmapOffset,HEX);
  Serial.print("Height:");
  Serial.println(height);

  
  for(int j=0; j<height;j++) //loop through each line in the file, then spit it out.
  {
    FSPIlineOut(j); //reads a line and displays
    delay(frameDelay);
  }

} //end void loop

void FSPIlineOut(unsigned int lineNo)
{
  myFile = SD.open("3.bmp", FILE_READ);
  //delay(100);
  filePosition = bitmapOffset;
  filePosition += (lineNo *(NUM_LEDS * 3) );
  myFile.seek(filePosition);//get to data
  delay(100);
  //memset(leds, 0, NUM_LEDS * 3);//blank the array
  Serial.print(lineNo);
  Serial.print(":");
  Serial.print(filePosition,HEX);
  Serial.print(":");
  Serial.print(myFile.position(),HEX);
  Serial.print(": ");
  for(int i=0; i < NUM_LEDS; i++)
  {
    leds[i].b=myFile.read();
    leds[i].g=myFile.read();
    leds[i].r=myFile.read();   
    Serial.print(leds[i].r,HEX);  
    Serial.print(",");
  }  
  Serial.println();
  if (!myFile){
    Serial.println("the card broke"); 

    //myFile.close();

  }//FastSPI_LED.show();   // write all the pixels out
  //delayMicroseconds(4);//to remove g-glitch (can be as low as 4)
  drawArray();
  //delay(frameDelay);//some actual display time
}





void drawArray() { //take the current pixel array, dump it to the leds, and shiftout
  //memset(leds, 0, NUM_LEDS * 3); //clear the led array
  //PORTD |= B00010000;// Set bit high
  //FastSPI_LED.show(); // shift out the array.
  //PORTD &= ~B00010000;// Set bit low - happens a bit fast for the NAND so above delay is required to de-glitch it
  //delay(1);
}








void fastSPIsetup()  //gets the fastspi library set up
{
  FastSPI_LED.setLeds(NUM_LEDS);
  FastSPI_LED.setChipset(CFastSPI_LED::SPI_WS2801);
  FastSPI_LED.setDataRate(3); //make the library work with 5v ws2801 strips


  FastSPI_LED.init();
  FastSPI_LED.start();

  leds = (struct CRGB*)FastSPI_LED.getRGBData(); 

}

**^^above, note how it loads the file, seeks to the correct byte loaded from the bitmap header (0x36 is the position to find the first byte of the data, read 3 bytes per pixel until the end). **
The serial output is below:
The line after 'Height:100' is where the data is actually spilling out.
The left columns are 'line of the bitmap file read from the bottom': "address of the start of this line byte requested", and "address actually being read"
The first 16 lines (0-15) are correct, the remainder (ad nauseum) are broken...not reading anything of value, and spitting out -1 (no data found)
Particular interest is bitmap line 0: everything is correct, just the 'red' values are displayed, also, the other good read lines are a vertical slant, also read properly.

Initializing SD card...card initialized.
BitmapOffsetStart:36
BitmapOffset:36
Height:100
0:36:36: 0,FF,0,FF,0,0,FF,0,0,FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,FF,FF,0,FF,FF,0,0,FF,0,FF,0,
1:F6:F6: FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
2:1B6:1B6: 0,FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
3:276:276: 0,0,FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
<<redacted to reduce character count>>
14:AB6:AB6: 0,0,0,0,0,0,0,0,0,0,0,0,0,FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
15:B76:B76: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
16:C36:C36: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,FF,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
17:CF6:FFFFFFFF: FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,
the card broke
18:DB6:FFFFFFFF: FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,FF,
the card broke

2.bmp (18.8 KB)

LedStripSDLoader.ino (3.08 KB)

Sorry for double post, previous post full.

If I hook this up with my mega1280, it works 'more' -- it loads the entire file twice before failing (same problem, 'random' pointer for file.position) After about 2.1 runs of the file it crashes entirely.

Makes me think more and more there is a memory leak and it's only manifesting because I'm loading so much at once

It appears that the function of the program is to read all the data in the file. Given that, it is not clear why you open and close the file so many times, and why you need to use seek() find the read position.

Opening the file, and not closing it is not a good thing to do.

struct CRGB *leds;  //I don't know what this does.

Then you need to learn. It is, I think, the crux of your problem.

You have declared a pointer, but it doesn't point anywhere. Then, you attempt to write to the memory that it points to, as you read data from the file.

You know how much data there is in the file. Therefore, you know how big the leds array should be. Therefore, you should be using an array, not a pointer. Effectively, you have a 0 element array that you are writing a lot more than 0 positions in.

Since I'm opening a potentially huge file, I can only ever load a line or two of it at a time. >192 bytes goes by fast when you only have 2k, and the SD library uses 1k of it.

Totally agreed, thats something I've needed to learn for a while, but 'struct' doesn't come up often in beginner coding
I'm using Daniel Garcia's 'fastspi' library to output the data to an led strip. It operates using this pointer and I can't eliminate it. In the past I've been able to write to leds (what I thought was an array but is actually a pointer)...procedurally for days on end, one byte at a time. the .h and .cpp files are unreadable as its just one block of code (no line endings).

Problem is, this is to load single lines into a persistence of vision led strip, and the SPI bus is used to keep things speedy. FASTSPI library defines the pointer, not the code I'm using, and is used in the background.

1)Would you recommend I create another actual array to load the data into from the sd card then copy it over/move the pointer somehow to that memory?
2) I can't seem to open the file either globally or in one function and have it available from another function, hence it needs to be opened every call of the for loop. Is there a good way around this? (nest the code in the main loop so it's parented?)
edit: 3) I get the same result whether I file.close or not (you're right, I should be closing it each time).

Since I'm opening a potentially huge file, I can only ever load a line or two of it at a time.

I understand reading the file one line at a time. I don't understand the need to open and close it after every read. Open the file. Read all the contents, shifting them out using the library. Close the file.

No need to make all the records the same size. No need to keep track of which record you are on. No need to seek to the start of the next record.

but 'struct' doesn't come up often in beginner coding

Once you get past chapter two, using the index an looking stuff is is something you should be able to do. I know that most books explain that pointers and arrays are interchangeable.I also know that most books don't make it clear enough that that only applies to properly initialized pointers, and that most books do a less than stellar job of explaining how to properly initialize a pointer.

Which is why, in your case, I suggest that you delete the pointer, and use an array, instead. Now, the only question is how big does the array need to be. Only you can answer that. How much data is on each row? How long is the longest row? You need to size the array big enough so that you never overrun the end of it, but small enough so that it fits in the available memory.

1)Would you recommend I create another actual array to load the data into from the sd card then copy it over/move the pointer somehow to that memory?

Yes, and no. Yes, you need an array. No, you don't need a pointer at all. By definition, the array itself points to the first element in the array, so wherever you need a pointer, just use the array name.

  1. I can't seem to open the file either globally or in one function and have it available from another function, hence it needs to be opened every call of the for loop. Is there a good way around this? (nest the code in the main loop so it's parented?)

Why can't you? The SD.open() function returns a value that can be stored in a local variable or a global variable.

  1. I get the same result whether I file.close or not (you're right, I should be closing it each time).

As I mentioned, the file opening and closing is only a potential problem area. The uninitialized pointer is the real problem. Let's work on that one first. When that is solved, then we can work on the open/close/reopen/reclose issue.

Once again thanks for the help;

My biggest trouble is I'm using the fastSPI library, and that pointer was set up by the library example - it would stand to reason if I want to use the spi output (addressable RGB leds are painfully slow to bitbang) -- I would need the data to be in the format required by the library.

*Should have posted before:
http://code.google.com/p/fastspi/

is the code I can't seem to edit because it has no line breaks. I wouldn't know where it begins or ends - it does seem well commented though.

EDIT: So, I got it working, sort of with the existing code, and I remember after I changed it now why it was the way it was;

I'm using SPI to read the card.
I'm using SPI to shift the data out (for speed). I need to close the file before calling the shiftout function or it crashes. It could also be my shiftout library that just plain crashes with other stuff on the SPI bus also. Guess I'm stuck not using the SPI shiftout code or I'm hooped. Having to close the file every iteration definitely slows things down. Right now I can read the 192 bytes I need into the array (pointer) in about 6 ms, reliably. shifting out is now the problem.

Newest version attached; now has a the mentioned bug but doesn't crash at least.

Any thoughts on how this can be mitigated? I've tried everything I can think of!

LedStripSDLoader.ino (4.16 KB)

now has a the mentioned bug but doesn't crash at least.

Something seems to be missing from this sentence. Can you summarize what the problem is?

Cross posted to the FastSPI code page:
http://code.google.com/p/fastspi/issues/detail?id=16

I'm pretty sure the FastSPI code/pointers are stepping on the memory buffer area of the SD library.
If I never call the fastSpi.show() function it reads the bitmap and serial.print's it just fine. If I call fastspi.show (the whole point of this exercise) every 8 and 8+2 lines have errors in them -- seemingly totally random data...amounts to every 1500 or so bytes.

Attached are the files for your viewing pleasure;

The log files are legend:
line number of the bitmap file : byte address requested of the SD library: byte address returned : 192 data bytes

Update:
Found a fix at a big performance hit:
David suggested putting a myFile.peek(); before each myFile.read(); . No idea why it works, but it does. It costs 17ms / 192 bytes read instead of just 7ms/ 192 bytes, but it works.

log no fastspi.txt (39.5 KB)

log with fastspi.txt (39.7 KB)

2.bmp (18.8 KB)

LedStripSDLoader.ino (4.43 KB)

how can i read .bmp file in sd card.
can i do this by using following code or need some changes in it.

#include <SPI.h>

#include <SD.h>

File myFile;

void setup()
{
// Open serial communications and wait for port to open:
 Serial.begin(9600);
  while (!Serial) {
   ; // wait for serial port to connect. Needed for Leonardo only
 }


 Serial.print("Initializing SD card...");
 // On the Ethernet Shield, CS is pin 4. It's set as an output by default.
 // Note that even if it's not used as the CS pin, the hardware SS pin 
 // (10 on most Arduino boards, 53 on the Mega) must be left as an output 
 // or the SD library functions will not work. 
  pinMode(10, OUTPUT);
  
 if (!SD.begin(4)) {
   Serial.println("initialization failed!");
   return;
 }
 Serial.println("initialization done.");
 
 // open the file. note that only one file can be open at a time,
 // so you have to close this one before opening another.
 myFile = SD.open("test.txt", FILE_WRITE);
 
 // if the file opened okay, write to it:
 if (myFile) {
   Serial.print("Writing to test.txt...");
   myFile.println("testing 1, 2, 3.");
// close the file:
   myFile.close();
   Serial.println("done.");
 } else {
   // if the file didn't open, print an error:
   Serial.println("error opening test.txt");
 }
 
 // re-open the file for reading:
 myFile = SD.open("test.txt");
 if (myFile) {
   Serial.println("test.txt:");
   
   // read from the file until there's nothing else in it:
   while (myFile.available()) {
    Serial.write(myFile.read());
   }
   // close the file:
   myFile.close();
 } else {
  // if the file didn't open, print an error:
   Serial.println("error opening test.txt");
 }
}

void loop()
{
// nothing happens after setup
}

(Next time, use code tags please. You can also select More, Modify and add to your post vs reposting the whole thing to add an attachment. Moderator)
.bmp file is given

Capture.JPG