Parsing CSV Data Stored on an SD Card

My data has 34 columns and 1000 plus rows. I read Serial Basics and also SD Card Library for the sketch.
The problem I have is that the code flow is not doing what I need it to do, which is parsing the CSV data.

// include the SD library:
#include <SPI.h>
#include <SD.h>


const byte numChars = 400;
char receivedChars[numChars];
char tempChars[numChars];


boolean newData = false;


int Nose = 0;
int LEye = 0;





File myFile;


void setup() {

  Serial.begin(9600);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  
     
  }


  Serial.print("Initializing SD card...");

  if (!SD.begin(4)) {
    Serial.println("initialization failed!");
    while (1);
  }
  Serial.println("initialization done.");
 
myFile = SD.open("ds2.CSV");


  
  }


void loop() {
    recvWithStartEndMarkers();
 while (myFile.available()) {
      Serial.write(myFile.read());



  if (newData == true)
  {
    strcpy(tempChars, receivedChars);
      parseData();
     showParsedData();
      newData = false; 
  }  
  
}
   }



void recvWithStartEndMarkers() {
 
  static boolean recvInProgress = false;
  static byte ndx = 0;
  char startMarker = '>';
  char endMarker = '<';
  char rc;
 
  
  
  while (Serial.available() > 0 && newData == false) { // <<== NEW - get all bytes from buffer
    rc = Serial.read();
     
    if (recvInProgress == true) {
      if (rc != endMarker) {
        receivedChars[ndx] = rc;
        ndx++;
        if (ndx >= numChars) {
          ndx = numChars - 1;
        }
      }
      else {
        receivedChars[ndx] = '\0'; // terminate the string
        recvInProgress = false;
        ndx = 0;
        newData = true;
      }
    }

    else if (rc == startMarker) {
      recvInProgress = true;
    }
  }
}



void parseData() {

    // split the data into its parts
    
  char * strtokIndx; // this is used by strtok() as an index
  strtokIndx = strtok(tempChars,"*");      // get the first part - the string
  Nose = atoi(strtokIndx);
  strtokIndx = strtok(NULL, "*"); // this continues where the previous call left off  
  LEye = atoi(strtokIndx);     // convert this part to an integer

}


void showParsedData() {
    Serial.print(Nose);
    Serial.print(" _ ");
    Serial.print(LEye);
    Serial.println("*****");  
}

My data looks like:

131281402412824131531184913266129*-14120781268125351265013912614973155168153170150247156249<

Any help will be highly appreciate it.

Thanks

This will get you a bit closer.

const char data[] = "14*120*78*126*8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249";

void setup() 
{
  char* token;
  
  Serial.begin(115200);
  token = strtok (data,"*");
  while (token != NULL)
  {
    Serial.println(token);
    token = strtok (NULL, "*");
  }
  
}

void loop() 
{
  // put your main code here, to run repeatedly:

}

You're reading from Serial instead of myFile

Can you please point me in the right direction. Thanks

arduino_new:
You're reading from Serial instead of myFile

I don't know why you said I don't any end markers. I do have them

Delta_G:
because as I said you don't have end markers so using a function that is called receiveWithStartEndMarkers .

. Each row starts with > and ends with < as you can see below

131281402412824131531184913266129*-14120781268125351265013912614973155168153170150247156249<
2012207*-12186-5227-23176-927728126-353429310863358714616294132142-13275194158143327237161259<
2012207*-13185-6227-22177-927928128-3734194110103348814917293133138-13274196160144326237162260<
200*-1205-15184-9225-24179-1328333130-313581501161434014215417327208141-13324210160142330239162257<
1990204*-14180-8225-29129-1627935168-27357149112153318715410323211178-13321209157140324236155249<
197*-2201-14179-8223-29126-1727333123-3235414689213341421538320213134-15321212155141322237154253<

arduino_new:
You're reading from Serial instead of myFile

I thought I had to tell the serial port that data is coming from a CSV file and the serial port had the job of receiving every single data row and convert it to a variable type. What specific lines are causing the problem? I think this is the part where I am confused.

Thanks

hbtousa:
I don't know why you said I don't any end markers. I do have them . Each row starts with > and ends with < as you can see below

Drop those begin end tags, and read my post # 2. That is one of several ways this can be done. Did you run my sample I provided?

To expand on my example....

** I forgot to add.
If you can not drop the silly begin end tags, simply change this line.
token = strtok (NULL, "");
To
token = strtok (NULL, "
<>");

const char data[] = "14*120*78*126*-8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249";

void setup()
{
  char* token;
  int intArray[21];
  int count = 0;

  Serial.begin(115200);
  Serial.println("Parse To Array");
  token = strtok (data, "*");
  while (token != NULL)
  {
    Serial.println(token);
    intArray[count++] = atoi(token);
    token = strtok (NULL, "*");

  }

  Serial.println("Print array");
  for (int counter = 0; counter < 21; counter++)
  {

    Serial.println(intArray[counter]);
  }
}

void loop()
{
  // put your main code here, to run repeatedly:

}

Romonaga:
Did you run my sample I provided?

No I haven't. My data has a pretty size. Is not just a single line. It might have over 1000 rows.
I have been looking at your code to get ideas.
Thanks

hbtousa:
No I haven't. My data has a pretty size. Is not just a single line. It might have over 1000 rows.
I have been looking at your code to get ideas.
Thanks

If you have control over how the data is stored, you should store it with a CR "\n" for each row. This will allow your code to scale better, what happens when you can not read the complete data-set in memory? Reading a line at a time can prevent that.

However, the example I provided can still work, it just requires more work.

Think about this...

You know how many elements are in each row, I believe you said 31. Now imagine,
the while loop I provided, after 31 times, you have collected a row. Process it, go get next row, until you have processed all the data. You now have enough clues to achieve this goal.

Romonaga:
, i believe you said 31. Now imagine,
the while loop I provided, after 31 times, you have collected a row. Process it, go get next row, until you have processed all the data. You now have enough clues as to how to achieve this goal.

I was thinking something very similar/identical . Thanks for your encouragement. This is the first time I work with SD Card. So I don't know how to treat the data. I deal with video pixels and rgb stuff. Lots of data to take care of.

Thanks.
I will trying.

Why are you using csv in the first place? It's a waste when your processing power and memory are limited. Not to mention it'll needlessly complicate your code. Why don't you just encode your data in a binary format instead when you store it on SD?

Romonaga:
Drop those begin end tags, and read my post # 2. That is one of several ways this can be done. Did you run my sample I provided?

To expand on my example....

** I forgot to add.
If you can not drop the silly begin end tags, simply change this line.
token = strtok (NULL, "");
To
token = strtok (NULL, "
<>");

const char data[] = "14*120*78*126*-8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249";

void setup()
{
 char* token;
 int intArray[21];
 int count = 0;

Serial.begin(115200);
 Serial.println("Parse To Array");
 token = strtok (data, "");
 while (token != NULL)
 {
   Serial.println(token);
   intArray[count++] = atoi(token);
   token = strtok (NULL, "
");

}

Serial.println("Print array");
 for (int counter = 0; counter < 21; counter++)
 {

Serial.println(intArray[counter]);
 }
}

void loop()
{
 // put your main code here, to run repeatedly:

}

He'd also have to change the initial call to strtok or the first int will be incorrectly parsed as 0 because of the leading "<".

If OP is forced to keep his format the same, I'd amend the code and just make it a single call

#define COLUMNS 31
#define DELIMS "<>*"

char data[] = ">131*28*140*24*128*24*131*53*118*49*132*66*129*14*120*78*126*8*125*35*126*50*139*126*149*73*155*168*153*170*150<";

void setup(){
  unsigned char row[COLUMNS];
  unsigned n;

  Serial.begin(115200);
  Serial.println("Parse To Array");
  
  for(n = 0; n < COLUMNS; n++){
    if(char* token = strtok(n ? NULL : data, DELIMS)){
      Serial.println(token);
      row[n] = (unsigned char)atoi(token);
    }
    else{
      Serial.print("error: no data at index "); Serial.println(n);
      break;
    }
  }

  Serial.println("Print array");
  for(unsigned m = 0; m < n; m++)
    Serial.println(row[m]);
  
}

void loop(){
  // put your main code here, to run repeatedly:
}

A lot cleaner imo.

OP, I assumed you are parsing 8-bit color values since you mention rgb. Just wrap this in a function and pass it the data you read from SD