Go Down

Topic: Parsing CSV Data Stored on an SD Card (Read 158 times) previous topic - next topic

hbtousa

My data has 34 columns and 1000 plus rows. I read Serial Basics and also SD Card Library for the sketch.
The problem I have is that the code flow is not doing what I need it to do, which is parsing the CSV data.
Code: [Select]

// include the SD library:
#include <SPI.h>
#include <SD.h>


const byte numChars = 400;
char receivedChars[numChars];
char tempChars[numChars];


boolean newData = false;


int Nose = 0;
int LEye = 0;





File myFile;


void setup() {

  Serial.begin(9600);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
 
     
  }


  Serial.print("Initializing SD card...");

  if (!SD.begin(4)) {
    Serial.println("initialization failed!");
    while (1);
  }
  Serial.println("initialization done.");
 
myFile = SD.open("ds2.CSV");


 
  }


void loop() {
    recvWithStartEndMarkers();
 while (myFile.available()) {
      Serial.write(myFile.read());



  if (newData == true)
  {
    strcpy(tempChars, receivedChars);
      parseData();
     showParsedData();
      newData = false;
  } 
 
}
   }



void recvWithStartEndMarkers() {
 
  static boolean recvInProgress = false;
  static byte ndx = 0;
  char startMarker = '>';
  char endMarker = '<';
  char rc;
 
 
 
  while (Serial.available() > 0 && newData == false) { // <<== NEW - get all bytes from buffer
    rc = Serial.read();
     
    if (recvInProgress == true) {
      if (rc != endMarker) {
        receivedChars[ndx] = rc;
        ndx++;
        if (ndx >= numChars) {
          ndx = numChars - 1;
        }
      }
      else {
        receivedChars[ndx] = '\0'; // terminate the string
        recvInProgress = false;
        ndx = 0;
        newData = true;
      }
    }

    else if (rc == startMarker) {
      recvInProgress = true;
    }
  }
}



void parseData() {

    // split the data into its parts
   
  char * strtokIndx; // this is used by strtok() as an index
  strtokIndx = strtok(tempChars,"*");      // get the first part - the string
  Nose = atoi(strtokIndx);
  strtokIndx = strtok(NULL, "*"); // this continues where the previous call left off 
  LEye = atoi(strtokIndx);     // convert this part to an integer

}


void showParsedData() {
    Serial.print(Nose);
    Serial.print(" _ ");
    Serial.print(LEye);
    Serial.println("*****"); 
}

 


My data looks like:
>131*28*140*24*128*24*131*53*118*49*132*66*129*-14*120*78*126*8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249<

Any help will be highly appreciate it.


Thanks

Delta_G

You're using a parsing method that looks for start and end markers in the data.  It expects each data element to be surrounded by <data>.  Your data looks nothing like that.  So it's no wonder that doesn't work. 
|| | ||| | || | ||  ~Woodstock

Please do not PM with technical questions or comments.  Keep Arduino stuff out on the boards where it belongs.

Romonaga

This will get you a bit closer.

Code: [Select]


const char data[] = "14*120*78*126*8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249";

void setup()
{
  char* token;
 
  Serial.begin(115200);
  token = strtok (data,"*");
  while (token != NULL)
  {
    Serial.println(token);
    token = strtok (NULL, "*");
  }
 
}

void loop()
{
  // put your main code here, to run repeatedly:

}
The universe exists only because we are aware of it.
We want a few mad people now. See where the sane ones have landed us!

arduino_new

You're reading from Serial instead of myFile

Delta_G

You're reading from Serial instead of myFile
Wow.  Great catch.  Definitely not going to parse the file like that. 
|| | ||| | || | ||  ~Woodstock

Please do not PM with technical questions or comments.  Keep Arduino stuff out on the boards where it belongs.

hbtousa

Can you please point me in the right direction. Thanks
You're reading from Serial instead of myFile

Delta_G

Can you please point me in the right direction. Thanks
Well, how hard would it be to use myFile.read when you want data from the file instead of Serial.read?  But it still won't work because as I said you don't have end markers so using a function that is called receiveWithStartEndMarkers is just kind of silly. 

Did you notice reply #2?  That's your pointer in the right direction. 
|| | ||| | || | ||  ~Woodstock

Please do not PM with technical questions or comments.  Keep Arduino stuff out on the boards where it belongs.

hbtousa


I don't know why you said I don't any end markers. I do have them
because as I said you don't have end markers so using a function that is called receiveWithStartEndMarkers . 
. Each row starts with > and ends with < as you can see below

>131*28*140*24*128*24*131*53*118*49*132*66*129*-14*120*78*126*8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249<
>201*2*207*-12*186*-5*227*-23*176*-9*277*28*126*-35*342*93*108*6*335*87*146*16*294*132*142*-13*275*194*158*143*327*237*161*259<
>201*2*207*-13*185*-6*227*-22*177*-9*279*28*128*-37*341*94*110*10*334*88*149*17*293*133*138*-13*274*196*160*144*326*237*162*260<
>200*-1*205*-15*184*-9*225*-24*179*-13*283*33*130*-31*358*150*116*14*340*142*154*17*327*208*141*-13*324*210*160*142*330*239*162*257<
>199*0*204*-14*180*-8*225*-29*129*-16*279*35*168*-27*357*149*112*15*331*87*154*10*323*211*178*-13*321*209*157*140*324*236*155*249<
>197*-2*201*-14*179*-8*223*-29*126*-17*273*33*123*-32*354*146*89*21*334*142*153*8*320*213*134*-15*321*212*155*141*322*237*154*253<


You're reading from Serial instead of myFile
I thought I had to tell the serial port that data is coming from a CSV file and the serial port had the job of receiving every single data row and convert it to a variable type. What specific lines are causing the problem? I think this is the part where I am confused.


Thanks

Delta_G

I don't know why you said I don't any end markers. I do have them . Each row starts with > and ends with < as you can see below
Do you want to parse the whole line?  I thought you wanted the individual numbers?  They don't have start or end markers, just separators. 


I thought I had to tell the serial port that data is coming from a CSV file and the serial port had the job of receiving every single data row and convert it to a variable type. What specific lines are causing the problem? I think this is the part where I am confused.

No, the Serial port only reads communication that comes in over serial.  What you said was like you thought you could tell the telephone to play the radio for you.  The serial line is only used to read when the data is coming from serial.  Now you may see lots of examples where a computer reads a CSV and sends it over the serial line for the Arduino to process.  But in that case the Arduino just knows that data is coming on the serial line, nothing about what it came from. 

If you want to read from myFile then you use myFile.read.  If you want to read from serial transmissions then you use Serial.read. 
|| | ||| | || | ||  ~Woodstock

Please do not PM with technical questions or comments.  Keep Arduino stuff out on the boards where it belongs.

Romonaga

#9
Jul 18, 2019, 04:03 am Last Edit: Jul 18, 2019, 04:18 am by Romonaga
I don't know why you said I don't any end markers. I do have them . Each row starts with > and ends with < as you can see below

Drop those begin end tags, and read my post # 2.  That is one of several ways this can be done.  Did you run my sample I provided?

To expand on my example....

** I forgot to add.
If you can not drop the silly begin end tags, simply change this line.
token = strtok (NULL, "*");
To
token = strtok (NULL, "*<>");


Code: [Select]

const char data[] = "14*120*78*126*-8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249";

void setup()
{
  char* token;
  int intArray[21];
  int count = 0;

  Serial.begin(115200);
  Serial.println("Parse To Array");
  token = strtok (data, "*");
  while (token != NULL)
  {
    Serial.println(token);
    intArray[count++] = atoi(token);
    token = strtok (NULL, "*");

  }

  Serial.println("Print array");
  for (int counter = 0; counter < 21; counter++)
  {

    Serial.println(intArray[counter]);
  }
}

void loop()
{
  // put your main code here, to run repeatedly:

}
The universe exists only because we are aware of it.
We want a few mad people now. See where the sane ones have landed us!

hbtousa

Did you run my sample I provided?
No I haven't. My data has a pretty size. Is not just a single line. It might have over 1000 rows.
I have been looking at your code to get ideas.
Thanks

Romonaga

#11
Jul 18, 2019, 04:21 am Last Edit: Jul 18, 2019, 04:30 am by Romonaga
No I haven't. My data has a pretty size. Is not just a single line. It might have over 1000 rows.
I have been looking at your code to get ideas.
Thanks
If you have control over how the data is stored, you should store it with a CR "\n" for each row.  This will allow your code to scale better, what happens when you can not read the complete data-set in memory?  Reading a line at a time can prevent that.

However, the example I provided can still work, it just requires more work.

Think about this...

You know how many elements are in each row, I believe you said 31.  Now imagine,
the while loop I provided, after 31 times, you have collected a row.  Process it, go get next row, until you have processed all the data. You now have enough clues to achieve this goal.

The universe exists only because we are aware of it.
We want a few mad people now. See where the sane ones have landed us!

hbtousa

, i believe you said 31.  Now imagine,
the while loop I provided, after 31 times, you have collected a row.  Process it, go get next row, until you have processed all the data. You now have enough clues as to how to achieve this goal.
I was thinking something  very similar/identical . Thanks for your encouragement. This is the first time I work with SD Card. So I don't know how to treat the data. I deal with video pixels and rgb stuff. Lots of data to take care of.

Thanks.
I will trying.

shazool

Why are you using csv in the first place? It's a waste when your processing power and memory are limited. Not to mention it'll needlessly complicate your code. Why don't you just encode your data in a binary format instead when you store it on SD?

shazool

Drop those begin end tags, and read my post # 2.  That is one of several ways this can be done.  Did you run my sample I provided?

To expand on my example....

** I forgot to add.
If you can not drop the silly begin end tags, simply change this line.
token = strtok (NULL, "*");
To
token = strtok (NULL, "*<>");


Code: [Select]

const char data[] = "14*120*78*126*-8*125*35*126*50*139*126*149*73*155*168*153*170*150*247*156*249";

void setup()
{
  char* token;
  int intArray[21];
  int count = 0;

  Serial.begin(115200);
  Serial.println("Parse To Array");
  token = strtok (data, "*");
  while (token != NULL)
  {
    Serial.println(token);
    intArray[count++] = atoi(token);
    token = strtok (NULL, "*");

  }

  Serial.println("Print array");
  for (int counter = 0; counter < 21; counter++)
  {

    Serial.println(intArray[counter]);
  }
}

void loop()
{
  // put your main code here, to run repeatedly:

}


He'd also have to change the initial call to strtok or the first int will be incorrectly parsed as 0 because of the leading "<".

If OP is forced to keep his format the same, I'd amend the code and just make it a single call


Code: [Select]

#define COLUMNS 31
#define DELIMS "<>*"

char data[] = ">131*28*140*24*128*24*131*53*118*49*132*66*129*14*120*78*126*8*125*35*126*50*139*126*149*73*155*168*153*170*150<";

void setup(){
  unsigned char row[COLUMNS];
  unsigned n;

  Serial.begin(115200);
  Serial.println("Parse To Array");
  
  for(n = 0; n < COLUMNS; n++){
    if(char* token = strtok(n ? NULL : data, DELIMS)){
      Serial.println(token);
      row[n] = (unsigned char)atoi(token);
    }
    else{
      Serial.print("error: no data at index "); Serial.println(n);
      break;
    }
  }

  Serial.println("Print array");
  for(unsigned m = 0; m < n; m++)
    Serial.println(row[m]);
  
}

void loop(){
  // put your main code here, to run repeatedly:
}


A lot cleaner imo.

OP, I assumed you are parsing 8-bit color values since you mention rgb. Just wrap this in a function and pass it the data you read from SD

Go Up