Go Down

Topic: Read CSV or TXT from SD card into string or array (Read 824 times) previous topic - next topic

Duke1215

I am working on an system and need to read values on an sd card. The txt or csv will have multiple lines with 2 rows of values. What I would like to achieve is for the arduino to open the sd file, take line one, split it into its two values. val1 and val2. It will then compare the values to what a sensor reads. If they match, it will stop searching the txt file. If the values don't match, it will read and compare the values from the next line in the txt file. I am very new to programming and think the values would either be a string or array. Please help! Thanks

CatweazleNZ

You can read SD card files line-by-line if this helps:

Code: [Select]
String l_line = "";
//open the file here
while (l_SDFile.available() != 0) {
    //A inconsistent line length may lead to heap memory fragmentation
    l_line = l_SDFile.readStringUntil('\n');
    if (l_line == "") //no blank lines are anticipated
      break;
    //

   //parse l_line here
}
//close the file here

   

Cheers

Catweazle NZ

CatweazleNZ

Hi

This more substantial procedure loads a backup file with multiple record types into memory arrays:

Code: [Select]


boolean LoadRecentClimateBackupFile(){
 const byte c_proc_num = 31;
 Push(c_proc_num);
 //After Climate Arrays have been initialised (nulled)
 //After call to CalcNextClimateCheck()

/*    
  30MN Left Temp 22 22 22 23 23 23 23 23 24 24 24 24
  30MN Left Humidity 62 62 62 60 60 60 60 59 59 59 59 59
  DAY Left Temp Max NULL NULL NULL NULL NULL 24 24
  DAY Left Temp Min NULL NULL NULL NULL NULL 22 20
  DAY Left Humidity Max NULL NULL NULL NULL NULL 67 62
  DAY Left Humidity Min NULL NULL NULL NULL NULL 58 59
  AIRPUMP 22 22 22 23 23 23 23 23 24 24 24 24
*/

 if (!G_SDCardOK) {
   Pop(c_proc_num);
   return false;
 }

 SPIDeviceSelect(DC_SDCardSSPin);

 int l_yyyy, l_mm, l_dd, l_hh, l_mn = 0; //l_ss
 String l_SD_filename = "";

 long l_PeriodTime   = 0;
 int  l_PeriodOffset = 0;
 boolean l_SD_file_found = false;
 //Look back up to 12 climate periods for the most recent backup file
 for (byte l_count = 1; l_count <= 12 ; l_count++) {
   l_PeriodTime = Date() + G_NextClimateUpdateTime - long(100000 * l_count / G_Periods_Per_Day) + 1; //Fourth backup ago, fraction adjusted
   DateDecode(l_PeriodTime, &l_dd, &l_mm, &l_yyyy);
   TimeDecode(l_PeriodTime, &l_hh, &l_mn);
   //l_SD_filename = String(ZeroFill(l_mm,2) + ZeroFill(l_dd,2) + ZeroFill(l_hh,2) + ZeroFill(l_mn,2) + ".TXT");
   String l_directory = String(EPSR(E_BACKUPSsl_2217) + ZeroFill(l_yyyy,4) + "/" + ZeroFill(l_mm,2) + "/" + ZeroFill(l_dd,2) );
   SD.mkdir(&l_directory[0]);
   l_SD_filename = String(l_directory + "/" + ZeroFill(l_mm,2) + ZeroFill(l_dd,2) + ZeroFill(l_hh,2) + ZeroFill(l_mn,2) + ".txt");
   //BACKUPS/2013/11/10/11101800.txt
   if (SD.exists(&l_SD_filename[0])) {
     l_SD_file_found = true;
     break;
   }
   l_PeriodOffset += 1;
 }
 CheckRAM();
 if (!l_SD_file_found) {
   SPIDeviceSelect(DC_EthernetSSPin);
   Pop(c_proc_num);
   return false;
 }

 int l_DayOffset = 0;
 //If the date of the backup file we will load is yesterday
 //we need to offset (7) DAY climate data by one day.
 if (l_PeriodTime < Date()) //Date() is midnight 12:00am - the start of today
   l_DayOffset = 1;
 //

 //We have found the most recent backup and have offsets for missed data
 //BACKUPS/2013/11/10/11101800.TXT
 //Serial.println(l_SD_filename);
 File l_SD_file = SD.open(&l_SD_filename[0],FILE_READ);
 CheckRAM();
 if (! l_SD_file) {
   ActivityWrite(String(EPSR(E_Could_not_open_file__1345) + l_SD_filename)); //Should not happen - we know it exists
   SPIDeviceSelect(DC_EthernetSSPin);
   Pop(c_proc_num);
   return false;
 }

 int* l_row; //An int array pointer
 String l_reqd_stattype = DetermineStatType(G_Periods_Per_Day);
 while (l_SD_file.available() != 0) {
   //Hopefully the relatively small size of the files and consistent
   //line length will minimise any String memory fragmentation
   String l_line = l_SD_file.readStringUntil('\n');
   if (l_line == "") //no blank lines are anticipated
     break;
   //
   //Serial.println(l_line);
   //30MN Right Humidity 60 60 60 60 60 59 58 58 58 58 58 58
   //DAY Left Temp Max NULL NULL NULL NULL NULL 24 24
   //AIRPUMP 22 22 22 23 23 23 23 23 24 24 24 24

   int l_posn = 0;
   String l_stattype = ENDF2(l_line,l_posn,'\t');
   if (l_stattype == "AIRPUMP") {
     int l_index1  = DC_ClimatePeriodCount - 1;
     int l_Offset = l_PeriodOffset;
     for (int l_count2 = 1; l_count2 <= DC_ClimatePeriodCount; l_count2++) { //twelve
       String l_value = ENDF2(l_line,l_posn,'\t');
       if (l_value == "") //If index not back to zero we have gaps in the recovered data
                          //because we could not load the most recent backup file.
         break; //we have finished
       //
       if (l_Offset == 0) { //We can load this value
         if (l_value != EPSR(E_NULL_2006))
           G_AirPumpModes[l_index1] = l_value.toInt();
         else
           G_AirPumpModes[l_index1] = 0;
         //
         l_index1--;
       }
       else //We discard end value(s) because we only have an old backup (not the most recent)
         l_Offset--; //By skipping index values we will leave null gaps in the data in the lower array index values
       //
       if (l_index1 < 0)
         break;
       //
     } //for each array element
   }
   else {
     String l_sensor   = ENDF2(l_line,l_posn,'\t');
     String l_stat     = ENDF2(l_line,l_posn,'\t');
     int l_sensor_index = 0; //scoped for later use
     boolean l_sensor_found = false;
     for (l_sensor_index = 0; l_sensor_index < DC_SensorCountTemp; l_sensor_index++) {
       if (C_SensorList[l_sensor_index] == l_sensor) {
         l_sensor_found = true;  
         break;
       }
       //
     } //find the sensor
     if (l_sensor_found) { //skip removed sensors
       if (l_reqd_stattype == l_stattype) {
         //This is a periodic data set
         int l_ValueOffset = 0;
         if (l_stat == EPSR(E_Temp_2398)) {
           l_row = G_ArrayTemp[l_sensor_index].TempPeriodic;
           l_ValueOffset = DC_Temp_UpShift;
         }
         else if (l_sensor_index < DC_SensorCountHum) //Humidity
           l_row = G_ArrayHum[l_sensor_index].HumPeriodic;
         //
 
         if ((l_stat == EPSR(E_Temp_2398)) || (l_sensor_index < DC_SensorCountHum)) {
           int l_index1  = DC_ClimatePeriodCount - 1;
           int l_Offset = l_PeriodOffset;
           for (int l_count2 = 1; l_count2 <= DC_ClimatePeriodCount; l_count2++) { //twelve
             String l_value = ENDF2(l_line,l_posn,'\t');
             if (l_value == "") //If index not back to zero we have gaps in the recovered data
               //because we could not load the most recent backup file.
               break; //we have finished
             //
             if (l_Offset == 0) { //We can load this value
               if (l_value != EPSR(E_NULL_2006))
                 l_row[l_index1] = int((StringToDouble(l_value) + l_ValueOffset) * 10);
               else
                 l_row[l_index1] = 0;
               //
               l_index1--;
             }
             else //We discard end value(s) because we only have an old backup (not the most recent)
               l_Offset--; //By skipping index values we will leave null gaps in the data in the lower array index values
             //
             if (l_index1 < 0)
               break;
             //
           } //for each array element
         } //skip humidity statistics not now required
       } //periodic stat type
       else if (l_stattype == "DAY") {
         String l_sub_stat = ENDF2(l_line,l_posn,'\t'); //Min or Max
         //This is a day min/max data set
         int l_ValueOffset = 0;
         l_row = NULL;
         if ((l_stat == EPSR(E_Temp_2398)) && (l_sub_stat == "Max")) {
           l_row = G_ArrayTemp[l_sensor_index].TempMax;
           l_ValueOffset = DC_Temp_UpShift;
         }
         else if ((l_stat == EPSR(E_Temp_2398)) && (l_sub_stat == "Min")) {
           l_row = G_ArrayTemp[l_sensor_index].TempMin;
           l_ValueOffset = DC_Temp_UpShift;
         }
         else if ((l_stat == EPSR(E_Humidity_1184)) && (l_sub_stat == "Max")) {
           if (l_sensor_index  < DC_SensorCountHum)
             l_row = G_ArrayHum[l_sensor_index].HumMax;
         }
         else { //if ((l_stat == "Humidity") && (l_sub_stat == "Min"))
           if (l_sensor_index  < DC_SensorCountHum)
             l_row = G_ArrayHum[l_sensor_index].HumMin;
         }
         //
         if ((l_stat == EPSR(E_Temp_2398)) || (l_sensor_index < DC_SensorCountHum)) {
           int l_index2 = DC_WeekDayCount - 1;
           int l_Offset = l_DayOffset;
           for (int l_count3 = 1; l_count3 <= DC_WeekDayCount; l_count3++) { //7 days
             String l_value = ENDF2(l_line,l_posn,'\t');
             if (l_value == "")
               break; //we have finished
             //
             if (l_Offset == 0) {
               if (l_value != EPSR(E_NULL_2006))
                 l_row[l_index2] = int((StringToDouble(l_value) + l_ValueOffset) * 10);
               else
                 l_row[l_index2] = 0;
               //
               l_index2--;
             }
             else
               l_Offset--;
             //
             if (l_index2 < 0)
               break;
             //
           } //Load each day of the week
         } //skip humidity statistics not now required
       } //Determine what type of statistic (Periodic or DAY)
     } //Skip removed sensors
   } //Process AIRPUMP data differently
 } //EOF
 CheckRAM();
 l_SD_file.close();

 SPIDeviceSelect(DC_EthernetSSPin);

 Pop(c_proc_num);
 return true;
} //proc


Catweazle NZ

CatweazleNZ

You will also need this to parse your input lines if you are using a delimiter between multiple fields/values on each line:

Code: [Select]

String ENDF2(String &p_line, int &p_start, char p_delimiter) {
  const byte c_proc_num = 27;
  Push(c_proc_num);
//Extract fields from a line one at a time based on a delimiter.
//Because the line remains intact we dont fragment heap memory
//p_start would normally start as 0
//p_start increments as we more along the line
//We return p_start = -1 with the last field

  //If we have already parsed the whole line then return null
  if (p_start == -1) {
Pop(c_proc_num);
return "";
  }

  CheckRAM();
  int l_start = p_start;
  int l_index = p_line.indexOf(p_delimiter,l_start);
  if (l_index == -1) { //last field of the data line
p_start = l_index;
Pop(c_proc_num);
return p_line.substring(l_start);
  }
  else { //take the next field off the data line
p_start = l_index + 1;
Pop(c_proc_num);
return p_line.substring(l_start,l_index); //Include, Exclude
  }
  //Pop(c_proc_num);
}


Ignore calls to Push, Pop and CheckRAM.

Cheers

Catweazle NZ

fat16lib

Here is a simple sketch that reads a CSV file with two numbers on each line.  It doesn't use the dangerous String class.
Code: [Select]

#include <SD.h>
File file;

bool readLine(File &f, char* line, size_t maxLen) {
  for (size_t n = 0; n < maxLen; n++) {
    int c = f.read();
    if ( c < 0 && n == 0) return false;  // EOF
    if (c < 0 || c == '\n') {
      line[n] = 0;
      return true;
    }
    line[n] = c;
  }
  return false; // line too long
}

bool readVals(long* v1, long* v2) {
  char line[40], *ptr, *str;
  if (!readLine(file, line, sizeof(line))) {
    return false;  // EOF or too long
  }
  *v1 = strtol(line, &ptr, 10);
  if (ptr == line) return false;  // bad number if equal
  while (*ptr) {
    if (*ptr++ == ',') break;
  }
  *v2 = strtol(ptr, &str, 10);
  return str != ptr;  // true if number found
}

void setup(){
  long x, y;
  Serial.begin(9600);
  if (!SD.begin(SS)) {
    Serial.println("begin error");
    return;
  }
  file = SD.open("TEST.CSV", FILE_READ);
  if (!file) {
    Serial.println("open error");
    return;
  }
  while (readVals(&x, &y)) {
    Serial.print("x: ");
    Serial.println(x);
    Serial.print("y: ");
    Serial.println(y);
    Serial.println();
  }
  Serial.println("Done");
}
void loop() {}
 

Reading this "TEST.CSV" file.
Quote

1,2
34,56
987,-765
1234,-5678


Results in this output
Quote

x: 1
y: 2

x: 34
y: 56

x: 987
y: -765

x: 1234
y: -5678

Done

Go Up