Reading csv file from SD card and converting it to dat file

I need to access large amount of data in my project. Did some researching and came across this site explaining that it is better and faster to store it in .dat file then in .csv:

So I made sketch that populates array of structs with values read from .csv file on SD card and then writes those structs in .dat file on SD card for later use:

#include <SD.h>

const int chipSelect = 53;

File csvFile;
File dataFile;

struct skyObject {
  char type[10];
  char name[20];
  char constelation[12];
};
struct skyObject stars[10];

void setup() {
  Serial.begin(9600);
  pinMode(chipSelect, OUTPUT);
  SD.begin(chipSelect);
  
  if (!card.init(SPI_HALF_SPEED, chipSelect)) {
    Serial.println("initialization failed");
    return;
  } else {
    Serial.println("wiring is correct and a card is present"); 
  }
 
 
  /*
   * reading .csv file from SD card
   * and populatingstruct with read values
   */
  csvFile = SD.open("stars.txt", FILE_READ);
  if(csvFile){
    Serial.println("file opened");
    int i = 0;
    int j = 0;
    char csvField[20];
    int fieldNo = 1;
    char temp;
    while(csvFile.available()){
      temp = csvFile.read();
      csvField[j] = temp;
      j++;
      if(temp == ',' || temp == '\n'){
        csvField[j - 1] = '\0';
        switch(fieldNo){
          case 1:
            strcpy(stars[i].type, csvField);
            break;
          case 2:
            strcpy(stars[i].name, csvField);
            break;
          case 3:
            strcpy(stars[i].constelation, csvField);
            break;
        }        
        fieldNo++;      
        csvField[0] = '\0'; 
        j = 0;  
      }
      if(temp == '\n'){
        i++;
        fieldNo = 1;
      }    
    }
    csvFile.close();
  }else{
    Serial.println("error opening file");
  }
  
  //checking if struct is populated --- it is
  for(int i = 0; i < 10; i++){
    Serial.print(stars[i].type);
    Serial.print('\t');
    Serial.print(stars[i].name);
    Serial.print('\t');
    Serial.println(stars[i].constelation);    
  }
  
  /*
   * Writing array of structs in .dat file on SD card
   */
  dataFile = SD.open("stars.dat", FILE_WRITE);
  if(dataFile){
    for(int i = 0; i < 10; i++){
      dataFile.write((const uint8_t *)&stars[i], sizeof(stars[i]));
      delay(50);
    }
    dataFile.close();
  }
}
  
void loop() {
  // put your main code here, to run repeatedly:

}

It stores values from .csv to struct… they are printed in Serial monitor.
But it doesn’t write them in .dat file. File remains empty (0 bytes).
And I have tried storing handwriten struct like this:

skyObject stars[4] = {
     {"Sirius", "45.67876", "17.5432"},
     {"Rigel", "56.45675", "38.9273"},
     {"Castor", "11.00236", "56.37109"},
     {"Pica", "126.30078", "30.09989"}
  };

and it works fine.

Can someone please point me in the right direction…

What arduino do you have ?
Do you get compile errors or warnings ?
Can you print when you actually write to the file to see if you enter that part?
Have you tried with a struct of 10 elements (are you running out of memory)?
Are you sure 20 is long enough for your longest field?

(you should use SdFat library it’s better)

Well, after 3 days of work I’ve managed to make it work. It was some C string, char, memory thing I don’t quite underestand yet …but I will :slight_smile:

Here is the code if someone needs to convert .csv entries (from SD card) to .dat file that represents some custom made struct, and store it back to SD card:

#include <SD.h>

//CS pin for SD card module
const int chipSelect = 53;

Sd2Card card;
SdVolume volume;
SdFile root;

File csvFile;
File dataFile;

//change to true if you just want to read .dat file from SD card
boolean readOnly = true;

//source file on SD card
char csvFileName[] = "stars.csv";
//destination file on SD card
char datFileName[] = "stars.dat";

//number of fields in struct must match number of values in .csv file
//when adjusting fields in struct need to adjust switch case accordingly
struct skyObject {
  char type[10];
  char name[20];
  char constelation[12];
  float ra;
  float dec;
  float magnitude;
  float distance;
  char info[20];
};
skyObject star;

int index = 1;
//entry number to read from .dat file on SD card
int indexToRead = 10;




void setup() {
  Serial.begin(9600);
  //setup SD card reader
  pinMode(chipSelect, OUTPUT);
  SD.begin(chipSelect);
  
  if (!card.init(SPI_HALF_SPEED, chipSelect)) {
    Serial.println("initialization failed. Things to check:");
    Serial.println("* is a card is inserted?");
    Serial.println("* Is your wiring correct?");
    Serial.println("* did you change the chipSelect pin to match your shield or module?");
    return;
  } else {
    Serial.println("Wiring is correct and a card is present."); 
  }
  // Now we will try to open the 'volume'/'partition' - it should be FAT16 or FAT32
  if (!volume.init(card)) {
    Serial.println("Could not find FAT16/FAT32 partition.\nMake sure you've formatted the card");
    return;
  }  
  Serial.println("\nFiles found on the card (name, date and size in bytes): ");
  root.openRoot(volume);  
  // list all files in the card with date and size
  root.ls(LS_R | LS_DATE | LS_SIZE);



  /*
   * reading values from .csv file on SD card
   * populating coresponding fields of struct
   * and writing those structs to .dat file on SD card
   */
  csvFile = SD.open(csvFileName, FILE_READ);
  if(csvFile && !readOnly){
    Serial.println("\n.csv file opened");
    int i = 0;
    char csvField[20];
    int fieldNo = 1;
    char temp;
    Serial.println("\nPopulating .dat from .csv file as follows:");
    while(csvFile.available()){
      temp = csvFile.read();
      csvField[i] = temp;
      i++;
      if(temp == ',' || temp == '\n'){
        csvField[i - 1] = '\0';
        switch(fieldNo){
          case 1:
            strcpy(star.type, csvField);
            break;
          case 2:
            strcpy(star.name, csvField);
            break;
          case 3:
            strcpy(star.constelation, csvField);
            break;
          case 4:
            star.ra = atof(csvField);
            break;
          case 5:
            star.dec = atof(csvField);
            break;
          case 6:
            star.magnitude = atof(csvField);
            break;
          case 7:
            star.distance = atof(csvField);
            break;
          case 8:
            strcpy(star.info, csvField);
            break; 
        }        
        fieldNo++;
        csvField[0] = '\0'; 
        i = 0;  
      }
      if(temp == '\n'){
        uploadToDat(star);
        printStruct(star);
        fieldNo = 1;
        index++;
      }    
    }
    index = 1;
    csvFile.close();
  }else{
    Serial.println("error opening file, try changing readOnly attribute");
  }
  
  /*
   * Reading data from SD card, 
   * populating structs with values from .dat file
   */
  dataFile = SD.open(datFileName, FILE_READ);
  Serial.println("\nCheck up... populating structs with values from .dat file on SD card:");
  while(dataFile.available()){
    dataFile.read((uint8_t *)&star, sizeof(star));    
    printStruct(star);
    index++;
    delay(50);
  }
  index = 1;
  dataFile.close();

  /*
   * Reading specific entry from .dat file on SD card
   */
  dataFile = SD.open(datFileName, FILE_READ);
  Serial.println("\nCheck up... reading specific entry from .dat file on SD card:");
  dataFile.seek(sizeof(star) * indexToRead);
  index = indexToRead;
  dataFile.read((uint8_t *)&star, sizeof(star));    
  printStruct(star);
  delay(50);
  dataFile.close(); 

  /*
   * Viewin files on SD card after writing to .dat file
   */
  Serial.println("\nFiles found on the card (name, date and size in bytes): ");
  root.openRoot(volume);  
  // list all files in the card with date and size
  root.ls(LS_R | LS_DATE | LS_SIZE);
  
}
  
void loop() {

}


/*
 * Writing struct in .dat file on SD card
 */
void uploadToDat(skyObject star){ 
  dataFile = SD.open(datFileName, FILE_WRITE);
  if(dataFile){
    dataFile.write((const uint8_t *)&star, sizeof(star));
    delay(50);    
    dataFile.close();
  }
}

 
/*
 * Printing struct fields in serial monitor
 */
void printStruct(skyObject star){
  Serial.print(index);
  Serial.print('\t');
  Serial.print(star.type);
  Serial.print('\t');
  Serial.print(star.name);
  Serial.print('\t');
  Serial.print(star.constelation); 
  Serial.print('\t');
  Serial.print(star.ra);
  Serial.print('\t');
  Serial.print(star.dec);
  Serial.print('\t');
  Serial.print(star.magnitude);
  Serial.print('\t');
  Serial.print(star.distance);
  Serial.print('\t');
  Serial.println(star.info);        
}

Later in my project I will just use this kind of snippet to get data from SD card:

dataFile = SD.open(datFileName, FILE_READ);
  Serial.println("\nCheck up... reading specific entry from .dat file on SD card:");
  dataFile.seek(sizeof(star) * indexToRead);
  index = indexToRead;
  dataFile.read((uint8_t *)&star, sizeof(star));    
  printStruct(star); //or do whatever you want with it
  delay(50);

If someone has some suggestions to make it better, faster, smaller… please respond…

J-M-L:
What arduino do you have ?
Do you get compile errors or warnings ?
Can you print when you actually write to the file to see if you enter that part?
Have you tried with a struct of 10 elements (are you running out of memory)?
Are you sure 20 is long enough for your longest field?

(you should use SdFat library it’s better)

In second post I've included working version with struct of 8 elements.
I'll include here short example of my .csv file:

star,Sirius,CMa,6.752481,-16.716116,-1.44,2.6371,add info..
star,Canopus,Car,6.399195,-52.69566,-0.62,94.7867,add info..
star,Arcturus,Boo,14.26103,19.18241,-0.05,11.2575,add info.
star,Rigil Kentaurus,Cen,14.660765,-60.833976,-0.01,1.3248,add info..
star,Vega,Lyr,18.61564,38.783692,0.03,7.6787,add info..
star,Capella,Aur,5.27815,45.997991,0.08,13.1234,add info..
star,Rigel,Ori,5.242298,-8.20164,0.18,264.5503,add info..
star,Procyon,CMi,7.655033,5.224993,0.4,3.5142,add info..

..but it can now be pretty big because I have ditched array approach, and taking entry after entry now

Thanks for taking interest...