SD Reading and Writing Multiple Files

Using Arduino Storage
1

Hello everybody

I'm new of the forum, thanks in advance for the precious contribute that many users give! seriously :slight_smile:

Hope to have searched well inside the forum since I did not find any solution for my problem that works.

I'm doing a project with my Arduino UNO, connected to a couple of sensors (tri-axial gyro and accel.), I'm storing the values read from sensors inside an SD card, in a file called for example "FILE 1".

My purpose is to acquire data and process them, so I do need to open FILE 1, read the three reading of angular acceleration (contained in a single line), and process them calculating the derivative.

In order to do processing in the meantime I read data from FILE 1 I need to open another file, let's call it "FILE 2" to save processed data. In the code below I'trying to read LINE 4 inside FILE 1 and when it happens, open FILE 2 and write just a random number.

The entire code is pretty heavy so I just upload the part relating to Read-Write on SD card, if anybody need the rest I can upload it.

Thanks to everyone that will help me!! :smiley:
F.

#include "I2cSerial.h"
#include "TimerOne.h"
#include "JointCalibration.h"
#include <SPI.h>
#include <SD.h>

#define CYCLE 10

I2cSerial *i2c = new I2cSerial();
JointCalibration *jC = new JointCalibration();

// Interrupt Constants y
const long int ISR_PERIOD = 20000.00;      // ISR Clock value in (us)
volatile float deltatime = ISR_PERIOD / 1000000.0000;
const byte chipSelect = 4;

// Variables

static float W1[3];                        // angular Velocities
static float W2[3];

// Flag and Counters
int new_data = 0;                         // flag for new read data from gyro and Acc
int isr_flag = 0;                         // Interrupt positive flag
volatile int count_intr = 0;                // Interrupt counter
int count_data = 0;
byte good_calib = 0;                       // counter for calibration cycle
volatile byte state = 0;                   // counter for setup calibration
boolean logic = false;
byte flagCalib = 1;                        // flag to create angular velocities matrix


File myFile;        // creation of new file on SD card


 /*Reading SD Card Parameters*/

float aW1[3];
float a1[3];
float der1[3];




void setup() {
  Serial.begin(9600);
  
  // LED INITIALIZATION
  pinMode(9, OUTPUT);
  pinMode(5, OUTPUT);
 

  // TURN ON SENSORS 
  i2c->i2cSetAD0();                                        // Setup Scale Sensors
  i2c->i2cSetAD1();
  
    if (!SD.begin(4)){
      Serial.print(F("\n\nInitialization failed!"));
      while(1);  
    } 
    Serial.print(F("\n\nInitialization done"));

    //################### Removing files from SD CARD
      Serial.print(F("\n-->Check if 1st Calibration File already exists")); 
      Serial.print(F("\nSearching for 1st Calibration file..."));
        if (SD.exists("calib1.txt")){
          Serial.print(F(" File present!"));
          Serial.print(F("\n\n...Proceeding removing existent file..."));  
          Serial.print(SD.remove("calib1.txt"));
           if (SD.exists("calib1.txt")){
             Serial.print(F("\nError removing 1st Calibration File"));                                                                      
               }else{
                Serial.print(F("\nFile removed successfully!"));
               }     
        } else {
           Serial.print(F("\n1st Calibration File NOT Present on SD")); 
        }
        Serial.print(F("\n\n-->Check if 1st Calibration Processing File already exists")); 
        Serial.print(F("\nSearching for 1st Calibration Processing file..."));
        if (SD.exists("calP1.txt")){
          Serial.print(F(" File present!"));
          Serial.print(F("\n\n...Proceeding removing existent file..."));  
          Serial.print(SD.remove("calP1.txt"));
           if (SD.exists("calP1.txt")){
             Serial.print(F("\nError removing 1° Calibration File"));                                                                      
               }else{
                Serial.print(F("\nFile removed successfully!"));
               }     
        } else {
           Serial.print(F("\n1st Calibration Processing File NOT Present on SD")); 
        }
  Serial.print(F("\n\nCreation of a new 1st Calibration File..."));
  
  // Remember files might be named with 8.3 DOS eg: 12345678.123
  myFile = SD.open("calib1.txt", FILE_WRITE);
  Serial.print(F("\nCheck if file has been created-->"));
  Serial.print(SD.exists("calib1.txt")); 
  (myFile == true) ? (Serial.print(F("\nDIO C'e'"))) : (Serial.print(F("\nDIO NON C'e'")));  
}

void loop() {
  
  Timer1.initialize(ISR_PERIOD);                             // initialize timer1, and set a the period in millisecond 
  Timer1.attachInterrupt(isrMethod);                         // attaches ISR() as a timer interrupt function
 
  if (!logic){
    if (myFile){
      Serial.print(F("\n\nTest writing to SD file"));
      while(( count_intr < CYCLE )){
        readData();                                           // Reading data from sensors
        if (isr_flag == 1){
//          printIsrGyro();
          isr_flag = 0;                                        // reinizializzazione del flag dell'interrupt
          for (int i=0; i<3; i++){
           /* Writing angular velocities*/
            myFile.print(W1[i],4);                          
            myFile.print("\t");
          }
          myFile.print("\n");
        }
     }
  /*closing the file*/
  myFile.close();
  Serial.println(F ("\nClosing File ...Done"));   
    } else {       
        Serial.println(F("\n\nerror during Writing on myFile"));      // Error message!
        myFile.close();
    }
    
    /* Reopen file for Reading and PROCESSING*/   <--- HERE WHERE COMES TROUBLE! 
    Serial.println(SD.remove("CALP2.txt"));
    int recNum1 = 0;
    int num = 888;
    myFile = SD.open("calib1.txt");
    while(myFile.available()){
      String valueD = myFile.readStringUntil('\t');
      recNum1++;
      Serial.print(valueD);
      Serial.print("\t");
      Serial.print(recNum1);
      Serial.print("\n");
      if(recNum1 == 4){ 
        File myFile2 = SD.open("calP2.txt",FILE_WRITE);
        Serial.println(myFile2);
        myFile2.print(num);
        myFile2.close();
        digitalWrite(5,HIGH);
        delay(500);
        digitalWrite(5,LOW);
      }
    }
//    myFile.close();

  logic = true;
  }
}


void readData() {         // Values read from Gyro without Offset
  i2c->i2cReadGyroAD0(0);
  new_data = 1;
  count_data++;
}

void isrMethod() {          
  if (new_data == 1) {
    digitalWrite(9, digitalRead(9) ^ 1); 
    W1[0] = (i2c->rot1.x);
    W1[1] = (i2c->rot1.y);
    W1[2] = (i2c->rot1.z);
    isr_flag = 1;
    new_data = 0;
    count_intr++;
  }
}  

void printIsrGyro(){
  Serial.print("\n");
  Serial.print(count_intr);
  Serial.print("  ");
  Serial.print(count_data);
  Serial.print("  ");
    for (int i=0;i<3;i++){
      Serial.print(W1[i],4);
      Serial.print("\t");  
    }  
}
2

Thanks to everyone that will help me!!

I'd be happy to, if I knew what your problem was. You seem to have forgotten to tell us what your problem is.

3

Hi Paul!

Thanks for your observation, actually reading again my post carefully there is not an explicit description of the problem involved.. sorry :confused:

Anyway I managed to solve my problem of reading and writing more files at the same time, I was going to write the new code (but you anticipated me); since it looks very slow to read and process data from SD card.. (Have I to open a new topic or rename the subject?

I read data from a file called file1, where are contained three readings from Gyroscope, so it's a kind of matrix (n x 3), number are separated by "\t". I have to read them from file 1, process them calculating the first finite derivative and write results on file 2. Since I'm using the function:

File.readStringUntil("\t")

when I'm reading file 1, this command start to read from the first bit until it reaches a specific data (n + k); the method that I'm using to do that is this:

/*
    The circuit on Arduino UNO
 ** MOSI - pin 11 
 ** MISO - pin 12 
 ** CLK - pin 13 
 ** CS - 4 Arduino Uno
 
    Arduino MEGA
 ** MOSI - pin 51
 ** MISO - pin 50  
 ** CLK - pin 52 
 ** CS - 53 Arduino Mega
*/

#include "I2cSerial.h"
#include "TimerOne.h"
#include "JointCalibration.h"
#include <SPI.h>
#include <SD.h>


I2cSerial *i2c = new I2cSerial();
JointCalibration *jC = new JointCalibration();


// ################################################################################################################
// ===                                        --> VARIABLES <--                                          ===
// ################################################################################################################

// Variables

static float W1[3];                        // angular Velocities Sensor 1
static float W2[3];                        // angular Velocities Sensor 2


// Sd Flags and Counters
int recNum = 0;                        // counter Data read from Sd card
byte flag = 0;                           // flag three values read  (x,y,z)
int limitLines = 3*CYCLE;           // Total number of Data acquired with Sensor 
int n = 0;                                  // counter 
const int CYCLE = 300;

File myFile;        // creation of new file on SD card
File myFile2;


 /*Reading SD Card Parameters*/
float aW1[3];                 // array containing the angular velocity at time n
float a1[3] = {0,0,0};    // variable temporary to store angular velocity at time n-1
float der1[3];                 // first derivative


void processCal1(){         /*Processing method of calibration file 1*/
  Serial.print(F("Processing Data...\n\n"));
  
  while(n<limitLines){    /*start from first row in the file*/
    flag = 0;
    recNum = 0;
    myFile = SD.open("File1.txt");
    
    while(myFile.available()){
      /*cycle continue until it reads value corrisponding to n*/
      String valueD = myFile.readStringUntil('\t');
      recNum++;     
      if(recNum == n+1){     /*recNum and n have a phase of one unit*/ 
        aW1[0] = valueD.toFloat();
        der1[0] = (aW1[0] - a1[0]) / 0.005;
        a1[0] = aW1[0];
      }
      if(recNum == n+2){ 
        aW1[1] = valueD.toFloat();
        der1[1] = (aW1[1] - a1[1]) / 0.005;
        a1[1] = aW1[1];
      }
      if(recNum == n+3){ 
        aW1[2] = valueD.toFloat();
        der1[2] = (aW1[2] - a1[2]) / 0.005;
        a1[2] = aW1[2];
        n = n+3;                    // updating index External While loop
        flag = 1;
        break;
       }
        if(flag == 1){
         return;
       }                      // exit from internal while.available to print on calib1PP() with return
    } // end myFile.available
     
  myFile.close();
  
  /*open Processing File*/
  File myFile2 = SD.open("File2.txt",FILE_WRITE);
//(myFile2 == true) ? (Serial.print(F("File opened\n"))) : (Serial.print(F("\nError Opening the 1st Calibration File for READING\n\n '")));
    
  /*Writing Data and Derivative New File*/
  for (int j=0; j<3; j++){
    myFile2.print(aW1[j],4);
    myFile2.print("\t");       
  }
  for (int j=0; j<3; j++){
    myFile2.print(der1[j],4);
    myFile2.print("\t");       
  }
  

  /* BEGINNING AXIS RECOGNITION ROTATION*/

  byte stateW1 = 0;

  //AXIS Label recognition 
  /* W1 = + + +   W2 = + + -   W3 = + - -   W4 = + - +   W5 = - - -   W6 = - - +   W7 = - + +  W8 = - + - */
  (aW1[0] >= 0 && aW1[1] >= 0 && aW1[2] >= 0) ? (stateW1=1) : (false);
  (aW1[0] >= 0 && aW1[1] >= 0 && aW1[2] < 0) ? (stateW1=2) : (false);
  (aW1[0] >= 0 && aW1[1] < 0 && aW1[2] < 0) ? (stateW1=3) : (false);
  (aW1[0] >= 0 && aW1[1] < 0 && aW1[2] >= 0) ? (stateW1=4) : (false);
  (aW1[0] < 0 && aW1[1] < 0 && aW1[2] < 0) ? (stateW1=5) : (false);
  (aW1[0] < 0 && aW1[1] < 0 && aW1[2] >= 0) ? (stateW1=6) : (false);
  (aW1[0] < 0 && aW1[1] >= 0 && aW1[2] >= 0) ? (stateW1=7) : (false);
  (aW1[0] < 0 && aW1[1] >= 0 && aW1[2] < 0) ? (stateW1=8) : (false);

  myFile2.print(stateW1); 

  myFile2.print("\n");
  myFile2.close(); 
  } // end Limitlines while
}

I'm aware that this approach is time consuming due to all the open and close file, I would like to fast it using another approach, I think that the problem is that I put the method file.readStringUntil("\t") in a while loop, in this way it starts to read always from the beginning I guess, is there any way to make it continue reading without start over again from beginning ?

Or maybe I should analyse binary data coming from the buffer instead of converting Strings to Float, but actually I'm pretty confused how data are saved and sent to buffer from Sd Card :confused:

Is there any method faster to read the file and process it without start over every time?

Even redirecting to other posts would be helpful!

Thank you very much! :slight_smile:
F.

4

I don't see where processCal1() is called, or where, outside of that function, n is manipulated.

n is a lousy name for a global variable. It conveys no information.

So, I really can't help you.

There ARE functions that you might want to look at, though - pos() and seek().

You can keep track of where you last read, and go right there next time. Like using a book marker to keep track of where you are in a book, instead of starting from page 1 each time you want to resume reading.

5

Thanks PaulS.

I looked for pos() and seek() functions and I think I can manipulate them for my purpose.

I received even a tip from a friend that told me, instead of write every time I find a value on SD card, I can store values on a global variable (2D matrix), and once the matrix has been filled write everything on SD card at once, in this way I can reduce the occurrences of open/close SD. Of course this method is memory demanding, but I just bought Arduino Mega so I hope to have no problem saving a float[10][4].

Thanks again for your help, I guess my answer has been replied :slight_smile:

F.