Optimizing SD Card Opening Cycle

Other Hardware Storage
1

Hello all,

Recently, I have been attempting to increase the frequency of cataloguing from an MPU6050, and BMP180, to an SD Card. I have been successful in reducing latency from several readings. However, one problem remains.

My code is inefficient in the fact that every loop in the "void loop()" function, I open, log the data, and close the SD Card. This entire cycle takes 10 ms. I have read that if I open the SD Card once, and close it once, I can drastically decrease the latency of the cycle.

I am unsure as to how the code would work to only open the SD Card once, and close it once as well.

Below is my current code (for testing, not actual usage):

#include <Adafruit_BMP085.h> //Include Adafuit BMP180 Library
#include <Adafruit_MPU6050.h> //Include Adafruit MPU6050 Library
#include <Adafruit_Sensor.h> //Include General Adafruit Sensor Library
#include <SD.h> //Include the SD Card Library
#include <SPI.h> //Include the SPI Communication Library
#include <Wire.h> //Include I2C Communication Lirary

Adafruit_MPU6050 mpu; //Define MPU Word as "mpu"
Adafruit_BMP085 bmp; //Define BMP Word as "bmp"

int CS_Pin = 10; //Define Chip Select Pin for SD Reader as Digital Pin 10 (as per PCB)
int BMP_IND_PIN = 4; //Define LED Indicator Pin for BMP Indicator LED as Digital Pin 4 (as per PCB)
int MPU_IND_PIN = 5; //Define LED Indicator Pin for MPU Indicator LED as Digital Pin 5 (as per PCB)
int SD_IND_PIN = 6; //Define LED Indicator Pin for SD Card Indicator LED as Digital Pin 6 (as per PCB)
int Increment = 0; //Define Increment variable as means for a timer
int j = 0; //Define j as increment for loops for indicating error of SD Card
unsigned long Time;

#define BMP085_ULTRAHIGHRES 3

void setup() {
  //DEFINING OUTPUTS
  pinMode(BMP_IND_PIN,OUTPUT); //Define BMP_IND_PIN as an OUTPUT - for LED
  pinMode(MPU_IND_PIN,OUTPUT); //Define MPU_IND_PIN as an OUTPUT - for LED
  pinMode(SD_IND_PIN,OUTPUT); //Define SD_IND_PIN as an OUTPUT - for LED
  pinMode(CS_Pin,OUTPUT); //Define CS_Pin as an OUTPUT - for Chip Select Pin, SPI Communication
  SPI.begin(); //Begin SPI Communication
  Wire.begin(); //Begin I2C Commauniction
  Wire.setClock(1000000L);
  Serial.begin(9600);

  //COMPUTER STARTUP SEQUENCE - CHECKING CONNECTIONS
  //CHECKING BMP180 CONNECTIONS
  if(bmp.begin(BMP085_ULTRALOWPOWER)){
    digitalWrite(BMP_IND_PIN,HIGH); //Will turn on LED if BMP180 is found
    delay(200);
    Serial.println("Found BMP180");
  }

  //CHECKING MPU6050 CONNECTIONS
  if(mpu.begin()){
    digitalWrite(MPU_IND_PIN,HIGH); //Will turn on LED if MPU6050 is found
    delay(200);
    Serial.println("Found MPU6050");  
  }

  //CHECKING SD CARD CONNECTIONS
  if(SD.begin(CS_Pin)){
    digitalWrite(SD_IND_PIN,HIGH); //Will turn on LED if SD Card is found
    delay(200);
    Serial.println("Found SD CARD MODULE");
  }
  else{
    Serial.println("NO SD CARD");
    for(j=1;j<=200;j=j+1){
      digitalWrite(SD_IND_PIN,HIGH);
      delay(100);
      digitalWrite(SD_IND_PIN,LOW);
      delay(100);
      //WILL BLINK LED 3 IF SD CARD IS NOT FOUND
    }
  }

  //FINAL STARTUP SEQUENCE CHECK
  if(SD.begin(CS_Pin) and mpu.begin() and bmp.begin(BMP085_ULTRALOWPOWER)){
    //STARTUP BLINK LED SEQUENCE
    digitalWrite(BMP_IND_PIN,LOW);
    digitalWrite(MPU_IND_PIN,LOW);
    digitalWrite(SD_IND_PIN,LOW);
    delay(200);
    digitalWrite(BMP_IND_PIN,HIGH);
    digitalWrite(MPU_IND_PIN,HIGH);
    digitalWrite(SD_IND_PIN,HIGH);
    delay(1000);
    digitalWrite(BMP_IND_PIN,LOW);
    digitalWrite(MPU_IND_PIN,LOW);
    digitalWrite(SD_IND_PIN,LOW); 
  }//END OF IF STATEMENT
  
  //CONFIGURE SENSOR SETTINGS
   mpu.setAccelerometerRange(MPU6050_RANGE_16_G);
   mpu.setGyroRange(MPU6050_RANGE_500_DEG);
   mpu.setFilterBandwidth(MPU6050_BAND_260_HZ);  

   
}//END OF VOID LOOP
   

void loop(){
  sensors_event_t a, g, temp;
  mpu.getEvent(&a, &g, &temp);
  float accX = (((a.acceleration.x)/9.81)-0.05); //Stores Calibrated X Acceleration Value in g
  Time = millis();
  Serial.print(Time);
  Serial.println("Accel X");
  float accY = (((a.acceleration.y)/9.81)+0.03); //Stores Calibrated Y Acceleration Value in g
  Time = millis();
  Serial.print(Time);
  Serial.println("Accel Y");
  float accZ = (((a.acceleration.z)/9.81)-0.16); //Stores Calibrated Z Acceleration Value in g  
  Time = millis();
  Serial.print(Time);
  Serial.println("Accel Z");
  float gyroX = (g.gyro.x+0.03); //Stores Calibrated X Angular Rate Value in Rad/s
  Time = millis();
  Serial.print(Time);
  Serial.println("Gyro x");
  float gyroY = (g.gyro.y-0.02); //Stores Calibrated Y Angular Rate Value in Rad/s
  Time = millis();
  Serial.print(Time);
  Serial.println("Gyro Y");
  float gyroZ = (g.gyro.z-0.02); //Stores Calirbated Z Angular Rate Value in Rad/s
  Time = millis();
  Serial.print(Time);
  Serial.println("Gyro Z");
  float Temperature = bmp.readTemperature();
  Time = millis();
  Serial.print(Time);
  Serial.println("Temp");
  float Pressure = (bmp.readPressure());
  Time = millis();
  Serial.print(Time);
  Serial.println("Pressure");

  

  
  File dataFile = SD.open("logs.csv", FILE_WRITE); //Create "File" object, referred to as dataFile. Will also create a file on Micro SD Card named "log.txt" with Write argument
  if (dataFile){ //If dataFile could be opened....
    dataFile.print(accX); //Print the first data string onto the MicroSD Card
    dataFile.print(",");
    dataFile.print(accY);
    dataFile.print(",");
    dataFile.print(accZ);
    dataFile.print(",");
    dataFile.print(gyroX);
    dataFile.print(",");
    dataFile.print(gyroY);
    dataFile.print(",");
    dataFile.print(gyroZ);
    dataFile.print(",");
    dataFile.print(Temperature);
    dataFile.print(",");
    dataFile.println(Pressure);
    dataFile.close(); //Close the DataFile
    digitalWrite(SD_IND_PIN,HIGH);
    Time = millis();
    Serial.println(Time);
    Serial.println("DATA LOGGED");
    Serial.println("");
  }

  else{
    digitalWrite(BMP_IND_PIN,HIGH); //INDICATES FAILURE OF OPENING SD CARD
    digitalWrite(MPU_IND_PIN,HIGH);
    Serial.println("OPENING DATAFILE IS UNSUCCESSFUL");
    delay(20000);
  }

  if(Increment == 500){ //If Micro SD Card uploads have occurred 18 times, turn on the LED
    Serial.println("Safe to remove Micro-USB"); //Will indicate on Serial when it is safe to remove the Micro-USB cable from the Teensy LC
    dataFile.close(); //Close Micro SD Card, just to be safe
    digitalWrite(BMP_IND_PIN,HIGH);
    digitalWrite(MPU_IND_PIN,HIGH);
    digitalWrite(SD_IND_PIN,HIGH);
    delay(20000); //Delay for 20 seconds, give ample time to disconnect Teensy
    Serial.println("Unsafe to remove Micro-USB"); //Will indicate that it is unsafe to remove the Micro-USB cable
    dataFile.close(); //Close the DataFile
  }
  
  else{
    Increment = Increment + 1; //Increment
  }

  digitalWrite(SD_IND_PIN,LOW);
}

Thank you in advance,
Dawn

2

The easiest is to make the file object 'global'

File dataFile;

make it a global variable.
Open it in setup()

dataFile = SD.open("logs.csv", FILE_WRITE);

and use it as normal.

It is possible you are also overflowing the Serial-TX buffer, which would slows things down.

Serial.begin(9600);

Increase that rate, up to 1Mbps is supported on an AVR using hwSerial. why waste the time ?

3

But you will be back with the same question when you discover the time it takes to write the 512 byte block of data to the card and update the directory record on the card with the new file size information.
That cannot be avoided when you are using an SD card.

4

...only issue is that data is first written to a 512 byte buffer, before being physically written to the file... and if you are not closing the file, then on power loss you may lose some data. If this is acceptable then no issue.

5

Sorry to mention - I am a beginner at this entire thing.

From @red_car comment, I assume that due to the structure of my code, it is being stored in the 512 byte buffer (in the Arduino?) and then afterwards, it is uploaded onto the SD Card.

I attempted the solution proposed by @Deva_Rishi and the if (dataFile){} function is returning false, and thus the data cannot be uploaded to the SD Card.

And @Paul_KD7HB, due to my lack of knowledge, I am a bit unsure as to what you're referring to? Have I hit some sort of limit for the data logging speed due to the innate nature of SD Cards?

Thank you for all the responses,
Dawn

6

try to just declare and assign in one go.

 File dataFile = SD.open("logs.csv", FILE_WRITE);

as a global declaration. Maybe the object needs to be declared with specs

7

I just attempted that and unfortunately, no luck.

8

An SD card is a solid state version of the old floppy disk on ancient personal computers. Data is always stored in 512 byte blocks on the SD card. All of your writes to the SD card are done in the Arduino memory until the 512 byte buffer limit is reached. Then all of your code stops until the 512 byte block is written to the SD card, the directory record is updated with your new file length data and the 512 byte record is rewritten to the SD card.

This is identical to what you are are already doing when you close the file after each writing of your data. But you then open the file, which cause the reverse of the close and that needs about the same amount of time to complete as the close did.

My point is you are only eliminating the time take to open the SD card file. All other times remain the same.

Think some about how you will tell your program it is time to finally close your file and stop processing. Likely, right now you are just turning to power off, by that will not save your last block of data when you turn the power off without closing the file.

9

People who understand the issues use a button or some other manual input to start and stop data recording, and have some indicator showing the "record" status, like an LED.

It is a very bad idea to open the file, write a bit of data and close it again, as that vastly increases power consumption and error rate, and dramatically slows down data collection.

But you see this in almost all tutorials, because those tutorials are all written by people who don't understand what they are doing.

A good approach is to push a button, open the file and if successful, light the "record" LED and start collecting and storing data. Push the button to close the file and turn off the LED. Do correct for switch bounce.

10

Perhaps the application might yield some other solutions.

I am using this data logging method for a flight computer. The flight computer will be powered on through a power switch (toggles power to PCB), and this will be turned on approximately 7 seconds before flight. The flight itself lasts only 5ish seconds, after which the computer continues to run on the ground.

As the system is set up currently, I just have an increment variable which waits until 500 readings are complete. I do not mind if after touchdown I have to wait until the flight computer gets the rest of the 500 readings and closes the file (through the code). And then I can manually power off the computer.

11

That is a perfectly acceptable solution.

12

I am still unsure as to how to implement that though. The delayed data logging will still be a problem if I continually open and close the file in void loop().

13

You will not be doing that.

In outline, this could even be in setup():

SD.open(file);
for (int i=0; i<500; i++) {
get_data();
SD.write(data);
delay(some_time); //if needed
}
SD.close(file);

Best to make sure you understand the ins and outs of data collection and make many practice runs, before a flight.

14

I got that flying part backwards!

I have already got a flight in, and that taught me a lot about the optimizations I have to make.

Right now I am working on increasing the frequency of the data logging. I will try to implement the solution you have suggested, and I will let you know that goes.

I really appreciate the input!

15

Ah yeah sorry about that, First the sd.begin() needs to be called.
how about like this ?

#include <Adafruit_BMP085.h> //Include Adafuit BMP180 Library
#include <Adafruit_MPU6050.h> //Include Adafruit MPU6050 Library
#include <Adafruit_Sensor.h> //Include General Adafruit Sensor Library
#include <SD.h> //Include the SD Card Library
#include <SPI.h> //Include the SPI Communication Library
#include <Wire.h> //Include I2C Communication Lirary

Adafruit_MPU6050 mpu; //Define MPU Word as "mpu"
Adafruit_BMP085 bmp; //Define BMP Word as "bmp"

int CS_Pin = 10; //Define Chip Select Pin for SD Reader as Digital Pin 10 (as per PCB)
int BMP_IND_PIN = 4; //Define LED Indicator Pin for BMP Indicator LED as Digital Pin 4 (as per PCB)
int MPU_IND_PIN = 5; //Define LED Indicator Pin for MPU Indicator LED as Digital Pin 5 (as per PCB)
int SD_IND_PIN = 6; //Define LED Indicator Pin for SD Card Indicator LED as Digital Pin 6 (as per PCB)
int Increment = 0; //Define Increment variable as means for a timer
int j = 0; //Define j as increment for loops for indicating error of SD Card
unsigned long Time;

#define BMP085_ULTRAHIGHRES 3

void setup() {
  //DEFINING OUTPUTS
  pinMode(BMP_IND_PIN, OUTPUT); //Define BMP_IND_PIN as an OUTPUT - for LED
  pinMode(MPU_IND_PIN, OUTPUT); //Define MPU_IND_PIN as an OUTPUT - for LED
  pinMode(SD_IND_PIN, OUTPUT); //Define SD_IND_PIN as an OUTPUT - for LED
  pinMode(CS_Pin, OUTPUT); //Define CS_Pin as an OUTPUT - for Chip Select Pin, SPI Communication
  SPI.begin(); //Begin SPI Communication
  Wire.begin(); //Begin I2C Commauniction
  Wire.setClock(1000000L);
  Serial.begin(9600);

  //COMPUTER STARTUP SEQUENCE - CHECKING CONNECTIONS
  //CHECKING BMP180 CONNECTIONS
  if (bmp.begin(BMP085_ULTRALOWPOWER)) {
    digitalWrite(BMP_IND_PIN, HIGH); //Will turn on LED if BMP180 is found
    delay(200);
    Serial.println("Found BMP180");
  }

  //CHECKING MPU6050 CONNECTIONS
  if (mpu.begin()) {
    digitalWrite(MPU_IND_PIN, HIGH); //Will turn on LED if MPU6050 is found
    delay(200);
    Serial.println("Found MPU6050");
  }

  //CHECKING SD CARD CONNECTIONS
  if (SD.begin(CS_Pin)) {
    digitalWrite(SD_IND_PIN, HIGH); //Will turn on LED if SD Card is found
    delay(200);
    Serial.println("Found SD CARD MODULE");
  }
  else {
    Serial.println("NO SD CARD");
    for (j = 1; j <= 200; j = j + 1) {
      digitalWrite(SD_IND_PIN, HIGH);
      delay(100);
      digitalWrite(SD_IND_PIN, LOW);
      delay(100);
      //WILL BLINK LED 3 IF SD CARD IS NOT FOUND
    }
  }

  //FINAL STARTUP SEQUENCE CHECK
  if (SD.begin(CS_Pin) and mpu.begin() and bmp.begin(BMP085_ULTRALOWPOWER)) {
    //STARTUP BLINK LED SEQUENCE
    digitalWrite(BMP_IND_PIN, LOW);
    digitalWrite(MPU_IND_PIN, LOW);
    digitalWrite(SD_IND_PIN, LOW);
    delay(200);
    digitalWrite(BMP_IND_PIN, HIGH);
    digitalWrite(MPU_IND_PIN, HIGH);
    digitalWrite(SD_IND_PIN, HIGH);
    delay(1000);
    digitalWrite(BMP_IND_PIN, LOW);
    digitalWrite(MPU_IND_PIN, LOW);
    digitalWrite(SD_IND_PIN, LOW);
  }//END OF IF STATEMENT

  //CONFIGURE SENSOR SETTINGS
  mpu.setAccelerometerRange(MPU6050_RANGE_16_G);
  mpu.setGyroRange(MPU6050_RANGE_500_DEG);
  mpu.setFilterBandwidth(MPU6050_BAND_260_HZ);


}//END OF VOID LOOP


void loop() {
  File dataFile = SD.open("logs.csv", FILE_WRITE); //Create "File" object, referred to as dataFile. Will also create a file on Micro SD Card named "log.txt" with Write argument
  while (1) {
    sensors_event_t a, g, temp;
    mpu.getEvent(&a, &g, &temp);
    float accX = (((a.acceleration.x) / 9.81) - 0.05); //Stores Calibrated X Acceleration Value in g
    Time = millis();
    Serial.print(Time);
    Serial.println("Accel X");
    float accY = (((a.acceleration.y) / 9.81) + 0.03); //Stores Calibrated Y Acceleration Value in g
    Time = millis();
    Serial.print(Time);
    Serial.println("Accel Y");
    float accZ = (((a.acceleration.z) / 9.81) - 0.16); //Stores Calibrated Z Acceleration Value in g
    Time = millis();
    Serial.print(Time);
    Serial.println("Accel Z");
    float gyroX = (g.gyro.x + 0.03); //Stores Calibrated X Angular Rate Value in Rad/s
    Time = millis();
    Serial.print(Time);
    Serial.println("Gyro x");
    float gyroY = (g.gyro.y - 0.02); //Stores Calibrated Y Angular Rate Value in Rad/s
    Time = millis();
    Serial.print(Time);
    Serial.println("Gyro Y");
    float gyroZ = (g.gyro.z - 0.02); //Stores Calirbated Z Angular Rate Value in Rad/s
    Time = millis();
    Serial.print(Time);
    Serial.println("Gyro Z");
    float Temperature = bmp.readTemperature();
    Time = millis();
    Serial.print(Time);
    Serial.println("Temp");
    float Pressure = (bmp.readPressure());
    Time = millis();
    Serial.print(Time);
    Serial.println("Pressure");

    if (dataFile) { //If dataFile could be opened....
      dataFile.print(accX); //Print the first data string onto the MicroSD Card
      dataFile.print(",");
      dataFile.print(accY);
      dataFile.print(",");
      dataFile.print(accZ);
      dataFile.print(",");
      dataFile.print(gyroX);
      dataFile.print(",");
      dataFile.print(gyroY);
      dataFile.print(",");
      dataFile.print(gyroZ);
      dataFile.print(",");
      dataFile.print(Temperature);
      dataFile.print(",");
      dataFile.println(Pressure);
      dataFile.close(); //Close the DataFile
      digitalWrite(SD_IND_PIN, HIGH);
      Time = millis();
      Serial.println(Time);
      Serial.println("DATA LOGGED");
      Serial.println("");
    }

    else {
      digitalWrite(BMP_IND_PIN, HIGH); //INDICATES FAILURE OF OPENING SD CARD
      digitalWrite(MPU_IND_PIN, HIGH);
      Serial.println("OPENING DATAFILE IS UNSUCCESSFUL");
      delay(20000);
    }

    if (Increment == 500) { //If Micro SD Card uploads have occurred 18 times, turn on the LED
      Serial.println("Safe to remove Micro-USB"); //Will indicate on Serial when it is safe to remove the Micro-USB cable from the Teensy LC
      dataFile.close(); //Close Micro SD Card, just to be safe
      digitalWrite(BMP_IND_PIN, HIGH);
      digitalWrite(MPU_IND_PIN, HIGH);
      digitalWrite(SD_IND_PIN, HIGH);
      delay(20000); //Delay for 20 seconds, give ample time to disconnect Teensy
      Serial.println("Unsafe to remove Micro-USB"); //Will indicate that it is unsafe to remove the Micro-USB cable
      //dataFile.close(); //Close the DataFile
    }

    else {
      Increment = Increment + 1; //Increment
    }

    digitalWrite(SD_IND_PIN, LOW);
  }
}

I am a little unsure about how you end the process, particularly since increment is never reset, but that is a different matter.
Basically you need to create the object and let it not go out of scope as a first priority.

16

Update:

I attempted to create a code akin to the structure provided by @jremington

This resulted in a code as:

#include <Wire.h>
#include <SPI.h>
#include <SD.h>
int CS_Pin = 10; //Define Chip Select Pin for SD Reader as Digital Pin 10 (as per PCB)
int Increment = 1;
int d = 0;

void setup() {
  Serial.begin(9600);
  SPI.begin();
  SD.begin();
  pinMode(CS_Pin,OUTPUT);
  File dataFile =  SD.open("logs.csv", FILE_WRITE); //Create "File" object, referred to as dataFile. Will also create a file on Micro SD Car
  
  if(SD.begin(CS_Pin)){
    Serial.println("Found SD CARD MODULE");
  }
  else{
    Serial.println("NO SD CARD");
  }   

  for(d = 1; d <100; d++){
    int data = 8008135;
    if (dataFile){
      dataFile.println(data);
      Serial.println("DATA LOGGED");
      Serial.println("");
    }

    else{
      Serial.println("OPENING DATAFILE IS UNSUCCESSFUL");
      delay(20000);
    }
  }
}

void loop() {
}

Correct me if I am mistaken, but the code above only opens and closes the file once.

I am now attempting to implement this same structure with the main code.

17

You need to reorder the steps SD.begin() and SD.open().

You don't ever close the file.

You should check for success immediately after opening the file, not in the program segment that attempts to write data to the file.

I am now attempting to implement this same structure with the main code

Don't, because it is mostly wrong. Get the posted program working perfectly, and check that all data are stored correctly, before moving on.

Hint: rather than writing an arbitrary constant in each record, write the record number (for example, the loop index d). That way you know, when reading the file back, whether every record was written properly.

18

Thank you very much for the feedback.

  1. My mistake for forgetting to close the file. I added that in this iteration of the code, after the for() loop.

  2. The reason I am writing the data after the if() statement is for troubleshooting, post-flight. During the flight I just made, I expected 500 data lines, but ended up with only 243. I think this is due to the fact that I hit the ground a little rough, perhaps dislodging the SD Card from its slot. If I stick with this code structure, I know for a fact that the issue is due the SD Card getting dislodged, if I encounter such an error again.

  3. I am unsure why I need to reorder the SD.begin() and SD.open()

#include <Wire.h>
#include <SPI.h>
#include <SD.h>
int CS_Pin = 10; //Define Chip Select Pin for SD Reader as Digital Pin 10 (as per PCB)
int Increment = 1;
int d = 0;

void setup() {
  Serial.begin(9600);
  SPI.begin();
  SD.begin();
  pinMode(CS_Pin,OUTPUT);
  File dataFile =  SD.open("logs.csv", FILE_WRITE); //Create "File" object, referred to as dataFile. Will also create a file on Micro SD Car
  
  if(SD.begin(CS_Pin)){
    Serial.println("Found SD CARD MODULE");
  }
  else{
    Serial.println("NO SD CARD");
  }   

  for(d = 1; d <100; d++){
    if (dataFile){
      dataFile.println(d);
      Serial.println("DATA LOGGED");
      Serial.println("");
    }

    else{
      Serial.println("OPENING DATAFILE IS UNSUCCESSFUL");
      delay(20000);
    }
  }

  dataFile.close();
}

void loop() {
}

I used this code to log the D (loop counter) into the SD Card, and I got the integers 1 through 99 onto the card. Thanks for the idea.

19

The library requires that particular order. Study the examples and documentation.

begin() establishes communications and initializes the SD card.

open() opens a particular file for reading or writing.

20

I believe that is the format seen in the current code...