Storing data to SD-Card in under 8ms - problems understanding SDFAT library

[xeonus]

Greetings community,

so I would like to give you a short introduction to my problem and thanks for taking your time!
I am using an IMU (BNO055) with an MKR Zero Board.
I want to store data on that card. The data can look like this: 359,22114,-6,2,-6,0,2,-6,,2225,-40,-159,0,0,0,0
I want to store it column per column. And the data output should be around 100 hz (every 10ms).
At first i was working with a buffer, but now, I would like to store every column on its one. Because the sensor delivers new data every 10 ms, we got time in between to store the data, without missing any data.

Without any optimization, it takes about 25ms to store the data on the SD-Card.
I tried to implement the SD-Fat library by adding something like :
if (!myFile.open("test.txt", O_RDWR | O_CREAT | O_AT_END)) {

But its now even slower?
My guess is, that it is not done right, and I need to add more. I already looked into some examples and the documentation, but i am still not sure what steps I need to take to make this a success. If you got some time and wisdom, i would be very pleased if you can share it with me. I also think, that my code atm isn't that quick because of some "if's" in it, which are maybe unnecessary.

In the end i would like to combine the different outputs of the sensor to a column like:
359,22114,-6,2,-6,0,2,-6,,2225,-40,-159,0,0,0,0
And store every time this column separately to the sd-card as fast as possible to use the 100 hz the sensor is having.

Yeah i know, saving the information binary is also a good thing, which I am already trying to add. In addition, I am also trying a faster SD-Card.

Here my code, auto formatted. It may look quite long, but most of it is just configuring the data from the sensor.

I would be glad if you could explain my how to use the SD-Fat library and further opitmize my code for speed.

Thanks for your help and time!

#include <Wire.h>
#include <Adafruit_Sensor.h>
#include <Adafruit_BNO055.h>
#include <utility/imumaths.h>
#include <SPI.h>
#include <SD.h>
#include <SPI.h>
#include "SdFat.h"
#include <avr/dtostrf.h>
#include "BNO055_support.h"


// Check I2C device address and correct line below (by default address is 0x29 or 0x28)
//                                   id, address
Adafruit_BNO055 bno = Adafruit_BNO055(55, 0x28);

const int SAMPLE_RATE = 100; //100 Hz
const int SAMPLE_TIME = 1000 / SAMPLE_RATE; // Milisekunde
const char dataFileName[] = "test20.txt";

const int SAVE_TO_SD_TIME = 0.01; //Sekunden
const int NUMBER_OF_SAMPLES = SAMPLE_RATE * SAVE_TO_SD_TIME; //Wie viele Daten eingelesen werde bevor auf SD gespeichert

File file;
const int chipSelect = 28; // CS Pin für SD
int last_data_read = 0; //Speichert letztes Dateneinlesen
char dataString[16384]; //Buffert/Speichert Daten
int counter = 0; //Zählt wie oft Daten eingelesen

struct bno055_euler myEulerData; //Structure to hold the Euler data
struct bno055_t myBNO;

// ------------------ initializeSD&Sensor ----------------------------------
void setup(void)
{
  //Initialize I2C communication
  Wire.begin();

  //Initialization of the BNO055
  BNO_Init(&myBNO); //Assigning the structure to hold information about the device

  //Configuration to NDoF mode
  bno055_set_operation_mode(OPERATION_MODE_NDOF);

  delay(1);
  Serial.begin(19200);

  //Wartet dass Serialer Monitor geöffnet wird
  //Lösch mich für Feldmessungen
  while (!Serial) {}

  file = SD.open(dataFileName, O_CREAT | O_WRITE);

  Serial.print("Number of Samples:");
  Serial.println(NUMBER_OF_SAMPLES);

  // SD-Card Initialise
  Serial.print("Initializing SD card...");
  if (!SD.begin(28)) {
    Serial.println("initialization failed!");
    while (1);
  }
  Serial.println("initialization done.");
}

void loop(void)
{
  // Sensor Calibration
  uint8_t system, gyro, accel, mag = 0;
  bno.getCalibration(&system, &gyro, &accel, &mag);

  //Wenn die System-Kalibration auf 3 ist leuchtet eine LED
  if (system == 3) {
    digitalWrite(LED_BUILTIN, HIGH);
    delay(500);
    digitalWrite(LED_BUILTIN, LOW);
    delay(500);
  } else {
    digitalWrite(LED_BUILTIN, LOW);
  }

  //Einlesen und speichern

  if (millis() -  last_data_read >= SAMPLE_TIME) {
    last_data_read = millis();

    storeValues();
  }
}

//Ließt Werte ein und speichert alle x Werte nach y auf die SD
void storeValues() {

  sensors_event_t orientationData , angVelocityData , linearAccelData, magnetometerData, accelerometerData, gravityData;
  bno.getEvent(&orientationData, Adafruit_BNO055::VECTOR_EULER);
  bno.getEvent(&angVelocityData, Adafruit_BNO055::VECTOR_GYROSCOPE);
  bno.getEvent(&linearAccelData, Adafruit_BNO055::VECTOR_LINEARACCEL);
  bno.getEvent(&magnetometerData, Adafruit_BNO055::VECTOR_MAGNETOMETER);
  bno.getEvent(&accelerometerData, Adafruit_BNO055::VECTOR_ACCELEROMETER);
  bno.getEvent(&gravityData, Adafruit_BNO055::VECTOR_GRAVITY);

  counter = counter + 1;

  //--------------------add to the array-----------------------------------

  // setting up the struct

  //  struct datastore {
  //    uint16_t count_c;
  //    struct datastore myData;
  //  uint16_t timestamp;
  //  uint16_t accelerometerX;
  //  uint16_t accelerometerY;
  //  uint16_t accelerometerZ;
  //  uint16_t gyroscopeX;
  //  uint16_t gyroscopeY;
  //  uint16_t gyroscopeZ;
  //  };

  //Speichert count
  char count_c[16];
  itoa(counter, count_c, 10);

  strcat(dataString, count_c);
  strcat(dataString, ",");

  //Speichert Zeit
  int t = millis();
  char timestamp[16];
  itoa(t, timestamp, 10); //Konvertiert int to char[]
  strcat(dataString, timestamp);
  strcat(dataString, ",");

  /*______ACCELEROMETER_____*/
  imu::Vector<3> acc = bno.getVector(Adafruit_BNO055::VECTOR_ACCELEROMETER);

  int ax = int(acc.x());
  char accelerometerX[8];
  itoa(ax, accelerometerX, 10); //Konvertiert int to char[]

  int ay = int(acc.y());
  char accelerometerY[8];
  itoa(ay, accelerometerY, 10); //Konvertiert int to char[]

  int az = int(acc.z());
  char accelerometerZ[8];
  itoa(az, accelerometerZ, 10); //Konvertiert int to char[]

  strcat(dataString, accelerometerX);
  strcat(dataString, ",");
  strcat(dataString, accelerometerY);
  strcat(dataString, ",");
  strcat(dataString, accelerometerZ);
  strcat(dataString, ",");

  /*______GYROSCOPE_____*/
  imu::Vector<3> gyr = bno.getVector(Adafruit_BNO055::VECTOR_GYROSCOPE);

  int gx = int(gyr.x());
  char gyroscopeX[8];
  itoa(gx, gyroscopeX, 10); //Konvertiert int to char[]

  int gy = int(acc.y());
  char gyroscopeY[8];
  itoa(gy, gyroscopeY, 10); //Konvertiert int to char[]

  int gz = int(acc.z());
  char gyroscopeZ[8];
  itoa(gz, gyroscopeZ, 10); //Konvertiert int to char[]

  strcat(dataString, gyroscopeX);
  strcat(dataString, ",");
  strcat(dataString, gyroscopeY);
  strcat(dataString, ",");
  strcat(dataString, gyroscopeZ);
  strcat(dataString, ",");

  /*______System Calibration_____*/
  uint8_t system, gyro, accel, mag = 0;
  bno.getCalibration(&system, &gyro, &accel, &mag);

  int systemcali = int(system);
  char systemcalibration[10];
  itoa(systemcali, systemcalibration, 10); //Konvertiert int to char[]

  strcat(dataString, ",");
  strcat(dataString, systemcalibration);
  Serial.print (systemcalibration);
  /*______VECTOR_EULER_____*/
  imu::Vector<3> euler = bno.getVector(Adafruit_BNO055::VECTOR_EULER);

  int ex = int(euler.x());
  char eulerX[8];
  itoa(ex, eulerX, 10); //Konvertiert int to char[]

  int ey = int(euler.y());
  char eulerY[8];
  itoa(ey, eulerY, 10); //Konvertiert int to char[]

  int ez = int(euler.z());
  char eulerZ[8];
  itoa(ez, eulerZ, 10); //Konvertiert int to char[]

  strcat(dataString, eulerX);
  strcat(dataString, ",");
  strcat(dataString, eulerY);
  strcat(dataString, ",");
  strcat(dataString, eulerZ);
  strcat(dataString, ",");

  /*______Quaternion_____*/
  imu::Quaternion quat = bno.getQuat();

  int qw = int(quat.w());
  char quaternionW[8];
  itoa(qw, quaternionW, 10); //Konvertiert int to char[]

  int qx = int(quat.y());
  char quaternionX[8];
  itoa(qx, quaternionX, 10); //Konvertiert int to char[]

  int qy = int(quat.x());
  char quaternionY[8];
  itoa(qy, quaternionY, 10); //Konvertiert int to char[]

  int qz = int(quat.z());
  char quaternionZ[6];
  itoa(qz, quaternionZ, 10); //Konvertiert int to char[]

  strcat(dataString, quaternionW);
  strcat(dataString, ",");
  strcat(dataString, quaternionX);
  strcat(dataString, ",");
  strcat(dataString, quaternionY);
  strcat(dataString, ",");
  strcat(dataString, quaternionZ);
  strcat(dataString, "\n");


  //Abspeichern auf SD nach NUMBER_OF_SAMPLES eingelesenen Daten
  //  if (counter >= NUMBER_OF_SAMPLES) {

  writeStringToSD(dataString, dataFileName); //Speichert auf SD

  Serial.println("DONE  ");

  //Zurücksetzen der Variablen
  memset(dataString, 0, sizeof(dataString)); //Speichert 0er in Datenarray



  //  }
}

// ------------------ writeStringToSD ----------------------------------
//Ein belibiger String wird in ein belibiges Files geschrieben

boolean writeStringToSD(const char *data, const char *filename) {
  stopWatch(); //Startet Stoppuhr
  Serial.print("Writing to: ");
  Serial.println(filename);
  file = SD.open(filename, FILE_WRITE); //Öffnet file zum Lesen

  if (file) { //Wenn File erfolgreich geöffnet wurde
    file.println(data); //Schreibt Datenstring ans Ende des Files
  }
  else {
    Serial.println("Error while opening the file!");
    return false;
  }

  file.close(); //Schließt File (Daten werden gesichert)
  stopWatch(); //Beendet Stoppuhr
  return true;

}

// ------------------ Stoppuhr ----------------------------------
boolean stopWatch_started = false;
int stopWatch_time;
void stopWatch() {
  if (stopWatch_started == false) {
    stopWatch_time = millis();
    stopWatch_started = true;
  }
  else {
    int duration = millis() - stopWatch_time;
    Serial.print("Duartion: ");
    Serial.print(duration);
    Serial.println(" ms");

    stopWatch_started = false;
  }
}

//Systemcalibration aufzeichnen geht nicht und aufnahmefrequenz ist bei etwa 35ms

[LightuC]

I just quickly went over your code and here are some suggestions:

1. Check your datatypes! For example this line doesn't make sense at all, because an integer (int) can only store numbers without decimals (so only -3, -2, -,1 0,1,2,3,4 and so on). The Compiler will put 0 into SAVE_TO_SD_TIME.

const int SAVE_TO_SD_TIME = 0.01; //Sekunden

2. Why do you need a 16kB Buffer if you only want to store something like this?

359,22114,-6,2,-6,0,2,-6,2225,-40,-159,0,0,0,0

3. I am german myself, so I do understand the comments, but in generel keep the comments in englisch so everybody understands them
4. In your writeStringToSD function, you re-open the file. Why is that? You already opened it in the setup function (which btw. should be after the call to SD.begin()). Also when opening a file the fat driver needs to search for the file, which takes some time. There is no point in doing so.
   You can use flush() after a write to make sure that the write is executed immediately (I think the fat driver does buffered writes, so not every call to write results in an actual write to the sd card)
5. This is the biggest point: Because the SD Card uses Flash memory, the SD controller will have to do some wear-leveling eventually. This will take a long time (up to 250ms according to this forum post https://chipkit.net/forum/viewtopic.php?t=3035#p12554) and it can occur at any time.

So my suggestion is, that you buffer your data first, and then write it to the sd card in larger chunks. Collect the sensor data inside a timer interrupt, firing at 100Hz, so that you don't miss out on sensor data during writes to the sd card.

[xeonus]

Thank you for your suggestions, i will work through them!

About the buffer. I already tried it with i buffer, that's the reason the 16kb buffer is still in there. I managed to store the data every 8secs. But then it took about 250 ms to store all that data, which was too long, because i need stable sample frequencies. But that was also without using the SDFat library. In theory my SD-Card can write up to 7 mb/sec.

[Paul_KD7HB]

Remember, the SD card is ALWAYS read/writes in 512 byte blocks. Greatest speed will be to use several 512 byte buffers and rotate through them, by writing one and moving to the next and begin filling it and then back to the first buffer or perhaps a third buffer. All being 512 bytes long.
Just like using the old floppy disks!
Paul

[drmpf]

Check out my project on High Speed, Long Duration Datalogging

[xeonus]

Is it possible with the arduino to write a buffer to the sd card and store data in another buffer at the same time? I thought a MC like the arduino can only do 1 task at a time. And while writing to the sd Card, there is no computing power left to buffer.

[LightuC]

Yes it is possible, if one of the writes is executed within an interrupt routine. The other way would be to use the dma, since you are using the samd21.

[xeonus]

I am not completely sure how to implement many buffers at the same time. At the moment I am using a buffer, which i can configure to any size i want. 1 sec of samples or just 0,01 sec (=1 sample). Maximum, because of the available ram is 7 Secs of buffering.

At the moment, it takes about 15 ms to store one column of data (=1 sample)
Did got the correct idea on that? If we lets say take 3 buffers, we can store in the time where the "old" sample is written, new samples in a buffer?

I think i implemented SDFAT library. But again, I am not sure how to use Flush and close correctly.
Can I flush like every sample and lets say every x milliseconds (extra timer) i use close?

I also thought, that your help maybe more effectiv, by explained via viocechat or screensharing.
If you are comfortable with that ..

Any help is very welcome.

Here is my latest code, comments are now in english:

#include <Wire.h>
#include <Adafruit_Sensor.h>
#include <Adafruit_BNO055.h>
#include <utility/imumaths.h>
#include <SPI.h>
#include <SD.h>
#include <avr/dtostrf.h>
#include "BNO055_support.h"
#include <SdFat.h>
#include "SdFat.h"

// Check I2C device address and correct line below (by default address is 0x29 or 0x28)
//                                   id, address
Adafruit_BNO055 bno = Adafruit_BNO055(55, 0x28);

const int SAMPLE_RATE = 100; //100 Hz
const int SAMPLE_TIME = 1000 / SAMPLE_RATE; // Milliseconds
const char dataFileName[] = "test4.txt";

const float SAVE_TO_SD_TIME = 0.01; //Sekunden
const int NUMBER_OF_SAMPLES = SAMPLE_RATE * SAVE_TO_SD_TIME; //how many samples are read in before being saved to the SD

File file;
const int chipSelect = 28; // CS Pin for SD
int last_data_read = 0; // variable for the last time data was read
char dataString[16384]; //Array to buffer data | 16384 because of the size of the SRAM
int counter = 0; //counts da columns with values (samples)

struct bno055_euler myEulerData; //Structure to hold the Euler data
struct bno055_t myBNO;

// ------------------ initializeSD&Sensor ----------------------------------
void setup(void)
{
  //Initialize I2C communication
  Wire.begin();

  //Initialization of the BNO055
  BNO_Init(&myBNO); //Assigning the structure to hold information about the device

  //Configuration to NDoF mode
  bno055_set_operation_mode(OPERATION_MODE_NDOF);

  delay(1);
  Serial.begin(19200);

  //wait for the serial monitor
  //delete line below for real measurements
  while (!Serial) {}

  Serial.print("Number of Samples:");
  Serial.println(NUMBER_OF_SAMPLES);

  // SD-Card Initialise
  Serial.print("Initializing SD card...");
  if (!SD.begin(28)) {
    Serial.println("initialization failed!");
    while (1);
  }
  Serial.println("initialization done.");
}

void loop(void)
{
  // Sensor Calibration
  uint8_t system, gyro, accel, mag = 0;
  bno.getCalibration(&system, &gyro, &accel, &mag);

  //  Serial.println (system);
  //  //Wenn die System-Kalibration auf 3 ist leuchtet eine LED
  //  if (system == 3) {
  //    digitalWrite(LED_BUILTIN, HIGH);
  //    delay(500);
  //    digitalWrite(LED_BUILTIN, LOW);
  //    delay(500);
  //  } else {
  //    digitalWrite(LED_BUILTIN, LOW);
  //  }

  //Storing values with timer

  if (millis() -  last_data_read >= SAMPLE_TIME) {
    last_data_read = millis();
    //Serial.println("Storing values"); // just to test
    storeValues();
  }
}

//Ließt Werte ein und speichert alle x Werte nach y auf die SD
void storeValues() {

  sensors_event_t orientationData , angVelocityData , linearAccelData, magnetometerData, accelerometerData, gravityData;
  bno.getEvent(&orientationData, Adafruit_BNO055::VECTOR_EULER);
  bno.getEvent(&angVelocityData, Adafruit_BNO055::VECTOR_GYROSCOPE);
  bno.getEvent(&linearAccelData, Adafruit_BNO055::VECTOR_LINEARACCEL);
  bno.getEvent(&magnetometerData, Adafruit_BNO055::VECTOR_MAGNETOMETER);
  bno.getEvent(&accelerometerData, Adafruit_BNO055::VECTOR_ACCELEROMETER);
  bno.getEvent(&gravityData, Adafruit_BNO055::VECTOR_GRAVITY);

  counter = counter + 1;

  //--------------------add to the array-----------------------------------

  //saves the column counter
  char count_c[16];
  itoa(counter, count_c, 10);

  strcat(dataString, count_c); //adds it to the array
  strcat(dataString, ",");

  //Speichert Zeit
  int t = millis();
  char timestamp[16];
  itoa(t, timestamp, 10); //converts int to char[]

  strcat(dataString, timestamp);
  strcat(dataString, ",");

  /*______ACCELEROMETER_____*/
  imu::Vector<3> acc = bno.getVector(Adafruit_BNO055::VECTOR_ACCELEROMETER);

  int ax = int(acc.x());
  char accelerometerX[6];
  itoa(ax, accelerometerX, 10); //converts int to char[]

  int ay = int(acc.y());
  char accelerometerY[6];
  itoa(ay, accelerometerY, 10); //converts int to char[]

  int az = int(acc.z());
  char accelerometerZ[6];
  itoa(az, accelerometerZ, 10); //converts int to char[]

  strcat(dataString, accelerometerX);
  strcat(dataString, ",");
  strcat(dataString, accelerometerY);
  strcat(dataString, ",");
  strcat(dataString, accelerometerZ);
  strcat(dataString, ",");

  /*______GYROSCOPE_____*/
  imu::Vector<3> gyr = bno.getVector(Adafruit_BNO055::VECTOR_GYROSCOPE);

  int gx = int(gyr.x());
  char gyroscopeX[6];
  itoa(gx, gyroscopeX, 10); //converts int to char[]

  int gy = int(acc.y());
  char gyroscopeY[6];
  itoa(gy, gyroscopeY, 10); //converts int to char[]

  int gz = int(acc.z());
  char gyroscopeZ[6];
  itoa(gz, gyroscopeZ, 10); //converts int to char[]

  strcat(dataString, gyroscopeX);
  strcat(dataString, ",");
  strcat(dataString, gyroscopeY);
  strcat(dataString, ",");
  strcat(dataString, gyroscopeZ);
  strcat(dataString, ",");

  /*______System Calibration_____*/
  uint8_t system, gyro, accel, mag = 0;
  bno.getCalibration(&system, &gyro, &accel, &mag);

  int systemcali = int(system);
  char systemcalibration[10];
  itoa(systemcali, systemcalibration, 10); //Konvertiert int to char[]

  strcat(dataString, ",");
  strcat(dataString, systemcalibration);

  /*______VECTOR_EULER_____*/
  imu::Vector<3> euler = bno.getVector(Adafruit_BNO055::VECTOR_EULER);

  int ex = int(euler.x());
  char eulerX[6];
  itoa(ex, eulerX, 10); //Konvertiert int to char[]

  int ey = int(euler.y());
  char eulerY[6];
  itoa(ey, eulerY, 10); //Konvertiert int to char[]

  int ez = int(euler.z());
  char eulerZ[6];
  itoa(ez, eulerZ, 10); //Konvertiert int to char[]

  strcat(dataString, eulerX);
  strcat(dataString, ",");
  strcat(dataString, eulerY);
  strcat(dataString, ",");
  strcat(dataString, eulerZ);
  strcat(dataString, ",");

  /*______Quaternion_____*/
  imu::Quaternion quat = bno.getQuat();


  char quaternionW[8]; // Buffer big enough for 7-character float
  dtostrf(quat.w(), 6, 3, quaternionW); // Leave room for too large numbers!

  char quaternionX[8]; // Buffer big enough for 7-character float
  dtostrf(quat.y(), 6, 3, quaternionX); // Leave room for too large numbers!

  char quaternionY[8]; // Buffer big enough for 7-character float
  dtostrf(quat.x(), 6, 3, quaternionY); // Leave room for too large numbers!

  char quaternionZ[8]; // Buffer big enough for 7-character float
  dtostrf(quat.z(), 6, 3, quaternionZ); // Leave room for too large numbers!


  //  int qw = int(quat.w());
  //  char quaternionW[6];
  //  itoa(qw, quaternionW, 10); //converts int to char[]

//  int qx = int(quat.y());
//  char quaternionX[6];
//  itoa(qx, quaternionX, 1); //converts int to char[]
//
//  int qy = int(quat.x());
//  char quaternionY[6];
//  itoa(qy, quaternionY, 1); //converts int to char[]
//
//  int qz = int(quat.z());
//  char quaternionZ[6];
//  itoa(qz, quaternionZ, 1); //converts int to char[]

  strcat(dataString, quaternionW);
  strcat(dataString, ",");
  strcat(dataString, quaternionX);
  strcat(dataString, ",");
  strcat(dataString, quaternionY);
  strcat(dataString, ",");
  strcat(dataString, quaternionZ);
  strcat(dataString, "\n");

  Serial.println(dataString);
  //  Abspeichern auf SD nach NUMBER_OF_SAMPLES eingelesenen Daten
  if (counter >= NUMBER_OF_SAMPLES) {

    digitalWrite(LED_BUILTIN, HIGH);
    delay(100);
    digitalWrite(LED_BUILTIN, LOW);
    delay(100);
    digitalWrite(LED_BUILTIN, HIGH);
    delay(100);
    digitalWrite(LED_BUILTIN, LOW);
    delay(100);

    writeStringToSD(dataString, dataFileName); //appends string to the SD card

    Serial.println("DONE  ");

    //setting the array back to 0
    memset(dataString, 0, sizeof(dataString)); //saves 0er in Datenarray (renews it for new data)

    // Setting counter back to 0 for new array
    counter = 0;
  }
}

// ------------------ writeStringToSD ----------------------------------
//Ein belibiger String wird in ein belibiges Files geschrieben

boolean writeStringToSD(const char *data, const char *filename) {
  stopWatch(); //starts stoppwatch
  Serial.print("Writing to: ");
  Serial.println(filename);
  file = SD.open(filename, O_CREAT | O_WRITE); //Öffnet file zum Lesen

  if (file) { // if the files opends
    file.println(data); //adds the data at the bottom of the file
  }
  else {
    Serial.println("Error while opening the file!");
    return false;
  }

  file.flush(); //closing file - saving data
  stopWatch(); //stopps stopwatch
  return true;
}

// ------------------ Stoppuhr ----------------------------------
boolean stopWatch_started = false; // function to measure time
int stopWatch_time;
void stopWatch() {
  if (stopWatch_started == false) {
    stopWatch_time = millis();
    stopWatch_started = true;
  }
  else {
    int duration = millis() - stopWatch_time;
    Serial.print("Duartion: ");
    Serial.print(duration);
    Serial.println(" ms");

    stopWatch_started = false;
  }
}

//Systemcalibration aufzeichnen geht nicht und aufnahmefrequenz ist bei etwa 35ms

[ShermanP]

I think you need to avoid all flushing and closing/reopening. All they do is slow things down. As previous comments have said, you can only write to the card in 512-byte chunks, and sometimes that takes a long time because of wear leveling. So you will need at least two buffers that are 512 bytes or a multiple of 512. The interrupt routine, which is triggered by one of the timers, lets you take readings at an exact time interval, then add the reading to the current buffer, and when it fills up, to the next buffer. Then your main loop detects then the first buffer has filled up, and then writes its contents to the card. You won't lose any data so long as the maximum time required to write to the card is less than the time required to fill up all but one of the buffers.

This is the only way I know of to write to an SD card without losing data.

You might also look at the LowLatencyLogger example in SdFat. He pre-defines the file, which must occupy consecutive clusters, and then the data logging consists of only writing data to consecutive card sectors. This avoids everything to do with the file system during data collection. Then when you're done, the file length and FAT table are modified to reflect what was actually saved.

[xeonus]

So would it be a better solution, to flush every time, but just close and open from now an then?
Because, what I read, I need to flush to get the data from the buffer (RAM) to the SD card. And because of the limited RAM i need to do this quite often. (also Ram speeds slows down, the fuller it gets.)

By the way, in the SDFat library are so many examples i am not sure, which one suits by project. Do you know, which one i can use?

I am having another problem. I want to write the system calibration (called "system") to the sd Card. When i do: Serial.print(system"; everything works as expected.
But when i do:

int systemcali = int(system);
  char systemcalibration[10];
  itoa(systemcali, systemcalibration, 10); //Konvertiert int to char[]

  strcat(dataString, ",");
  strcat(dataString, systemcalibration);

I just get ",0357"
how comes?

[ShermanP]

My understanding is that flushing writes whtever is currently in the buffer to the SD card. But you can't just write a few more bytes to the card each time you generate data. You can only write a complete sector, which is 512 bytes. If you do multiple flushes to the same sector, then the sector has to be erased each time, and re-written, or the card's controller has to read in all the data currently in the erase block containing that sector, and re-write it to a new physical sector that is already erased. All of this can take a lot of time.

Why do you think you need to write the data to the card immediately? That's what the buffer is for. Let your library manage its buffer, and it will accumulate the data in the buffer, and write it to the card in 512-byte chunks. When you've finished the logging session, THEN you can flush the buffer and close the file - once.

In the latest SdFat version, look in the Examples/examplesV1 folder for sketches that might be useful, including the LowLatencyLogger.

I don't know what's wrong with the system calibration code.

[xeonus]

Thanks for the reply, I am making progress.

I want write the data "immediately" to kind of use the 10 millisecond window in between the samples to write it to the sd Card. So I can have a 100HZ dataflow.
Because if i fill a "big" buffer it takes longer to write it to the sd in my code. It takes about 15-25 ms just to write it. If I buffer more data it would take about lets say 100 ms. So at one point I would have a big gap in my data.

It is not easy for me to understand the SDFat library and the part with the 512 chunks.

So I get access to my data with this:

  imu::Quaternion quat = bno.getQuat();
  Serial.println(quat.w());

quat.w is my data. Now lets assume, I just want the data of quat.w to be stored to my SD-Card with 100 hz continuously for lets say 5 minutes. (in reality I would like to store about 10 more like quat.x ...)
At this another question: To conclude the data together to a column I used an array. But I think this isnt a good way, because for writing into this array, I needed to convert int to char and float to char and so an. Alot of code. Can be seen in the Code i already posted.

I thought, if i can get the time it takes to write to the SDCard below 10 ms, there is a chance i can read and write data with 100hz.

I imagine, if it would be possible to write to the SD quickly enough, the Code would be quite easy and doable for me.
I am at the moment working on it. Is my thought right, to use flush every loop, and make a counter which closes and reopens the sd Card every lets say minute to completely safe the data?

I am not confident with the LowLatencyLogger. There are so many things in it i dont understand.

Thanks and best wishes
xeonus

[Paul_KD7HB]

Perhaps this description will help. I hope it doesn't confuse you.
Consider, there are "logical write" data to the SD card and there are "physical write" data to the SD card. Your program only, ever, does a logical write. The library software is watching the number of bytes your logically write to the SD card and when the total reaches 512, the library software does a "physical write" to the SD card.
When your program does a "close" of the SD card file or if it does a "flush", the library program will do a "physical write" of what is in the SD card buffer to the actual card.
The reverse process takes place when your program "reads" data from the SD card. The library "physical reads" a block of 512 bytes and then lets your program "logically read" the bytes you want. When your "logical read" goes beyond 512, the library code "physical read" the next block from the SD card and points your "logical read" to the new physical block of data.
Paul

[xeonus]

Okay that makes sense.
One more question about the "library program". Is that something integrated in the SD-Card, the "library" of the SD-Card, or something, that has to do with the Arduino&Code?

Thanks Paul!

[Paul_KD7HB]

All in your Arduino code.
Paul

[xeonus]

And the library you are refering to is the standard SD-Card library or the SDFAT or maybe both?

And another question
I am quoting you here:"The library software is watching the number of bytes your logically write to the SD card and when the total reaches 512, the library software does a “physical write” to the SD card."

Do you mean, the library is selfmanaging and i dont need to flush/close at all to write to the card?
Just flush/close it when i want to stop actually recording.

I tried around. Did just run a test where I flushed every 50 samples and I never closed the file at all. Data still got saved to the SD.
So again regarding the 512byte buffers. How often do I need to flush or is it just fine to flush right before detaching it from the power source?

Clemens

[Paul_KD7HB]

You will have to study the source code of the libraries.
You do not ever need to "flush" or "close" and "open" a file, except at the beginning of the program when you "open" and at the end when you "close". There are exceptional conditions when you may need to "flush" to physically write the 512 byte buffer to the SD card. I can't think of any right now. The "close" when you are ready to stop the program is necessary.
Part of your time is taken by the library having to read the directory record for the file, update the length of the file and rewrite the record. This is also done when you "flush" and when you "close" the file.
Study the source for the libraries you are using.
Paul

[xeonus]

Hell yeah! I think I understood it and also got it to work almost flawlessly.
Thanks alot to all of you!

The code does now what i want. Just one tiny problem, I really hope the last one.

THe problem:
I am not having stable 100hz. I am capturing the millis of the datapoints and it is 9/10 10milliseconds. But 1/10 is about 15. Not a huuuge problem, but I was wondering, if this is solvable (of course it is somehow.. but doable for me xD)

I tried to simplify the code as good as possible.. here it is:

#include <Wire.h>
#include <Adafruit_Sensor.h>
#include <Adafruit_BNO055.h>
#include <utility/imumaths.h>
#include <SPI.h>
#include <SD.h>
#include <avr/dtostrf.h>
#include "BNO055_support.h"
#include <SdFat.h>
#include "SdFat.h"

// Check I2C device address and correct line below (by default address is 0x29 or 0x28)
//                                   id, address
Adafruit_BNO055 bno = Adafruit_BNO055(55, 0x28);

//creating a File
File file;

// declaring global variables
const int chipSelect = 28; // CS Pin for SD
int last_data_save = 0; // variable for the last time data was read
int counter = 0; //counts the columns with values (samples)

//sensor data global
struct bno055_t myBNO;
imu::Quaternion quat;
imu::Vector<3> acc;
imu::Vector<3> gyr;

// Name of the filename__________________________________
const char dataFileName[14] = "test5.txt";

//How often should be flushed?
int flushtimer = 10000; //milliseconds

//managing sample timing
int last_data_read = 0; // variable for the last time data was read
const int SAMPLE_RATE = 100; //100 Hz
const int SAMPLE_TIME = 1000 / SAMPLE_RATE; // 1000 = 1 sec divided by the Sample Rate = Time between samples

//_________________________________________________________________________________________
// ------------------ initializeSD&Sensor ----------------------------------
void setup(void)
{
  //Initialize I2C communication
  Wire.begin();

  //Initialization of the BNO055
  BNO_Init(&myBNO); //Assigning the structure to hold information about the device

  //Configuration to NDoF mode
  bno055_set_operation_mode(OPERATION_MODE_NDOF);

  delay(1);
  Serial.begin(19200);

  //wait for the serial monitor
  //delete line below for real measurements
  while (!Serial) {}

  // SD-Card Initialise
  Serial.print("Initializing SD card...");
  if (!SD.begin(28)) {
    Serial.println("initialization failed!");
    while (1);
  }
  file = SD.open(dataFileName, O_CREAT | O_WRITE); //Opens file - SDFAT library
}

void loop(void)
{
  if (millis() -  last_data_read >= SAMPLE_TIME) {
    last_data_read = millis();

    counter = counter + 1;
    storeData();
  }
}

void storeData() {

  stopWatch(); //starts stopwatch
  /*___Sample Counter___*/

  file.print(counter);
  file.print(";");

  /*__Tracking time/millis__*/

  file.print(millis());
  file.print(";");

  /*______Quaternion_____*/
  quat = bno.getQuat();

  file.print(quat.w());
  file.print(";");
  file.print(quat.y());
  file.print(";");
  file.print(quat.x());
  file.print(";");
  file.print(quat.z());
  file.print(";");

  /*______ACCELEROMETER_____*/
  acc = bno.getVector(Adafruit_BNO055::VECTOR_ACCELEROMETER);

  file.print(acc.x());
  file.print(";");
  file.print(acc.y());
  file.print(";");
  file.print(acc.z());
  file.print(";");

  /*______GYROSCOPE_____*/
  gyr = bno.getVector(Adafruit_BNO055::VECTOR_GYROSCOPE);

  file.print(gyr.x());
  file.print(";");
  file.print(gyr.y());
  file.print(";");
  file.println(gyr.z());

  stopWatch(); //stopps stopwatch

  if (millis() -  last_data_save >= flushtimer) {
    last_data_save = millis();
    saveData();
  }
  //delay(1000);
}

void saveData() {
  file.flush(); //writing data to SD
  Serial.println("Writing to SD");
}

// ------------------ Stoppuhr ----------------------------------
boolean stopWatch_started = false; // function to measure time
int stopWatch_time;
void stopWatch() {
  if (stopWatch_started == false) {
    stopWatch_time = millis();
    stopWatch_started = true;
  }
  else {
    int duration = millis() - stopWatch_time;
    Serial.print("Duartion: ");
    Serial.print(duration);
    Serial.println(" ms");

    stopWatch_started = false;
  }
}

Does anyone of you know what is the reason for the frequency spikes and how I can get it to run stable?

And also: is there a cleaner way to format the data the way i would like it to be?
I mean this part for example:

  file.print(acc.x());
  file.print(";");
  file.print(acc.y());
  file.print(";");
  file.print(acc.z());
  file.print(";");

Thanks again
Clemens