SDFat logger 400 SPS

Hi community!
I'm trying to log analog data on my motorbike with the 6 ADC inputs of my Arduino MKR Zero. It already works using the SD.h library. The problem is that even using all the tips and tricks on the internet, I cannot write fast enough to the sd. I would need about 400 SPS on every ADC channel. The ADC seems to be fast enough, but the writing to the SD takes about 6-8 ms for all channels. When I flush/close to write the data it even takes about 20 ms. That's not yet good enough.
Now I thought of switching to the SDFat library. Does anyone have enough experience to tell if that's going to solve my problem, or have a good idea on how to solve it?
Ideally, I wouldn't want to switch to binary data, as I'd like to be able to read the log file.
Further, my code does not write a file with the SDFat library. What am I missing?

//General Libraries-------------
#include <Arduino.h>
#include <SPI.h>
#include <math.h>
#include <stdlib.h>

//MKRZero specific libraries----
#include "SdFat.h"
#include "sdios.h"
#include <RTCZero.h>
RTCZero rtc;
int rtccounter = 0;
byte seconds;
byte minutes;
byte hours;
byte day;
byte month;
byte year;

//set up variables using the SD utility library functions---------------------------
/*
  Set DISABLE_CS_PIN to disable a second SPI device.
  For example, with the Ethernet shield, set DISABLE_CS_PIN
  to 10 to disable the Ethernet controller.
*/
const int8_t DISABLE_CS_PIN = -1;
// SDCARD_SS_PIN is defined for the built-in SD on some boards.
const uint8_t SD_CS_PIN = 28;

// Try to select the best SD card configuration.
#if HAS_SDIO_CLASS
#define SD_CONFIG SdioConfig(FIFO_SDIO)
#elif ENABLE_DEDICATED_SPI
#define SD_CONFIG SdSpiConfig(SD_CS_PIN, DEDICATED_SPI, SD_SCK_MHZ(4))
#else  // HAS_SDIO_CLASS
#define SD_CONFIG SdSpiConfig(SD_CS_PIN, SHARED_SPI, SD_SCK_MHZ(4))
#endif  // HAS_SDIO_CLASS

// SD_FAT_TYPE = 0 for SdFat/File as defined in SdFatConfig.h,
// 1 for FAT16/FAT32, 2 for exFAT, 3 for FAT16/FAT32 and exFAT.
#define SD_FAT_TYPE 3

#if SD_FAT_TYPE == 0
SdFat sd;
File myFile;
#elif SD_FAT_TYPE == 1
SdFat32 sd;
File32 myFile;
#elif SD_FAT_TYPE == 2
SdExFat sd;
ExFile myFile;
#elif SD_FAT_TYPE == 3
SdFs sd;
FsFile myFile;
#else  // SD_FAT_TYPE
#error Invalid SD_FAT_TYPE
#endif  // SD_FAT_TYPE

String dataString = "";
String Header = "";


//AnalogPins
int sP0 = A0;
int sP1 = A1; 
int sP2 = A2; 
int sP3 = A3; 
int sP4 = A4; 
int sP5 = A5; 
int sP6 = A6; 

//DigitalPins
int sP7 = 0;
int sP8 = 1;

//Variables
uint16_t s0 = 0;  // variable to store the value coming from the sensor
uint16_t s1 = 0;
uint16_t s2 = 0;
uint16_t s3 = 0;
uint16_t s4 = 0;
uint16_t s5 = 0;
uint16_t s6 = 0;
//uint16_t s7 = 0;
//uint16_t s8 = 0;

//Timer
int timer0 = 0;
unsigned int time0 = 0;

//Samplerate
int s_rate0 = 3000;
int s_rate1 = 100;
//Counter
int filen = 0;
int fileCountOnSD = 0; // for counting files

//Funktionen
void printDirectory(File dir, int numTabs) {
  while (true) {

    File entry =  dir.openNextFile();
    if (! entry) {
      // no more files
      break;
    }
    for (uint8_t i = 0; i < numTabs; i++) {
      Serial.print('\t');
    }
    Serial.print(entry.name());

    // for each file count it
    fileCountOnSD++;

    if (entry.isDirectory()) {
      Serial.println("/");
      printDirectory(entry, numTabs + 1);
    } else {
      // files have sizes, directories do not
      Serial.print("\t\t");
      Serial.println(entry.size(), DEC);
    }
    entry.close();
  }
}

// int smooth(int x){ //Pin übergeben
//   int i;
//   uint16_t value = 0;
//   int numReadings = 12;

//   for (i = 0; i < numReadings; i++){
//     // Read light sensor data.
//     value = value + analogRead(x);
    
//   }

//   // Take an average of all the readings.
//   value = value / numReadings;
//   return value;
// }

void setup() {

  //Serial.begin(115200);
  analogReadResolution(12); //10 bit showed no imrovement in terms of sps
  //pinMode(sP7, INPUT);
  //pinMode(sP8, INPUT);

  //Initialisieren SD Karte------------------------------------
  sd.cardBegin(SD_CONFIG);
  //Count files on sd
  //myFile = SD.open("/");
  myFile.open("/");
  printDirectory(myFile, 0);
  filen = fileCountOnSD;
  if (sd.exists(String(filen) + "DATA.txt")) {
       return;
  } else {
    myFile = sd.open(String(filen) + "DATA.txt", FILE_WRITE);
    myFile.close();
  }
  Header = ("Time_rel.,A0,A1,A2,A3,A4,A5,A6");
  myFile = sd.open(String(filen)+ "DATA.txt", FILE_WRITE);
  if (myFile) {
    myFile.println(Header);
    myFile.close();   
  }
  timer0 = millis();
  time0 = millis();

  rtc.begin(); // initialize RTC
  /* Change these values to set the current initial time */
  seconds = 0;
  minutes = 0;
  hours = 0; 
  /* Change these values to set the current initial date */
  day = 1;
  month = 1;
  year = 2021; 
  rtc.setTime(hours, minutes, seconds);
  rtc.setDate(day, month, year);
}

void loop() {
/*
If you replace FILE_WRITE with O_WRITE | O_CREAT and control when flush is called you can increase the speed by a large factor.
*/
  myFile = sd.open(String(filen) + "DATA.txt", O_WRITE| O_APPEND | O_CREAT);

  for (uint8_t i = 0; i < 100; i++) { 
    //if ((millis() - timer0) > 2) {
      s0 = analogRead(sP0);
      s1 = analogRead(sP1);
      s2 = analogRead(sP2);
      s3 = analogRead(sP3);
      s4 = analogRead(sP4); 
      s5 = analogRead(sP5);
      s6 = analogRead(sP6);
      time0 = millis();
      //s7 = digitalRead(sP7);
      //s8 = digitalRead(sP8); 
      //Save the values on SD
      dataString = String(time0) + ',' + String(s0) + "," + String(s1) + "," + String(s2) + "," + 
                  String(s3) + "," + String(s4) + "," + String(s5) + ","
                  + String(s6);

      myFile.println(dataString);
      //}
    timer0 = millis();       
  }           
  //myFile.flush();                  
  myFile.close();   
}

using the String class is not known to make things fast... you could replace this

dataString = String(time0) + ',' + String(s0) + "," + String(s1) + "," + String(s2) + "," +
             String(s3) + "," + String(s4) + "," + String(s5) + ","
             + String(s6);

myFile.println(dataString);

by

myFile.print(time0); myFile.write(',');
myFile.print(s0); myFile.write(',');
myFile.print(s1); myFile.write(',');
myFile.print(s2); myFile.write(',');
myFile.print(s3); myFile.write(',');
myFile.print(s4); myFile.write(',');
myFile.print(s5); myFile.write(',');
myFile.println(s6);

If write speed is an issue, you should consider writing the binary file.

It's not complicated to write another Arduino Sketch later on that will dump the binary into something you can inject into a CSV file

1 Like

Thanks for the great proposal!

Do you by chance know if there is considerable difference in write speed between the standard SD.h and the SDFat?
And why is my sketch above, using the SDFat library not writing a file? Doing the same thing using the SD.h works just fine...

SDFat is supposed to be faster than the plain SD version (if I remember correctly SD.h is just a wrapper for a 2009 version of SdFat that is maintained by the Arduino company) but many other things can come into play like the type of SD card you use.

have you tried a simple sketch to see if you can write into a file ?

1 Like

@J-M-L : I tried your proposed change in how to write, not using the String class in the way I did. Unfortunately, I'm still on 6ms per 7 ADC channels plus timestamp writing speed.
If I could improve the writing speed to 1 ms I would have already enough SPS to successfully monitor the engines hall sensors up to 4000-5000 rpm.

The simple sketch somehow doesn't work for the MKR Zero. I found out that there is a mismatch in the clock frequency that most of the examples of the SDFat library use and the one that the MKR Zero uses (which I found out to be 4 MHz). That's why most examples fail. But not the simple sketch, there must be something else still at play. I tried to modify it and still wasn't successful. Also using SDCARD_SS_PIN as the pin declaration for the CS doesn't usually work. I had to specifically name pin number 28. In the standard SD library, the declaration via SDCARD_SS_PIN does work. But that's a minor issue...

Still, I can't get my modified simple sketch example to work. I have to say it's a bit frustrating. Though, I'm not the brightest programmer after all. My background is more on analogue electronics.

I'll attach my modified simple sketch, maybe someone sees something straight away, knowing what to do (my SD card is preformatted as FAT32):

/*
  SD card read/write
 This example shows how to read and write data to and from an SD card file
 The circuit:
 * SD card attached to SPI bus as follows:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 created   Nov 2010
 by David A. Mellis
 modified 9 Apr 2012
 by Tom Igoe
 This example code is in the public domain.
 */

#include <SPI.h>
//#include <SD.h>
#include "SdFat.h"
SdFat SD;

//set up variables using the SD utility library functions---------------------------
/*
  Set DISABLE_CS_PIN to disable a second SPI device.
  For example, with the Ethernet shield, set DISABLE_CS_PIN
  to 10 to disable the Ethernet controller.
*/

const uint8_t SD_CS_PIN = 28;

// Try to select the best SD card configuration.
# define SD_CONFIG SdSpiConfig(SD_CS_PIN, DEDICATED_SPI, SD_SCK_MHZ(4))


// SD_FAT_TYPE = 0 for SdFat/File as defined in SdFatConfig.h,
// 1 for FAT16/FAT32, 2 for exFAT, 3 for FAT16/FAT32 and exFAT.
SdFat sd;
File myFile;



void setup() {
  // Open serial communications and wait for port to open:
  Serial.begin(9600);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  }


  Serial.print("Initializing SD card...");

  if (!SD.begin(SD_CS_PIN)) {
    Serial.println("initialization failed!");
    return;
  }
  Serial.println("initialization done.");

  // open the file. note that only one file can be open at a time,
  // so you have to close this one before opening another.
  myFile = SD.open("test.txt", FILE_WRITE);

  // if the file opened okay, write to it:
  if (myFile) {
    Serial.print("Writing to test.txt...");
    myFile.println("testing 1, 2, 3.");
    // close the file:
    myFile.close();
    Serial.println("done.");
  } else {
    // if the file didn't open, print an error:
    Serial.println("error opening test.txt");
  }

  // re-open the file for reading:
  myFile = SD.open("test.txt");
  if (myFile) {
    Serial.println("test.txt:");

    // read from the file until there's nothing else in it:
    while (myFile.available()) {
      Serial.write(myFile.read());
    }
    // close the file:
    myFile.close();
  } else {
    // if the file didn't open, print an error:
    Serial.println("error opening test.txt");
  }
}

void loop() {
  // nothing happens after setup
}

I answered in your other post, I'll try to pull out my Zero and make a test. Many SD card readers require 5V, have you taken that into account ?

I do power the MKR Zero through its VIN with 5V with a low noise LDV-Regulator (LM1117) that offers a max 800mA

I did a quick test and whether with the SD or SDFat library, I did not get my SD card recognised on an Arduino Zero...

It is a real pity, as the MKR Zero is rather perfect with the SAMD21, the SD Card reader integrated and on a dedicated SPI bus for it and its small footprint

I've not explored further, might be due to 3.3V SPI versus 5V and possibly some physical adaptation needed... As I said in the other post, the teensy with a builtin SD card is doing fine for my needs, so that's what I would use.

If you have it working with the standard SD library and want a quick win in terms of performance, I would suggest to move to binary format and write a separate reader app when you need to extract the data into CSV.

1 Like

I don't have any experience with the MKR Zero, but there ought to be a way to get the card reader to work normally. I don't know what the 4MHz thing you refer to is all about.

In general, you should avoid flushing and closing until you are ready to end the session. Just continue to write to the file and let the library write it to the card in 512-byte chunks. Doing anything else just slows everything down.

But even so, writing to a file can be very slow at times. If the library has to open a new cluster, then it has to update the FAT, and the second copy of the FAT as well as write to the file. And then from time to time the card's controller will need to do some wear leveling. All of that can take up to 20ms or more for a single write.

You might look at the LowLatencyLogger example in the SdFat library for the fastest writing speed. It uses multiple buffers which allow you to continue to collect data if a slow write occurs. But the main thing it does is to create in advance 128MB files consisting of consecutive clusters, and even erasing the data contents in advance, so writing to the file just consists of writing to successive sectors on the card. The file system entries aren't made at all until the session is over when the directory entry and the FATs are updated to reflect what was actually written. It's pretty slick.

But we do need to get to the bottom of problems with the onboard card reader.

This topic was automatically closed 120 days after the last reply. New replies are no longer allowed.