Teensy 3.1 / Data Logging / SD Card / Latency / Non-Blocking SDFAT Library

I am working on a logging device for paragliding, using a Teensy 3.1, a MS5637 pressure sensor and an SD card. The code I am using is as follows:

#include <SdFat.h>    // SD card library
#include <math.h>
#include <Wire.h>
const int chipSelect = 10;

#define ADDRESS 0x76 //define adress of MS5637 Barometer, infos from datasheet

// Create file system object
SdFat sd;

// Define text file for logging
ofstream logfile;

uint32_t D1 = 0;
uint32_t D2 = 0;
int64_t dT = 0;
int64_t TEMP = 0;
int64_t OFF = 0;
int64_t SENS = 0;
int64_t P = 0;
int64_t T2    = 0;
int64_t OFF2  = 0;
int64_t SENS2 = 0;
uint16_t C[7];

float temperature;
float pressure;
float altBaro;
const float sea_press = 1013.3;

String bufferString = "";
String SDbufferString = "";

uint64_t time = 0;
uint64_t timeOld = 0;




void setup() {

  Wire.begin();
  Serial.begin(115200);
  
  I2C0_F = 0x1A;   //set the I2C speed to 400KHz for Teensy 3.1

  delay(100);
  initBaro(ADDRESS);

  // initialize the SD card at SPI_FULL_SPEED
  if (!sd.begin(chipSelect, SPI_FULL_SPEED )) sd.initErrorHalt();

  // Create a new file in the current working directory
  // and generate a new name if the file already exists
  char fileName[13] = "LOGGER00.CSV";

  for (uint8_t i = 0; i < 150; i++) 
  {
    fileName[6] = i/10 + '0';
    fileName[7] = i%10 + '0';
    if (sd.exists(fileName)) continue;
    logfile.open(fileName);
    break;
  }

  logfile << ("Time,Temperature,Pressure,AltitudeRaw") << flush;   //Write header to logfile
  timeOld = millis();
}



void loop() {

  time = millis();
  int dt = time - timeOld;
  timeOld = time;

  requestPressure();      // Request pressure conversion raw
  delay(20);              // OSR=8192, minimal delay ro read result is 17ms
  // Instead of delay(20) something useful can be done during this time, like reading an IMU
  readPressure();         // Read raw pressure

  requestTemperature();   // Request pressure conversion raw
  delay(20);              // OSR=8192, minimal delay ro read result is 17ms
  // Instead of delay(20) something useful can be done during this time, like reading an IMU 
  readTemperature();      // Read raw pressure

  calculatePressureTemperature();
  altBaro = 44330 * (1.0 - pow(pressure /sea_press,0.1903));

  // make a string for assembling the data to log:
  String dataString = "";

  dataString += String("Time,");
  dataString += String(int(time));
  dataString += String(",");
  dataString += String("Temperature,");
  dataString += String(temperature);
  dataString += String(",");
  dataString += String("Pressure,");
  dataString += String(pressure);
  dataString += String(",");
  dataString += String("AltitudeBaro,");
  dataString += String(altBaro);
  dataString += String("\n");  //Create a new line on the SD card

  bufferString += dataString;
  int length = bufferString.length();

  if (length > 512)
  {

    SDbufferString = bufferString.substring(0,511);    //Create a string with 512 bytes for writing on the SD card.
    bufferString = bufferString.substring(511,length); //Remove the 512 bytes for the SD card from the main string. 

    char charBuf[512];
    SDbufferString.toCharArray(charBuf,512);
    logfile << charBuf << flush;      //Write to SD Card  
  }

  Serial.print(int(dt));
  Serial.println();

}


void requestPressure()
{
  Wire.beginTransmission(ADDRESS);
  Wire.write(0x4A); //OSR=8192, minimal delay ro read result is 17ms
  Wire.endTransmission();
  //delay(20);
}


void readPressure()
{
  Wire.beginTransmission(ADDRESS);
  Wire.write((byte) 0x00);
  Wire.endTransmission();

  Wire.beginTransmission(ADDRESS);
  Wire.requestFrom(ADDRESS, (int)3);
  if (Wire.available() >= 3)
  {
    D1 = Wire.read() * (unsigned long)65536 + Wire.read() * (unsigned long)256 + Wire.read();
  }
  Wire.endTransmission();
}


void requestTemperature()
{
  Wire.beginTransmission(ADDRESS);
  Wire.write(0x5A);  //OSR=8192, minimal delay ro read result is 17ms
  Wire.endTransmission();
  //delay(20);
}


void readTemperature()
{
  Wire.beginTransmission(ADDRESS);
  Wire.write((byte) 0x00);
  Wire.endTransmission();

  Wire.beginTransmission(ADDRESS);
  Wire.requestFrom(ADDRESS, (int)3);
  if (Wire.available() >= 3)
  {
    D2 = Wire.read() * (unsigned long)65536 + Wire.read() * (unsigned long)256 + Wire.read();
  }
  Wire.endTransmission();
}


void calculatePressureTemperature()
{
  OFF2 = 0;
  SENS2 = 0;

  dT=D2-C[5]*pow(2,8);
  TEMP=2000+(dT*C[6])/pow(2,23);
  OFF=C[2]*pow(2,17)+dT*C[4]/pow(2,6);
  SENS=C[1]*pow(2,16)+dT*C[3]/pow(2,7);

  if (TEMP < 2000) // if temperature lower than 20 Celsius
  {
    OFF2 =  61*pow((TEMP-2000),2)/pow(2,4);
    SENS2 = 29*pow((TEMP-2000),2)/pow(2,4);
  }

  TEMP=TEMP-T2;
  OFF=OFF-OFF2;
  SENS=SENS-SENS2;

  P=(((D1*SENS)/pow(2,21)-OFF)/pow(2,15));

  temperature=(float)TEMP/100;
  pressure=(float)P/100;

}


void initBaro(uint8_t address)
{
  Serial.println("Reading Calibration Data");

  Wire.beginTransmission(address);
  Wire.write(0x1E); // reset
  Wire.endTransmission();
  delay(10);


  for (int i = 0; i < 6  ; i++) {

    Wire.beginTransmission(address);
    Wire.write(0xA2 + (i * 2));
    Wire.endTransmission();

    Wire.beginTransmission(address);
    Wire.requestFrom(address, (uint8_t) 6);
    delay(1);
    if (Wire.available())
    {
      C[i + 1] = Wire.read() << 8 | Wire.read();
    }
    else {
      Serial.println("Error Reading Calibration Data"); // error reading the PROM or communicating with the device
    }

    Serial.print("C");
    Serial.print(i + 1);
    Serial.print(" ");
    Serial.println(C[i + 1]);
  }
  Serial.println();
}

The issue is now that it takes too long (and the dt time is not constant, varies between 41ms to 300ms) to write the data to the SD card, and that the code does nothing useful when the SD card is busy writing the data. The idea is to have a code that always needs the same time for the loop, and that there would no useful processor time wasted waiting for the SD card to write the data.

Ideally it would be great if the writing of the data to the SD card could be split into several actions:

Buffer the data that have to be written to the SD card in strings of 512 bytes.

Check if the SD card is ready to accept data. If not, continue directly with the main loop. But if it's possible to write data then this data should be sent to the SD card, without waiting that the SD card actually writes the data. Continue immediately after the write command with the main loop.

Check later on if the data has been written to the SD card in the loop. If the SD card is still busy writing data continue immediately with the main loop. Otherwise write another 512 bytes from the buffer.

Like this the loop would always take almost the same time to execute, but the data would only be sent to the SD card if that makes sense. The intervals where the data are written to the SD card would then be somewhat written randomly, but the code would not be slowed down due to the fact that data gets written to the SD card.

It doesn't matter if it takes time to write to the card, but it would be very useful if the Arduino or Teensy could do something else while the card is writing the data, instead of waiting for the SD card until the write process has finished. If such an approach is feasible then it might take much less processor time to write the data. And if the data is written not in constant time steps this doesen't matter, it's important that the correct data is written to the SD card, not when that data is written.

A non-blocking SD library or SDFAT library would solve that problem.

I am quite new to programming, so help to solve that problem would greatly be appreciated.

You'd probably be better off asking in the PJRC Teensy forum.

Pete

The SD card latency problem / code blocking is not Teensy related, its the same problem with Arduinos. Thats why I posted in this forum, but also at PJRC (Teensy) Forum. Teensy 3.1 has more memory than an Arduino Uno, so buffering might be easier to realize with a Teensy. But a non-locking SDFAT library might be also solve many SD card related problems with Arduinos.

logfile << charBuf << flush;      //Write to SD Card

Writing single 512 byte blocks to files on modern SD cards will always have long unpredictable write delays. The basic flash write size is not 512 bytes.

Modern SD cards have a Recording Unit (RU) that is a multiple of 16 KB. High performance cards may have 32 KB or larger RUs.

When a 512 byte block is forced to the card, a new RU must be written. Existing data must be copied to the new RU by the card controller. This could mean 32 KB of data is written to flash each time you do a flush. This soon exhausts the free erased RUs and the card will block while it erased an AU of flash which may be several MB.

Check if the SD card is ready to accept data.

There is no command to do this. Any new features that could help are not available in SPI mode. SPI mode is very limiting and only marginally supported in modern cards.

There are several ways to deal with this problem. You can use an RTOS to buffer sensor data in a high priority thread and write to the SD in a low priority thread. See the SD logger example in NilRTOS Google Code Archive - Long-term storage for Google Code Project Hosting..

I wrote SdFat which is also the base for SD.h. I have no plans to add new features to avoid this problem. Writing to a file system using SPI mode with a 512 byte cache buffer will have long delays with modern SD cards.

I am experimenting with SDIO using 32 KB buffers on an STM32F407. I can write at over 5 MB/sec with no busy delays. I can't port this code to Arduino.

Edit: I can write at over 20 MB/sec with only small, less than 1 ms, occasional busy delays using 4-bit SDIO.

fat16lib:
Writing single 512 byte blocks to files on modern SD cards will always have long unpredictable write delays. The basic flash write size is not 512 bytes.

This would also be true of any smaller buffer size?

This would also be true of any smaller buffer size?

SPI mode performance of the new generation of SD cards varies greatly and is not very predictable. Manufactures only include SPI mode since the SD spec requires it. Sometimes larger buffers reduce the frequency of long latency write but not the time. In other cases the time is reduced.

New cards are optimized for phones and tablets in 4-bit SDIO mode. There is little correlation between SPI performance with small writes and large writes in SDIO mode.

I don't think there is a future in optimizing SPI mode performance for fast data logging.

I am buying development boards with SDIO mode SD sockets. The SDIO controller on these boards have CRC and a fast DMA mode.

Increased the buffer to 8 KB, with a buffer of 16 KB the code does not execute. But there is still the problem that writing to the SC card blocks the code.

What about the idea to first open a file on the SD card (T). Then pre-erase a block of 8 KB on the SD card in the next loop (T+1). Send 8 KB from the buffer to the SD card in the following loop (T+2) , but in a way that the code is not blocked. Then a few loops later (T + 10) close the SD file. If there is not a lot of data to be written, maybe every 120 loops in my case, then it should be possible to log the data correctly, without delaying the main code, even when the SD card has sometimes a big latency.

This won't work.

What about the idea to first open a file on the SD card (T). Then pre-erase a block of 8 KB on the SD card in the next loop (T+1). Send 8 KB from the buffer to the SD card in the following loop (T+2) , but in a way that the code is not blocked. Then a few loops later (T + 10) close the SD file.

Use an RTOS, that will work.

There are several ways to deal with this problem. You can use an RTOS to buffer sensor data in a high priority thread and write to the SD in a low priority thread. See the SD logger example in NilRTOS Google Code Archive - Long-term storage for Google Code Project Hosting..

fat16lib:

This would also be true of any smaller buffer size?

SPI mode performance of the new generation of SD cards varies greatly and is not very predictable. Manufactures only include SPI mode since the SD spec requires it. Sometimes larger buffers reduce the frequency of long latency write but not the time. In other cases the time is reduced.

New cards are optimized for phones and tablets in 4-bit SDIO mode. There is little correlation between SPI performance with small writes and large writes in SDIO mode.

I don't think there is a future in optimizing SPI mode performance for fast data logging.

I am buying development boards with SDIO mode SD sockets. The SDIO controller on these boards have CRC and a fast DMA mode.

Thanks for the reply, I sure do appreciate your library and the effort and expertise you've put into it.

To use an RTOS might work, but right now that is far too complex for my programming skills.

From How to Use MMC/SDC it looks like it is possible to eliminate waste time:

Single Block Write

The Single Block Write writes a block to the card. After a CMD24 is accepted, the host controller sends a data packet to the card. The packet format is same as block read operations. Most cards cannot change write block size and it is fixed to 512. The CRC field can have any fixed value unless the CRC function is enabled. The card responds a Data Response immediataly following the data packet from the host. The Data Response trails a busy flag and host controller must wait until the card goes ready.
In principle of the SPI mode, the CS signal must be kept asserted during a transaction. However there is an exception to this rule. When the card is busy, the host controller can deassert CS to release SPI bus for any other SPI devices. The card will drive DO low again when reselected during internal process is in progress. Therefore a preceding busy check, check if card is busy prior to each command and data packet, instead of post wait can eliminate waste wait time. In addition the internal write process is initiated a byte after the data response, this means eight clocks are required to initiate internal write operation. The state of CS signal during the eight clocks can be either low or high, so that it can be done by bus release process described below.

So basically it would be sufficient to check if the card is busy in checking if DO is low, and only send commands and / or data packets to the card when DO is high, and continue with the code after a data packet was sent and the data response signal was received, without waiting for the SD card and DO to go high again, as described above. Then the latency problem might be almost eliminated.

So basically it would be sufficient to check if the card is busy in checking if DO is low, and only send commands and / or data packets to the card when DO is high

Won't work. Please read the following carefully.

When you write to a file, a large number of SD commands and operations operations are required. There is no way you could check for busy before executing this statement.

logfile << charBuf << flush;

This statement may require a cluster to be allocated. That requires reading FAT blocks until a free cluster is found. Updating the FAT block in memory, writing it back to the SD, then updating the backup FAT. Even if you write 512 bytes, the data may cross a block so there could be read of a block, copy part of the data to that block, write the block back, then write a second block. Then the flush requires reading the directory entry for the file, updating the directory entry and writing it back.

The SdFat library does check for busy before every write since the write would fail otherwise. The problem here is that the card won't indicate busy until you send the write command. Then you must wait before sending the data packet. If you write a RU which may be 16 KB, there may be 32 busy periods.

SdFat can't check for busy before reads. The card sends a stream of busy tokens for read and finally sends a start data token.

No reasonable SD library can do what you want. Chan's FatFS does not have a non-blocking mode. FatFS is widely use and the latency problem is handled by using FatFS in an RTOS.

I also have a new replacement for SdFat that I use in RTOSs like ChibiOS and FreeRTOS. The problem is simpler on ARM processors with more features than Teensy. You can use SDIO and write a driver that sleeps while the SDIO controller does all the busy checking for a large read or write. Other threads run and no CPU time is lost in busy checks.

There is another possibility. You can create a large continuous file and write it with low level SD operations. You must still buffer data but the check of MISO should work.

This program used one large multiple block write and avoids most busy periods. It can log data much faster than your requirement on an Uno Try this super fast analog pin logger - Storage - Arduino Forum .

I did a reliability test logging five analog pins at 5,000 samples per second. This is an ADC rate of 25,000 values per second. I logged 512 MB of data without dropping any values.

Thanks for the quick response fat16lib. It looks like I am stuck with my problem. The example from Try this super fast analog pin logger - Storage - Arduino Forum does not work with Teensy 3.1. But I need a Teensy 3.1 for the processing power for the filtering of the data using a Kalman Filter.

My last hope is your hint to use a large continuous file and write it with low level SD operations. Is it possible to use a CSV file as the large continuous file? If yes then this might work, since buffering data is not a problem with Teensy. It's really just writing data to the SD card what I want, with the lowest latency possible. Nothing else.

The example Try this super fast analog pin logger - Storage - Arduino Forum shows how to use a per-allocated contiguous file. Study it and use a similar method on Teensy.

Are you a lazy student trying to get others to think for you?

I won't be replying to any more of your questions since the examples I pointed out have the basic solution.

No I am not a lazy student. Just a newbie in programming, began playing around with Arduino about a year ago.

The hint with the Analog Logger helped a lot, and also the information from Teensy 3.1: ADC Measurement, SD logging, problem with Sample Rate?.

/*
ECG measurement Project:
 - Sensing two ADC values,
 - Analog files being logged on sd card
 Time  A-B  A  B
 */

//Including Libraries
#include <string.h>
#include <Time.h>  
#include <SdFat.h>
#include <SdFatUtil.h>
#include <BufferedWriter.h>
SdFat sd;
SdFile myFile;

//Teensy 3.1 has the LED on pin 13
const int ledPin = 13;
const int readPin0 = A2;  //ADC read A2
const int readPin1 = A3;  //ADC read A3
const int chipSelect = SS;
//char filename[] = "00000000.txt";
#define SD_CHIP_SELECT  10  // SD chip select pin
#define error(s) sd.errorHalt_P(PSTR(s))
#define FILE_BASE_NAME "LOG" //Can be max size of 5 character, XXXXX000.BIN DOS 8.3 format
#define TMP_FILE_NAME "TMP_LOG.BIN"
const uint8_t BASE_NAME_SIZE = sizeof(FILE_BASE_NAME) - 1;
const uint32_t FILE_BLOCK_COUNT = 500UL; //500=250kb 1 block=512bytes
const uint32_t ERASE_SIZE = 262144L;
uint32_t bgnBlock, endBlock;
uint8_t*  pCache;
char binName[13] = FILE_BASE_NAME "000.BIN";

// Define text file for logging
ofstream myfile;

float SDBuffer [128];
int count = 0;
uint32_t bn = 1; //block number
int startTime; //to reset milli count at each recording

String dataString = "";
String SDbufferString = "";

//Interval Timer
IntervalTimer timer0;
char c=0;

uint64_t time = 0;

//------------------------------------------------------------------------------
/*
 * User provided date time callback function.
 * See SdFile::dateTimeCallback() for usage.
 */
void dateTime(uint16_t* date, uint16_t* time) {
  // User gets date and time from GPS or real-time
  // clock in real callback function
  time_t now();
  // return date using FAT_DATE macro to format fields
  *date = FAT_DATE(year(), month(), day());

  // return time using FAT_TIME macro to format fields
  *time = FAT_TIME(hour(), minute(), second());
}


void createFile(){

  bn=1; //reset block count
  count=0;
  // Find unused file name.
  if (BASE_NAME_SIZE > 5) {
  }
  while (sd.exists(binName)) {
    if (binName[BASE_NAME_SIZE + 2] != '9') {
      binName[BASE_NAME_SIZE + 2]++; //changed from +1 since now 0-999 instead of 0-99
    } 
    else {
      binName[BASE_NAME_SIZE + 2] = '0';
      //binName[BASE_NAME_SIZE + 1]++;
      if (binName[BASE_NAME_SIZE+1] == '9') {
        binName[BASE_NAME_SIZE + 1] = '0';
        binName[BASE_NAME_SIZE ]++;
      }
      else{
        if (binName[BASE_NAME_SIZE] == '9') ;
        binName[BASE_NAME_SIZE + 1]++;
      }
    }

    // Serial.println(binName);
    // delete old log file
    if (sd.exists(TMP_FILE_NAME)) {
      sd.remove(TMP_FILE_NAME);
    }

  }


  // create new file
  myFile.close();
  if (!myFile.createContiguous(sd.vwd(),
  TMP_FILE_NAME, 512 * FILE_BLOCK_COUNT));
  // get address of file on SD
  if (!myFile.contiguousRange(&bgnBlock, &endBlock));
  // use SdFat's internal buffer
  pCache = (uint8_t*)sd.vol()->cacheClear();
  if (pCache == 0); 
  memset(pCache, '0', 512); 
  // flash erase all data in file
  uint32_t bgnErase = bgnBlock;
  uint32_t endErase;
  while (bgnErase < endBlock) {
    endErase = bgnErase + ERASE_SIZE;
    if (endErase > endBlock) endErase = endBlock;
    if (!sd.card()->erase(bgnErase, endErase)) ;
    bgnErase = endErase + 1;
  }
}

time_t getTeensy3Time()
{
  return Teensy3Clock.get();
}

void setup() {
  Serial.begin(115200);
  setSyncProvider(getTeensy3Time);
  SdFile::dateTimeCallback(dateTime);
  pinMode(ledPin, OUTPUT);
  pinMode(readPin0, INPUT);  //Read analog signal pin1
  pinMode(readPin1, INPUT);  //Read analog signal pin2  

  // Initialize SdFat or print a detailed error message and halt
  // Use half speed like the native library.
  // change to SPI_FULL_SPEED for more performance.
  if (!sd.begin(chipSelect, SPI_FULL_SPEED)) sd.initErrorHalt();

  createFile();  
}

void loop() {
  //not sure how you start/stop data collection
  //Example using serial input
  if (Serial.available()) {
    c = Serial.read();
    if(c=='r') { // Start recording
      if (!sd.card()->writeStart(bgnBlock, FILE_BLOCK_COUNT))Serial.println("Recording") ;
      //digitalWrite(LED,ledValue);//Turn LED on to indicate recording if you want
      timer0.begin(timerCallback0, 20000); //start timer 4000=250sps
      startTime=millis();
    } 
    else if(c=='s') { // stop recording
      stopRecording();  
    } 

  }
  delay(1);

}

void stopRecording(){
  timer0.end(); //Turn off timer to stop SD card write
  if (!sd.card()->writeStop())Serial.println("Stopped");
  // Truncate file if recording stopped early.
  if (bn != FILE_BLOCK_COUNT) {    
    if (!myFile.truncate(512L * bn)) ;
  }
  binaryToCsv();
  if (!myFile.rename(sd.vwd(), binName)) ;
  createFile(); 
}

void timerCallback0(){

  time = millis();

  float ADCvalue_0 = analogRead(A2)*3.3/1024;
  float ADCvalue_1 = analogRead(A3)*3.3/1024;

  Serial.print("Time: ");
  Serial.print(int(time));
  Serial.print("\t"); 
  Serial.print("A-B: ");
  Serial.print(ADCvalue_0-ADCvalue_1,3);  
  Serial.print("\t");  
  Serial.print("A: ");  
  Serial.print(ADCvalue_0,3);
  Serial.print("\t");  
  Serial.print("B: ");
  Serial.print(ADCvalue_1,3);
  Serial.print("\t");
  Serial.println("Additional_Data... ");  

  //Serial.println(int(time));

  dataString += String("Time: ");
  dataString += String(int(time));
  dataString += String("\t"); 
  dataString += String("A-B: ");
  dataString += String(ADCvalue_0-ADCvalue_1,3);  
  dataString += String("\t"); 
  dataString += String("A: ");
  dataString += String(ADCvalue_0,3);
  dataString += String("\t");  
  dataString += String("B: ");
  dataString += String(ADCvalue_1,3); 
  dataString += String("\t");  
  dataString += String("Additional_Data... ");
  dataString += String("\n");  //Create a new line on the SD card

  int length = dataString.length();

  if (length > 512)
  {
    noInterrupts();

    SDbufferString = dataString.substring(0,511);    //Create a string with 512 bytes for writing on the SD card.
    dataString = dataString.substring(511,length);   //Remove the 512 bytes for the SD card from the main string.

    //char charBuf[512];
    //SDbufferString.toCharArray(charBuf,512);
    //memcpy(pCache, &charBuf, 512);
    
    memcpy(pCache, &SDbufferString, 512);
    
    if (!sd.card()->writeData((uint8_t*)pCache)) ;  //Here is the problem, probably the wrong data type get's written to the SD Card

    bn++; //increment block number
    count = 0; 
    memset(pCache, '0', 512); 
    interrupts(); 
  }


}

void binaryToCsv() {
  uint8_t lastPct = 0;
  int buf[512];
  uint32_t t0 = millis();
  uint32_t syncCluster = 0;
  SdFile csvFile;
  char csvName[13];
  BufferedWriter bw;

  if (!myFile.isOpen()) {
    Serial.println(F("No current binary file"));
    return;
  }
  myFile.rewind();
  //if (!binFile.read(&buf , 512) == 512) error("Read metadata failed");
  // create a new csvFile
  strcpy(csvName, binName);
  strcpy_P(&csvName[BASE_NAME_SIZE + 3], PSTR(".CSV"));

  if (!csvFile.open(csvName, O_WRITE | O_CREAT | O_TRUNC)) {
    error("open csvFile failed");  
  }
  Serial.println();
  Serial.print(F("Writing: "));
  Serial.print(csvName);
  Serial.println(F(" - type any character to stop"));
  bw.init(&csvFile);
  bw.putStr_P(PSTR("Time,")); //Write header at top of file
  bw.putStr_P(PSTR("A-B,"));
  bw.putStr_P(PSTR("A,"));
  bw.putStr_P(PSTR("B"));
  bw.putCRLF();

  uint32_t tPct = millis();
  while (!Serial.available() && myFile.read(&buf, 512) == 512) {
    uint16_t i;
    //read 512 bytes of data here i.e. 512 samples
    for (i = 0; i < 512; i++) {
      bw.putChar(buf[i]);
      //bw.putStr(buf[i]);  //Invalid conversion from "int" to "char*" (-fpermissive), does not compile
    }

    //bw.putCRLF();


    if (csvFile.curCluster() != syncCluster) {
      bw.writeBuf();
      csvFile.sync();
      syncCluster = csvFile.curCluster();
    }
    //looks like code to print out % done
    if ((millis() - tPct) > 1000) {
      uint8_t pct = myFile.curPosition()/(myFile.fileSize()/100);
      if (pct != lastPct) {
        tPct = millis();
        lastPct = pct;
        Serial.print(pct, DEC);
        Serial.println('%');
      }
    }
    if (Serial.available()) break;
  }
  bw.writeBuf();
  csvFile.close();
  Serial.print(F("Done: "));
  Serial.print(0.001*(millis() - t0));
  Serial.println(F(" Seconds"));
}

Looks like it's "only" a data conversion problem. The logging itself seems to work, BIN files get created, and I didn't observe any latencies. But data in the csv file is just garbage because the data is not fed with the correct format / variable to the SD card.

How can I solve this problem?

Slowly it's getting better, the code almost does what I want it to do. A *.BIN file is written, and when renaming that *.BIN file to *.TXT file or *.CSV file on my windows machine then the result looks like this:

Time: 21713,A-B: -0.071,A: 0.393,B: 0.464,Additional_Data...
Time: 21733,A-B: 0.026,A: 0.416,B: 0.390,Additional_Data...
Time: 21753,A-B: -0.026,A: 0.412,B:NUL 0.438,Additional_Data...
Time: 21773,A-B: 0.113,A: 0.516,B: 0.403,Additional_Data...
Time: 21793,A-B: 0.016,A: 0.354,B: 0.338,Additional_Data...
Time: 21813,A-B: 0.071,A: 0.435,B: 0.364,Additional_Data...
Time: 21833,A-B: 0.122,A: 0.274,B: 0.151,Additional_Data...
Time: 21853,A-B: -0.084,A: 0.116,B: 0.200,Additional_Data...
Time: 21873,A-B: -0.016,A: 0.293,B: 0.309,Additional_Data...
Time: 21893,A-B: 0.216,A: 0.538,B: 0.322,Additional_Data...
Time: 21913,A-B: -0.061,A: 0.416,B: 0.477,Additional_DaNULta...
Time: 21933,A-B: 0.229,A: 0.480,B: 0.251,Additional_Data...
Time: 21953,A-B: -0.039,A: 0.361,B: 0.400,Additional_Data...
Time: 21973,A-B: -0.132,A: 0.416,B: 0.548,Additional_Data...

There is somewhere a NUL character added every 512 bytes, but I couldn't figure out how to avoid that. Except of that the logged data looks ok, and apparently it's quite fast. The code that I logged the data above is as follows:

  dataString += String("Time: ");
  dataString += String(int(time));
  dataString += String("\t"); 
  dataString += String("A-B: ");
  dataString += String(ADCvalue_0-ADCvalue_1,3);  
  dataString += String("\t");  
  dataString += String("A: ");
  dataString += String(ADCvalue_0,3);
  dataString += String("\t");   
  dataString += String("B: ");
  dataString += String(ADCvalue_1,3); 
  dataString += String("\t");  
  dataString += String("Additional_Data... ");
  dataString += String("\n");  //Create a new line on the SD card

  int length = dataString.length();

  if (length > 512)
  {
    noInterrupts();

    SDbufferString = dataString.substring(0,511);         //Create a string with 512 bytes for writing on the SD card.
    dataString = dataString.substring(511,length);        //Remove the 512 bytes for the SD card from the main string.

    char charBuf[512];
    SDbufferString.toCharArray(charBuf,512);              //Convert String to Char Array
     
    memcpy(pCache, &charBuf, 512);                        //Write binary data to cache

    if (!sd.card()->writeData((uint8_t*)pCache)) ;  

    bn++; //increment block number
    count = 0; 
    memset(pCache, '0', 512); 
    interrupts(); 
  }

The binaryToCsv conversion code does not work, there is only garbage to see later on the computer. But as already mentioned above, when renaming the BIN file to TXT on my computer then the result is as above.

The binaryToCsv code I used is the following:

void binaryToCsv() {
  uint8_t lastPct = 0;
  int buf[512];
  uint32_t t0 = millis();
  uint32_t syncCluster = 0;
  SdFile csvFile;
  char csvName[13];
  BufferedWriter bw;

  if (!myFile.isOpen()) {
    Serial.println(F("No current binary file"));
    return;
  }
  myFile.rewind();
  //if (!binFile.read(&buf , 512) == 512) error("Read metadata failed");
  // create a new csvFile
  strcpy(csvName, binName);
  strcpy_P(&csvName[BASE_NAME_SIZE + 3], PSTR(".CSV"));

  if (!csvFile.open(csvName, O_WRITE | O_CREAT | O_TRUNC)) {
    error("open csvFile failed");  
  }
  Serial.println();
  Serial.print(F("Writing: "));
  Serial.print(csvName);
  Serial.println(F(" - type any character to stop"));
  bw.init(&csvFile);
  bw.putStr_P(PSTR("Time,")); //Write header at top of file
  bw.putStr_P(PSTR("A-B,"));
  bw.putStr_P(PSTR("A,"));
  bw.putStr_P(PSTR("B"));
  bw.putCRLF();

  uint32_t tPct = millis();
  while (!Serial.available() && myFile.read(&buf, 512) == 512) {
    uint16_t i;
    //read 512 bytes of data here i.e. 512 samples
    for (i = 0; i < 512; i++) {
     bw.putChar(buf[i]); 
    }
    
    //bw.putCRLF();

    if (csvFile.curCluster() != syncCluster) {
      bw.writeBuf();
      csvFile.sync();
      syncCluster = csvFile.curCluster();
    }
    //looks like code to print out % done
    if ((millis() - tPct) > 1000) {
      uint8_t pct = myFile.curPosition()/(myFile.fileSize()/100);
      if (pct != lastPct) {
        tPct = millis();
        lastPct = pct;
        Serial.print(pct, DEC);
        Serial.println('%');
      }
    }
    if (Serial.available()) break;
  }
  bw.writeBuf();
  csvFile.close();
  Serial.print(F("Done: "));
  Serial.print(0.001*(millis() - t0));
  Serial.println(F(" Seconds"));
}

I'm not sure if it is more appropriate for me to make a new thread, or to post on your thread since you appear to have relevant experience.

My question: Did you modify any of the SPI controls to account for the teensy 3.1 vs teensy 3.0?

I am struggling with getting sdfat to work in my system. After literal days, from hardware to software problems, I can read, but not write. Since there are some issues with entering spi idle mode I'm trying to investigate how the spi stuff works.

What I did notice was that there's a file called "SdSpimk20dx128.h" which is for the teensy 3.0 presumably. However the teensy 3.1 is defined as mk20dx256, so I'm wondering if I don't need to modify things to account for that.

Any guesses? Did you have this problem?

Thanks.
-Tomek

Are you using this version of SdFat? GitHub - greiman/SdFat: Arduino FAT16/FAT32 exFAT Library

There is a mod in SdSpiMK20DX128.cpp by Paul Stoffregen that fixes SPI problems on Teensy 3.0 and 3.1 with newer versions of the Teensy IDE. This fix may help.

Another option is to edit SdFatConfig.h and use the standard SPI.h library. Change this line.

/**
 * Force use of Arduino Standard SPI library if USE_ARDUINO_SPI_LIBRARY
 * is nonzero.
 */
#define USE_ARDUINO_SPI_LIBRARY 0

I tested this version on Teensy 3.1 using the bench example with 32 KB reads and writes. about 2200 KB/sec write and 2500 KB/sec read.

Use a freshly formatted SD for best performance.

Type any character to start
Free RAM: 28315
Type is FAT32
Card size: 15.93 GB (GB = 1E9 bytes)

Manufacturer ID: 0X3
OEM ID: SD
Product: SE16G
Version: 8.0
Serial number: 0X3752B221
Manufacturing date: 10/2013

File size 5 MB
Buffer size 32768 bytes
Starting write test, please wait.

write speed and latency
speed,max,min,avg
KB/Sec,usec,usec,usec
2260.89,21620,13472,14475
2285.79,21563,13417,14323
2287.89,20618,13423,14306
2292.10,20659,13448,14280
2285.79,20639,13418,14316
2292.10,20628,13449,14279
2278.47,27660,13419,14371
2286.84,20569,13422,14310
2292.10,20587,13448,14279
2286.84,20682,13429,14316

Starting read test, please wait.

read speed and latency
speed,max,min,avg
KB/Sec,usec,usec,usec
2550.30,13285,12824,12847
2551.61,13205,12780,12844
2550.30,13205,12778,12843
2551.61,13207,12778,12843
2551.61,13206,12778,12844

Done

Here is the LowLatencyLogger example logging four analog pins per sample at 2500 samples/sec.

Creating new file
Erasing all data
Logging - type any character to stop
Truncating file
File renamed: DATA03.BIN
Max block write usec: 200
Record time sec: 9.747
Sample count: 24365
Samples/sec: 2499.74
Overruns: 0
Done

Super cool to get a reply from [who I believe to be] the SdFat creator. Thanks!

Ok, to answer the questions:
(1) Yes, I am using the most recent github pull, authored 5 days ago. Originally I was using old code, because the links on the web are a bit messy due to the old google code and people not catching on the fact its on git now. But I've been using the new code for the last two days.

(2)

There is a mod in SdSpiMK20DX128.cpp by Paul Stoffregen that fixes SPI problems on Teensy 3.0 and 3.1 with newer versions of the Teensy IDE. This fix may help.

I might be simply unaware, but that's the file that confused me a little when I looked at it. It seems to only offer a fix for teensy 3.0 since it only specifies the arm processor with the 128 suffix, not the 256 that the teensy 3.1 uses. So I thought maybe something was going on there. When you say there is a mod in the file, do you mean there's a mod, elsewhere, that I need to add to the git file? Or is it already included?

(3) I'll try using the default arduino library. Actually the first time I tried that once something got confused, because the compiler said "spi.h" not found. I didn't pursue that, since I was on a different troubleshooting bent at the time.

(4) Question to sdfat creator or anyone else: Is it common that I can read from my card, open a file and such, but not write to the file? I was thinking that might suggest a lot about what my error is. I can do card.init, vol.init, those basics. but card.begin() fails, in the past I've narrowed it to failing to enter SPI idle mode in sdfat.begin() [which calls card.begin, i believe]. I am doing this on a teensy 3.1.

SdSpiMK20DX128.cpp works for both Teensy 3.0 and Teensy 3.1. It is Paul's version. Ignore the file name it dates from when only Teensy 3.0 existed.

I can do card.init, vol.init, those basics. but card.begin() fails,

Why are you doing this? You should not be using these calls.

You should only need sd.begin() in your code.

Does the bench example work?

The results above are running the GitHub version on a Teensy 3.1 with no mods.

Have you tried any of the SdFat examples ?