LowLatencyLogger Guide

Hi all,

I am a mechanical guy who does not familiar with all this electronic and programming stuff. So please forgive my ignorance.

I am trying to log reading from this i2c IMU (MPU6050) into a microSD card. After a little research on this arduino forum, there are few (unsolved) examples on lowlatencylogger.

I know the author of this SDFat fat16lib had an explanation on how to modified this lowlatencylogger example.

//UserDataType.h
const uint8_t ADC_DIM = 4;
struct data_t {
  unsigned long time;
  unsigned short adc[ADC_DIM];
};

// Acquire a data record.
void acquireData(data_t* data) {
  data->time = micros();
  for (int i = 0; i < ADC_DIM; i++) {
    data->adc[i] = analogRead(i);
  }
}

// Print a data record.
void printData(Print* pr, data_t* data) {
  pr->print(data->time);
  for (int i = 0; i < ADC_DIM; i++) {
    pr->write(',');  
    pr->print(data->adc[i]);
  }
  pr->println();
}

// Print data header.
void printHeader(Print* pr) {
  pr->print(F("time"));
  for (int i = 0; i < ADC_DIM; i++) {
    pr->print(F(",adc"));
    pr->print(i);
  }
  pr->println();
}

So after look at some of the unresolved example, this is what I did :

#include <SPI.h>
#include <SdFat.h>
#include <SdFatUtil.h>
#include "I2Cdev.h"
#include "MPU6050.h" 

#if I2CDEV_IMPLEMENTATION == I2CDEV_ARDUINO_WIRE
    #include "Wire.h"
#endif
//------------------------------------------------------------------------------
// User data functions.  Modify these functions for your data items.
#include "UserDataType.h"  // Edit this include file to change data_t.


//define
MPU6050 mpu;
int16_t  ax, ay, az, gx, gy, gz;

bool dmpReady = false;  // set true if DMP init was successful
uint8_t mpuIntStatus;   // holds actual interrupt status byte from MPU
uint8_t devStatus;      // return status after each device operation (0 = success, !0 = error)
uint16_t packetSize;    // expected DMP packet size (default is 42 bytes)
uint16_t fifoCount;     // count of all bytes currently in FIFO
uint8_t fifoBuffer[64]; // FIFO storage buffer

unsigned long time;

volatile bool mpuInterrupt = false;     // indicates whether MPU interrupt pin has gone high
void dmpDataReady()
{
    mpuInterrupt = true;
}

////////////////////////////////////////////////////////////////////////////

// Acquire a data record.
void acquireData(data_t* data) {
  
  time=millis();
    while (!mpuInterrupt && fifoCount < packetSize) {

    }

    // reset interrupt flag and get INT_STATUS byte
    mpuInterrupt = false;
    mpuIntStatus = mpu.getIntStatus();

    // get current FIFO count
    fifoCount = mpu.getFIFOCount();

    // check for overflow 
    if ((mpuIntStatus & 0x10) || fifoCount == 1024) {
        // reset so we can continue cleanly
        mpu.resetFIFO();
        

   
    } else if (mpuIntStatus & 0x02) {
        // wait for correct available data length, should be a VERY short wait
        while (fifoCount < packetSize) fifoCount = mpu.getFIFOCount();

        // read a packet from FIFO
        mpu.getFIFOBytes(fifoBuffer, packetSize);
        
        // track FIFO count here in case there is > 1 packet available
        // (this lets us immediately read more without waiting for an interrupt)
        fifoCount -= packetSize;
    }
  mpu.getMotion6(&ax, &ay, &az, &gx, &gy, &gz);
  data->time = micros();
  for (int i = 0; i < ADC_DIM; i++) {
    data->time = millis();
    data->adc[1] = ax;
    data->adc[2] = ay;
    data->adc[3] = az;
    data->adc[4] = gx;
    data->adc[5] = gy;
    data->adc[6] = gz;
  }
}

// Print a data record.
void printData(Print* pr, data_t* data) {
  pr->print(data->time);
  for (int i = 0; i < ADC_DIM; i++) {
    pr->write(',');
    pr->print(data->adc[i]);
  }
  pr->println();
}

// Print data header.
void printHeader(Print* pr) {
  pr->print(F("time"));
  for (int i = 0; i < ADC_DIM; i++) {
    pr->print(F(",adc"));
    pr->print(i);
  }
  pr->println();
}
//==============================================================================
void setup(void) {
  if (ERROR_LED_PIN >= 0) {
    pinMode(ERROR_LED_PIN, OUTPUT);
  }
  #if I2CDEV_IMPLEMENTATION == I2CDEV_ARDUINO_WIRE
        Wire.begin();
        TWBR = 24; // 400kHz I2C clock (200kHz if CPU is 8MHz)
    #elif I2CDEV_IMPLEMENTATION == I2CDEV_BUILTIN_FASTWIRE
        Fastwire::setup(400, true);
    #endif
  Serial.begin(115200);
  while (!Serial) {}
 mpu.initialize();
  Serial.print(F("FreeRam: "));
  Serial.println(FreeRam());
  Serial.print(F("Records/block: "));
  Serial.println(DATA_DIM);
  if (sizeof(block_t) != 512) {
    error("Invalid block size");
  }
  // initialize file system.
  if (!sd.begin(SD_CS_PIN, SPI_FULL_SPEED)) {
    sd.initErrorPrint();
    fatalBlink();
  }
}

So one thing I do not understand, i need to type in characters “c”, “d”, “e” and “r” to convert file to csv,dump data to Serial, overrun error details and record data when i am in Serial Monitor. Even after i key in those characters, I still get rubbish results in bin file.

My question is, can I get rid of keying in these characters c,d,e and r to data logging? And why I did not get a csv files as promised and I only get a bin file instead (as shown as the attached file). Those results are unreadable.

Am i doing the correct modification on this lowlatencylogger example for my MPU6050? Can anyone please guide me and correct me if I am wrong?

Thank you.

Hey,

I have recently been exploring different methods of high speed data logging using an arduino. I was capable of making a reliable method for storing a single variable at 40khz for over 10 hours using an ad card module and a teensy 3.1 (arduino compatible but unofficial board).

First a summary of what I have learnt:

  • Using internal arrays is much faster (unsurprising)
  • Using external SRAM Is much faster (again unsurprising)
  • Storing binary data is much faster (yet again unsurprising)
  • SD cards are annoyingly inconsistent in their write times
  • SD cards are allot faster if you write in multiples of their page size (normally 512bytes)
  • Using a double, or triple, buffer system such that data is stored into buff A whilst writing buff B to card enables much fast data storage than writing it one variable at a time

I have not used the SDfat lib but using the normal SD library the file type was purely done by the name you assigned it i.e.

myFile = SD.open("buffTest.bin", FILE_WRITE); 
myFile2 = SD.open("buffTest.txt", FILE_WRITE);

Here is a link to a double buffer SD write speed test that I was doing. You will have to adjust the arrays to allow you to write your two dimensional arrays rather than my one dimensional one.

https://www.dropbox.com/sh/xv1xnyp086ikqfs/AAB_YG-yhGV19qNPDtAzf8nNa?dl=0

slzer,

I have not used the SDfat lib but using the normal SD library the file type was purely done by the name you assigned it i.e.

Actual you are using SdFat. SD.h is just a wrapper for a very old version of SdFat.

The SD.h wrapper limits SPI speed to 4 MHz even though SdFat's default speed is 8 MHz.

The LowLatencyLogger does all the things you suggest for an SD card.

Storing binary data is much faster (yet again unsurprising) SD cards are annoyingly inconsistent in their write times SD cards are allot faster if you write in multiples of their page size (normally 512bytes) Using a double, or triple, buffer system such that data is stored into buff A whilst writing buff B to card enables much fast data storage than writing it one variable at a time

LowLatencyLogger does more. It allocates a large contiguous file and does raw multi-block writes to the file. This increases write speed by avoiding file system overhead and reduces the inconsistency in write latency.

LowLatencyLogger avoids waiting while the SD card is busy by checking the card before writing.

LowLatencyLogger is intended to be easy to modify while providing data logging at up to 500 Hz on AVR boards without using multi-threading or interrupts.

The AnalogBinLogger demonstrates faster logging, up to 25,000 Hz for a single pin. AnalogBinLogger uses interrupts and a timer so it is more difficult to modify for another sensor.

fat16lib: slzer,

Actual you are using SdFat. SD.h is just a wrapper for a very old version of SdFat.

The SD.h wrapper limits SPI speed to 4 MHz even though SdFat's default speed is 8 MHz.

The LowLatencyLogger does all the things you suggest for an SD card. LowLatencyLogger does more. It allocates a large contiguous file and does raw multi-block writes to the file. This increases write speed by avoiding file system overhead and reduces the inconsistency in write latency.

LowLatencyLogger avoids waiting while the SD card is busy by checking the card before writing.

LowLatencyLogger is intended to be easy to modify while providing data logging at up to 500 Hz on AVR boards without using multi-threading or interrupts.

The AnalogBinLogger demonstrates faster logging, up to 25,000 Hz for a single pin. AnalogBinLogger uses interrupts and a timer so it is more difficult to modify for another sensor.

Hmm i didn't realise that it was simply a wrapper for an older version.

I will need to look into the analogBinLogger as the sensors I typically am working with have an analog output so it would suit my needs! But you are write it would be harder to modify it to work with the IMU used in the question!

mechanical_guy,

I assumed most people would call the functions directly and not use the simple Serial interface.

Several people have modified LowLatencyLogger for the MPU6050. Search the forum.

I may provide an example for the MPU6050. The problem is that the MPU6050 can be used in several modes.

You use the FIFO and delay while waiting for data so I am not sure your functions will work.

What data rate do you require? The MPU6050 is slow in "Motion Processing" mode.

fat16lib: mechanical_guy,

I assumed most people would call the functions directly and not use the simple Serial interface.

Several people have modified LowLatencyLogger for the MPU6050. Search the forum.

I may provide an example for the MPU6050. The problem is that the MPU6050 can be used in several modes.

You use the FIFO and delay while waiting for data so I am not sure your functions will work.

What data rate do you require? The MPU6050 is slow in "Motion Processing" mode.

Hi fat16lib,

Thank you so much for your reply. My require sampling rate will be around 512Hz (roughly). I am not using this IMU for motion processing mode but the code I used to modified is from Mr Jeff Rowberg DMP code. Again I am not familiar with all this coding and processing stuff so i assume Mr Jeff code on DMP will require the MPU6050 to work as fast as possible by using FIFO method. What I need to get from MPU6050 is raw data with high sampling rate as I will use them for off line processing.

And for my second question, every time when I wanted to start data logging, is it really necessary to press the characters 'c', 'd', 'e' and 'r' to start recording data? Or it just because the way I modified the lowlatencylogger example was wrong?

Thank you so much.

My require sampling rate will be around 512Hz (roughly).

I don't think you will get close to this rate.

If you look at about line 307 of MPU6050/MPU6050_6Axis_MotionApps20.h

You will find this comment.

   // This very last 0x01 WAS a 0x09, which drops the FIFO rate down to 20 Hz. 0x07 is 25 Hz,
   // 0x01 is 100Hz. Going faster than 100Hz (0x00=200Hz) tends to result in very noisy data.

fat16lib:
I don’t think you will get close to this rate.

Hi fat16lib,

After few rounds of struggles, i had decided not to use Mr Jeff DMP code as shown before because when i started recording data and press any character to stop, it keep stucking at Truncating file. I do not understand where did I get wrong.

So I decided to use the Mr Jeff raw MPU6050 code as and implement it into this lowlatencylogger:

#include <SPI.h>
#include <SdFat.h>
#include <SdFatUtil.h>
#include "Wire.h"
#include "I2Cdev.h"
#include "MPU6050.h"




#include "UserDataType.h"  // Edit this include file to change data_t.

// define MPU6050 constant
MPU6050 accelgyro; 
int16_t ax, ay, az, gx, gy, gz;

unsigned long time;
// Acquire a data record.
void acquireData(data_t* data) {
  time = micros();
  accelgyro.getMotion6(&ax, &ay, &az, &gx, &gy, &gz);
  
  data->time = micros();
    data->adc[0] = ax;
    data->adc[1] = ay;
    data->adc[2] = az;
    data->adc[3] = gx;
    data->adc[4] = gy;
    data->adc[5] = gz;
}


// Print a data record.
void printData(Print* pr, data_t* data) {
  pr->print(data->time);
  for (int i = 0; i < ADC_DIM; i++) {
    pr->write(',');
    pr->print(data->adc[i]);
  }
  pr->println();
}

// Print data header.
void printHeader(Print* pr) {
  pr->print(F("time"));
  for (int i = 0; i < ADC_DIM; i++) {
    pr->print(F(",adc"));
    pr->print(i);
  }
  pr->println();
}

const uint32_t LOG_INTERVAL_USEC = 5000;

const uint8_t SD_CS_PIN = 8;


////////////////////////////


[/void setup(void) {
  if (ERROR_LED_PIN >= 0) {
    pinMode(ERROR_LED_PIN, OUTPUT);
  }
  
        #if I2CDEV_IMPLEMENTATION == I2CDEV_ARDUINO_WIRE
        Wire.begin();
          TWBR = 24;
    #elif I2CDEV_IMPLEMENTATION == I2CDEV_BUILTIN_FASTWIRE
        Fastwire::setup(400, true);
    #endif
  Serial.begin(38400);
  
  
  
  
 
  accelgyro.initialize();

  Serial.print(F("FreeRam: "));
  Serial.println(FreeRam());
  Serial.print(F("Records/block: "));
  Serial.println(DATA_DIM);
  if (sizeof(block_t) != 512) {
    error("Invalid block size");
  }
  // initialize file system.
  if (!sd.begin(SD_CS_PIN, SPI_FULL_SPEED)) {
    sd.initErrorPrint();
    fatalBlink();
  }
}

I can successfully convert the recorded data into csv file as shown as attached file. As you can see the results were weird. When I put mpu6050 at rest, the raw accelerometer reading suppose to be around 16000 when facing to gravity. Instead I got 66000 as a result.

Any idea why is this happening??

Thank you.

The attached file as mentioned.

Document.txt (35.2 KB)

Try the attached MPU6050RawLowLatencyLogger.zip. You will need to edit

// SD chip select pin.
const uint8_t SD_CS_PIN = 4;

The MPU6050 int is connected to pin 2 but I don’t think that matters in raw mode.

Here is data for MPU6050 at rest component side up:

micros,ax,ay,az,gx,gy,gz
3246008,1948,-108,15708,180,-402,-286
3248008,1968,-124,15472,218,-381,-309
3250008,1880,-96,15432,238,-373,-278
3252008,1908,-12,15744,227,-371,-253

See data07.txt. I moved the MPU6050 during the run.

data07.txt (79.1 KB)

MPU6050RawLowLatencyLogger.zip (5 KB)

fat16lib: Try the attached MPU6050RawLowLatencyLogger.zip. You will need to edit

// SD chip select pin.
const uint8_t SD_CS_PIN = 4;

The MPU6050 int is connected to pin 2 but I don't think that matters in raw mode.

Here is data for MPU6050 at rest component side up: See data07.txt. I moved the MPU6050 during the run.

THANK YOU THE GREAT fat16lib SO SO MUCH!!!

It works exactly I want!!!

Sincerely thank you for you guidance (although my code is totally trash and you totally just passed me your code). Thanks again.

How to stop data logging?

I started to realize that in order to truncate the file I need to key in any character on Serial monitor to stop data logging. However most of the time my arduino will not be connected to ladtop and I cannot key in any character to stop logging earlier (since i have no keyboard on arduino).

From my understanding now, everytime I start logging data, a temporary file will be open and data will be written in it. If this file is full (based on FILE_BLOCK_COUNT), then this temporary file will be deleted and it will be converted to binary file. However, if I wish to stop earlier before the file is full, I need to key in character on serial monitor to truncate the file which is impossible without a pc.

So my question is, how to stop data logging before the file is full without key in any character to terminate data logging?

Thank you.

Hi,

Thanks fat16lib for the low latency logger library.

I just wanted to add that the sketch posted above, combining LowLatencyLogger and MPU6050 Raw values also should have the line:

Serial.println("Initializing I2C devices...");
    mpu.initialize();

Otherwise you will just get 0 values out of the mpu, (unless you have already initialised the mpu using another sketch while still powered up).

mechanical_guy:
How to stop data logging?

You may have solved your issue, but here’s what I do. I use a 5v Wireless Remote 4-Channel Receiver Board RF Control Switch that sells for under $5 to start and stop the data logging. I also use a buzzer that gives me feedback when measurements start & stop. For example buzz(3) means the buzzer will beep 3 times.

void loop(void) {
 while (Serial.read() >= 0) {}
   radioState = digitalRead(radioPin);               // read remote on/off switch
 if((radioState == LOW) && (measureState > 0)){      // continue to read ADXL, measureState = 1 ->measure
    logData();
  }  
 if((radioState == HIGH) && (measureState < 0)){      // change the state.  Start measure
   measureState = -measureState;
   startFlag = true;
   stopFlag = false;
   //wait for switchState to go LOW
    do{
      radioState = digitalRead(radioPin);
    }while (radioState == HIGH);
   }  
 if((radioState == HIGH) && (measureState > 0)){      // change the state.  Stop measure 
        measureState = -measureState;
        stopFlag = true;
        startFlag = false;

        //wait for switchState to go LOW
        do{
          radioState = digitalRead(radioPin);
        }while (radioState == HIGH);
      }
      if(startFlag){
        if(buzzFirst){
          buzz(2);
          buzzFirst = false;
        }
        logData();
        storeOnce = true;
      }
      if(stopFlag && storeOnce){
        buzz(3);
        
        binaryToCsv();                     // save to SD CSV
        buzz(4);                             // done saving, ready for next data logging
 
        
        storeOnce = false;
        buzzFirst = true;
        
      }
}

Gerry48: You may have solved your issue, but here's what I do. I use a 5v Wireless Remote 4-Channel Receiver Board RF Control Switch that sells for under $5 to start and stop the data logging. I also use a buzzer that gives me feedback when measurements start & stop. For example buzz(3) means the buzzer will beep 3 times.

wow, cool idea....thanks for the suggestion...will give it a try when i am free...thank you very much...

Just a heads up for anyone trying to use this MPU6050RawLowLatencyLogger code kindly provided by fat16lib on an ARM processor.

I am using an adafruit feather M0. Changing the UserDataType.h file fixed the following compile problem.

C:\Users\Bull\Documents\Arduino\MPU6050RawLowLatencyLogger\MPU6050RawLowLatencyLogger.ino: In function 'void acquireData(data_t*)':

MPU6050RawLowLatencyLogger:31: error: no matching function for call to 'MPU6050::getMotion6(int*, int*, int*, int*, int*, int*)'

   &data->gx, &data->gy, &data->gz);

                                  ^

C:\Users\Bull\Documents\Arduino\MPU6050RawLowLatencyLogger\MPU6050RawLowLatencyLogger.ino:31:34: note: candidate is:

In file included from C:\Users\Bull\Documents\Arduino\MPU6050RawLowLatencyLogger\MPU6050RawLowLatencyLogger.ino:21:0:

C:\Users\Bull\Documents\Arduino\libraries\MPU6050/MPU6050.h:624:14: note: void MPU6050::getMotion6(int16_t*, int16_t*, int16_t*, int16_t*, int16_t*, int16_t*)

         void getMotion6(int16_t* ax, int16_t* ay, int16_t* az, int16_t* gx, int16_t* gy, int16_t* gz);

              ^

C:\Users\Bull\Documents\Arduino\libraries\MPU6050/MPU6050.h:624:14: note:   no known conversion for argument 1 from 'int*' to 'int16_t* {aka short int*}'

Using library SPI at version 1.0 in folder: C:\Users\Bull\AppData\Local\Arduino15\packages\adafruit\hardware\samd\1.0.11\libraries\SPI 
Using library SdFat at version 2015.4.26 in folder: C:\Users\Bull\Documents\Arduino\libraries\SdFat 
Using library Wire at version 1.0 in folder: C:\Users\Bull\AppData\Local\Arduino15\packages\adafruit\hardware\samd\1.0.11\libraries\Wire 
Using library I2Cdev in folder: C:\Users\Bull\Documents\Arduino\libraries\I2Cdev (legacy)
Using library MPU6050 in folder: C:\Users\Bull\Documents\Arduino\libraries\MPU6050 (legacy)
exit status 1
no matching function for call to 'MPU6050::getMotion6(int*, int*, int*, int*, int*, int*)'

You will need to change the integer type in UserDataType.h for it to work.

int needs to be changed to int16_t.

#ifndef UserDataType_h
#define UserDataType_h
struct data_t {
  unsigned long time;
  int16_t ax;
  int16_t ay;
  int16_t az;
  int16_t gx;
  int16_t gy;
  int16_t gz;
};
#endif  // UserDataType_h

fat16lib:
Try the attached MPU6050RawLowLatencyLogger.zip. You will need to edit

// SD chip select pin.

const uint8_t SD_CS_PIN = 4;




The MPU6050 int is connected to pin 2 but I don't think that matters in raw mode.

Here is data for MPU6050 at rest component side up:
See data07.txt. I moved the MPU6050 during the run.

many thanks for your code.

which software (in windows) can read these “.bin” files?