Data logger with file timestamps and C++ I/O operators

Here is a data logging sketch that I developed for a shield with an SD card and an optional DS1307 RTC.

It illustrates new features in V2 of the SdFat library.

It has time stamped files - Windows dir listing

04/06/2011 07:30 AM 684 LOGGER15.CSV
04/06/2011 07:51 AM 4,954 LOGGER16.CSV
04/06/2011 07:55 AM 606 LOGGER17.CSV
04/06/2011 08:02 AM 9,665 LOGGER18.CSV

C++ style I/O for simpler and better data formatting.

Code to print time and date to Serial.

  DateTime now = RTC.now();
  cout  << now << endl;

The result is:

2011/4/6 7:57:41

Sample logfile - note zero fill in minutes and seconds.

millis,date time,sens0,sens1,sens2
141000,2011/4/6 7:59:58,211,209,203
142000,2011/4/6 7:59:59,211,209,203
143000,2011/4/6 8:00:00,211,208,203
144000,2011/4/6 8:00:01,211,208,203
145000,2011/4/6 8:00:02,211,208,203
146000,2011/4/6 8:00:03,210,209,203

Better error handling and messages.

init failure message:

Can’t access SD card. Do not reformat.
SD errorCode: 1
No card or SPI problem?

To use this sketch install RTClib:

Install the new beta version of SdFat:

http://code.google.com/p/beta-lib/downloads/list

Here is the code:

// A simple data logger for the Arduino analog pins
#define LOG_INTERVAL  1000  // mills between entries
#define SENSOR_COUNT     3  // number of analog pins to log
#define ECHO_TO_SERIAL   1  // echo data to serial port
#define WAIT_TO_START    1  // Wait for serial input in setup()
#define ADC_DELAY       10  // switch ADC and delay for high impedence sensors
#define USE_DS1307       1  // shield has DS1307 RTC
#include <SdFat.h>
#include <SdFatUtil.h>  // define FreeRam()

// file system object
SdFat sd;

// text file for logging
ofstream logfile;

// Serial print stream
ArduinoOutStream cout(Serial);

// buffer to format data - makes it eaiser to echo to Serial
char buf[80];
//------------------------------------------------------------------------------
#if SENSOR_COUNT > 6
#error SENSOR_COUNT too large
#endif  // SENSOR_COUNT
//------------------------------------------------------------------------------
// store error strings in flash to save RAM
#define error(s) sd.errorHalt_P(PSTR(s))
//------------------------------------------------------------------------------
#if USE_DS1307
#include <Wire.h>
// get RTClib from Adafruit here
// https://github.com/adafruit/RTClib
#include "RTClib.h"

RTC_DS1307 RTC; // define the Real Time Clock object

// call back for file timestamps
void dateTime(uint16_t* date, uint16_t* time) {
    DateTime now = RTC.now();

  // return date using FAT_DATE macro to format fields
  *date = FAT_DATE(now.year(), now.month(), now.day());

  // return time using FAT_TIME macro to format fields
  *time = FAT_TIME(now.hour(), now.minute(), now.second());
}
// format date/time
ostream& operator << (ostream& os, DateTime& dt) {
  os << dt.year() << '/' << int(dt.month()) << '/' << int(dt.day()) << ' ';
  os << int(dt.hour()) << ':' << setfill('0') << setw(2) << int(dt.minute());
  os << ':' << setw(2) << int(dt.second()) << setfill(' ');
  return os;
}
#endif  // USE_DS1307
//------------------------------------------------------------------------------
void setup() {
  Serial.begin(9600);
  
  // pstr stores strings in flash to save RAM
  cout << endl << pstr("FreeRam: ") << FreeRam() << endl;

#if WAIT_TO_START
  cout << pstr("Type any character to start\n");
  while (!Serial.available());
#endif  // WAIT_TO_START

#if USE_DS1307
  // connect to RTC
  Wire.begin();
  if (!RTC.begin()) error("RTC failed");

  // set date time callback function
  SdFile::dateTimeCallback(dateTime);
  DateTime now = RTC.now();
  cout  << now << endl;
#endif  // USE_DS1307

  // initialize the SD card at SPI_HALF_SPEED to avoid bus errors with
  // breadboards.  use SPI_FULL_SPEED for better performance.
  // if SD chip select is not SS, the second argument to init is CS pin number
  if (!sd.init(SPI_HALF_SPEED)) sd.initErrorHalt();

  // create a new file in root, the current working directory
  char name[] = "LOGGER00.CSV";
  
  for (uint8_t i = 0; i < 100; i++) {
    name[6] = i/10 + '0';
    name[7] = i%10 + '0';
    if (sd.exists(name)) continue;
    logfile.open(name);
    break;
  }
  if (!logfile.is_open()) error("file.open");

  cout << pstr("Logging to: ") << name << endl;

  //format header in buffer
  obufstream bout(buf, sizeof(buf));
  
  bout << pstr("millis");
  
#if USE_DS1307
  bout << pstr(",date time");
#endif USE_DS1307

  for (uint8_t i = 0; i < SENSOR_COUNT; i++) {
    bout << pstr(",sens") << int(i);
  }
  logfile << buf << endl;
  
#if ECHO_TO_SERIAL
  cout << buf << endl;
#endif  // ECHO_TO_SERIAL
}
//------------------------------------------------------------------------------
void loop() {
  uint32_t m;

  // wait for time to be a multiple of interval
  do {
    m = millis();
  } while(m % LOG_INTERVAL);

  // use buffer stream to format line
  obufstream bout(buf, sizeof(buf));

  // start with time in millis
  bout << m;
  
#if USE_DS1307
  DateTime now = RTC.now();
  bout << ',' << now;
#endif
  
  // read analog pins and format data
  for (uint8_t ia = 0; ia < SENSOR_COUNT; ia++) {
#if ADC_DELAY
    analogRead(ia);
    delay(ADC_DELAY);
#endif  // ADC_DELAY
    bout << ',' << analogRead(ia);
  }
  bout << endl;
  
  // log data
  logfile << buf;
  
  // flush data to SD
  logfile.flush();
  
  // check for error
  if (!logfile) error("write data failed");
  
#if ECHO_TO_SERIAL
  cout << buf;
#endif  // ECHO_TO_SERIAL

  // don't log two points in the same millis
  if (m == millis()) delay(1);
}

When time permits I''ll give it a try, did the size of the lib became bigger?

Hope I get a chance to try this out.

The size is about the same for sketches that only use features in the previous version. I added path names and I rewrote the SPI SD access functions to be more robust and play better with other libraries. That added a few bytes.

The new features, C++ I/O, Serial streams, and error handlers are larger. I have found that complex sketches that parse or format a lot text are not larger since the C++ extraction and insertion operators require less user code.

Hi all,

i know this is a very old post and sorry for bringing it up. I am currently using SDFat lowlatencylogger to log MPU6050, so far everything went well until i realize I need to include timestamp for each sample i receive from the MPU6050. I came across this very old post and realise this is actually what I need. However I am not doing analog sensor.
So can anyone guide me on how to modify this code below to fit my purpose?

// A simple data logger for the Arduino analog pins
#define LOG_INTERVAL  1000  // mills between entries
#define SENSOR_COUNT     3  // number of analog pins to log
#define ECHO_TO_SERIAL   1  // echo data to serial port
#define WAIT_TO_START    1  // Wait for serial input in setup()
#define ADC_DELAY       10  // switch ADC and delay for high impedence sensors
#define USE_DS1307       1  // shield has DS1307 RTC
#include <SdFat.h>
#include <SdFatUtil.h>  // define FreeRam()

// file system object
SdFat sd;

// text file for logging
ofstream logfile;

// Serial print stream
ArduinoOutStream cout(Serial);

// buffer to format data - makes it eaiser to echo to Serial
char buf[80];
//------------------------------------------------------------------------------
#if SENSOR_COUNT > 6
#error SENSOR_COUNT too large
#endif  // SENSOR_COUNT
//------------------------------------------------------------------------------
// store error strings in flash to save RAM
#define error(s) sd.errorHalt_P(PSTR(s))
//------------------------------------------------------------------------------
#if USE_DS1307
#include <Wire.h>
// get RTClib from Adafruit here
// https://github.com/adafruit/RTClib
#include "RTClib.h"

RTC_DS1307 RTC; // define the Real Time Clock object

// call back for file timestamps
void dateTime(uint16_t* date, uint16_t* time) {
   DateTime now = RTC.now();

 // return date using FAT_DATE macro to format fields
 *date = FAT_DATE(now.year(), now.month(), now.day());

 // return time using FAT_TIME macro to format fields
 *time = FAT_TIME(now.hour(), now.minute(), now.second());
}
// format date/time
ostream& operator << (ostream& os, DateTime& dt) {
 os << dt.year() << '/' << int(dt.month()) << '/' << int(dt.day()) << ' ';
 os << int(dt.hour()) << ':' << setfill('0') << setw(2) << int(dt.minute());
 os << ':' << setw(2) << int(dt.second()) << setfill(' ');
 return os;
}
#endif  // USE_DS1307
//------------------------------------------------------------------------------
void setup() {
 Serial.begin(9600);
 
 // pstr stores strings in flash to save RAM
 cout << endl << pstr("FreeRam: ") << FreeRam() << endl;

#if WAIT_TO_START
 cout << pstr("Type any character to start\n");
 while (!Serial.available());
#endif  // WAIT_TO_START

#if USE_DS1307
 // connect to RTC
 Wire.begin();
 if (!RTC.begin()) error("RTC failed");

 // set date time callback function
 SdFile::dateTimeCallback(dateTime);
 DateTime now = RTC.now();
 cout  << now << endl;
#endif  // USE_DS1307

 // initialize the SD card at SPI_HALF_SPEED to avoid bus errors with
 // breadboards.  use SPI_FULL_SPEED for better performance.
 // if SD chip select is not SS, the second argument to init is CS pin number
 if (!sd.init(SPI_HALF_SPEED)) sd.initErrorHalt();

 // create a new file in root, the current working directory
 char name[] = "LOGGER00.CSV";
 
 for (uint8_t i = 0; i < 100; i++) {
   name[6] = i/10 + '0';
   name[7] = i%10 + '0';
   if (sd.exists(name)) continue;
   logfile.open(name);
   break;
 }
 if (!logfile.is_open()) error("file.open");

 cout << pstr("Logging to: ") << name << endl;

 //format header in buffer
 obufstream bout(buf, sizeof(buf));
 
 bout << pstr("millis");
 
#if USE_DS1307
 bout << pstr(",date time");
#endif USE_DS1307

 for (uint8_t i = 0; i < SENSOR_COUNT; i++) {
   bout << pstr(",sens") << int(i);
 }
 logfile << buf << endl;
 
#if ECHO_TO_SERIAL
 cout << buf << endl;
#endif  // ECHO_TO_SERIAL
}
//------------------------------------------------------------------------------
void loop() {
 uint32_t m;

 // wait for time to be a multiple of interval
 do {
   m = millis();
 } while(m % LOG_INTERVAL);

 // use buffer stream to format line
 obufstream bout(buf, sizeof(buf));

 // start with time in millis
 bout << m;
 
#if USE_DS1307
 DateTime now = RTC.now();
 bout << ',' << now;
#endif
 
 // read analog pins and format data
 for (uint8_t ia = 0; ia < SENSOR_COUNT; ia++) {
#if ADC_DELAY
   analogRead(ia);
   delay(ADC_DELAY);
#endif  // ADC_DELAY
   bout << ',' << analogRead(ia);
 }
 bout << endl;
 
 // log data
 logfile << buf;
 
 // flush data to SD
 logfile.flush();
 
 // check for error
 if (!logfile) error("write data failed");
 
#if ECHO_TO_SERIAL
 cout << buf;
#endif  // ECHO_TO_SERIAL

 // don't log two points in the same millis
 if (m == millis()) delay(1);
}

Thank you in advance.