Go Down

Topic: Using buffers to log data quickly (Read 137 times) previous topic - next topic

dragonofthewest

I am using an Arduino Uno and Adafruit datalogging shield.  I need to be able to log all the data to an SD card every millisecond.  How do i incorporate buffers into my code so I can write data to the card from one buffer while the data is constantly being collected to the other buffer?

Here is my code

#include <SD.h>
#include <Wire.h>
#include "RTClib.h"
#include "DHT.h"
// A simple data logger for the Arduino analog pins

// how many milliseconds between grabbing data and logging it. 1000 ms is once a second
#define LOG_INTERVAL  10 // mills between entries (ECG)

#define SYNC_INTERVAL (1) // mills between calls to flush() - to write data to the card
uint32_t syncTime = 0; // time of last sync()

#define ECHO_TO_SERIAL   0 // echo data to serial port
#define WAIT_TO_START    0 // Wait for serial input in setup()



// the digital pins that connect to the LEDs
#define DHTPin 3    // what pin we're connected to
#define DHTTYPE DHT22

// The analog pins that connect to the sensors
#define ecgPin 0
#define C02Pin 1

DHT dht(DHTPin, DHTTYPE);
RTC_DS1307 RTC; // define the Real Time Clock object

// for the data logging shield, we use digital pin 10 for the SD cs line
const int chipSelect = 10;

// the logging file
File logfile;

void error(char *str)
{
  Serial.print("error: ");
  Serial.println(str);
 


  while(1);
}

void setup(void)
{
  Serial.begin(9600);
  Serial.println();
  dht.begin();
 

 
#if WAIT_TO_START
  Serial.println("Type any character to start");
  while (!Serial.available());
#endif //WAIT_TO_START

  // initialize the SD card
  Serial.print("Initializing SD card...");
  // make sure that the default chip select pin is set to
  // output, even if you don't use it:
  pinMode(10, OUTPUT);
 
  // see if the card is present and can be initialized:
  if (!SD.begin(chipSelect)) {
    error("Card failed, or not present");
  }
  Serial.println("card initialized.");
 
  // create a new file
  char filename[] = "LOGGER00.CSV";
  for (uint8_t i = 0; i < 100; i++) {
    filename[6] = i/10 + '0';
    filename[7] = i%10 + '0';
    if (! SD.exists(filename)) {
      // only open a new file if it doesn't exist
      logfile = SD.open(filename, FILE_WRITE);
      break;  // leave the loop!
    }
  }
 
  if (! logfile) {
    error("couldnt create file");
  }
 
  Serial.print("Logging to: ");
  Serial.println(filename);

  // connect to RTC
  Wire.begin(); 
  if (!RTC.begin()) {
    logfile.println("RTC failed");
#if ECHO_TO_SERIAL
    Serial.println("RTC failed");
#endif  //ECHO_TO_SERIAL
  }
 

  logfile.println("millis,stamp,datetime,ecg,C02,rh,temp_c,temp_f,sat_vapor_density,vapor_density");   
#if ECHO_TO_SERIAL
  Serial.println("millis,stamp,datetime,ecg,C02,rh,temp_c,temp_f,sat_vapor_density,vapor_density");
#endif //ECHO_TO_SERIAL

}
 
  // If you want to set the aref to something other than 5v
  //analogReference(EXTERNAL);



void loop(void)
{
  DateTime now;

  // delay for the amount of time we want between readings
  delay((LOG_INTERVAL -1) - (millis() % LOG_INTERVAL));
 
  String dataString = "";

 
  // log milliseconds since starting
  uint32_t m = millis();
  logfile.print(m);           // milliseconds since start
  logfile.print(", ");   
#if ECHO_TO_SERIAL
  Serial.print(m);         // milliseconds since start
  Serial.print(", "); 
#endif

  // fetch the time
  now = RTC.now();
  // log time
  logfile.print(now.unixtime()); // seconds since 1/1/1970
  logfile.print(", ");
  logfile.print('"');
  logfile.print(now.year(), DEC);
  logfile.print("/");
  logfile.print(now.month(), DEC);
  logfile.print("/");
  logfile.print(now.day(), DEC);
  logfile.print(" ");
  logfile.print(now.hour(), DEC);
  logfile.print(":");
  logfile.print(now.minute(), DEC);
  logfile.print(":");
  logfile.print(now.second(), DEC);
  logfile.print('"');
#if ECHO_TO_SERIAL
  Serial.print(now.unixtime()); // seconds since 1/1/1970
  Serial.print(", ");
  Serial.print('"');
  Serial.print(now.year(), DEC);
  Serial.print("/");
  Serial.print(now.month(), DEC);
  Serial.print("/");
  Serial.print(now.day(), DEC);
  Serial.print(" ");
  Serial.print(now.hour(), DEC);
  Serial.print(":");
  Serial.print(now.minute(), DEC);
  Serial.print(":");
  Serial.print(now.second(), DEC);
  Serial.print('"');
#endif //ECHO_TO_SERIAL

  analogRead(ecgPin);
  delay(0);
  int ecgReading = analogRead(ecgPin); 

  analogRead(C02Pin);
  delay(0);
  int C02Reading = analogRead(C02Pin);

  logfile.print(", ");   
  logfile.print(ecgReading);
  logfile.print(", ");   
  logfile.print(C02Reading);
  logfile.print(", ");
#if ECHO_TO_SERIAL
  logfile.print(", ");   
  logfile.print(ecgReading);
  logfile.print(", ");   
  logfile.print(C02Reading);
  logfile.print(", ");
#endif //ECHO_TO_SERIAL
 
 // Reading temperature or humidity takes about 250 milliseconds!
  // Sensor readings may also be up to 2 seconds 'old' (its a very slow sensor)
  float h = dht.readHumidity(); // relative humidity, %
  float t_c = dht.readTemperature(); // air temp, degC
  float t_f = t_c*9.0/5.0 + 32.0; // air temp, degF
  float svd; // saturation vapor density, g/m3
  float vd; // vapor density, g/m3

  // check if returns are valid, if they are NaN (not a number) then something went wrong!
  if (isnan(t_c) || isnan(h)) {
    Serial.println("Failed to read from DHT");
  } else {
    svd = 5.018 + 0.32321*t_c + 8.1847e-3*pow(t_c,2) + 3.1243e-4*pow(t_c,3);
    vd = h/100*svd;
    logfile.print(",");   
    logfile.print(h, 2);
    logfile.print(",");   
    logfile.print(t_c, 2);
    logfile.print(",");   
    logfile.print(t_f, 2);
    logfile.print(",");   
    logfile.print(svd, 4);
    logfile.print(",");   
    logfile.print(vd, 4);
#if ECHO_TO_SERIAL
    Serial.print(",");   
    Serial.print(h, 2);
    Serial.print(",");   
    Serial.print(t_c, 2);
    Serial.print(",");   
    Serial.print(t_f, 2);
    Serial.print(",");   
    Serial.print(svd, 4);
    Serial.print(",");   
    Serial.print(vd, 4);
#endif //ECHO_TO_SERIAL
  }
 
 
  logfile.println();
#if ECHO_TO_SERIAL
  Serial.println();
#endif // ECHO_TO_SERIAL



  // Now we write data to disk! Don't sync too often - requires 2048 bytes of I/O to SD card
  // which uses a bunch of power and takes time
  if ((millis() - syncTime) < SYNC_INTERVAL) return;
  syncTime = millis();
 

  logfile.flush();

 
}


jremington

#1
Aug 19, 2016, 01:24 am Last Edit: Aug 19, 2016, 01:55 am by jremington
Please follow the instructions in "How to use this forum" and post your code properly, with code tags.

MarkT

#2
Aug 19, 2016, 01:02 pm Last Edit: Aug 19, 2016, 01:03 pm by MarkT
The normal strategy would be to make the I/O interrupt driven.

Or perhaps the data gathering could be interrupt driven and the I/O normal?

You appear to be logging data every 10ms, yet flushing the SDcard every 1ms.  That's sounds
very wrong.
[ I will NOT respond to personal messages, I WILL delete them, use the forum please ]

mrburnette

Quote
I am using an Arduino Uno and Adafruit datalogging shield.  I need to be able to log all the data to an SD card every millisecond.  How do i incorporate buffers into my code so I can write data to the card from one buffer while the data is constantly being collected to the other buffer?
When asked about buffering (double-buffering), I usually point to the Adafruit GPS library as a study example.

But I'm disturbed by you presumption that the SD card write is your bottleneck.  You should run your code full-speed and use micros() to time the start and end.  This will profile your base code.

If your base code is over 1mS (or even very close) then you are going to need to put effort into refining/optimizing the main routine.

Generally speaking, any use of delay() in the main loop is not good practice.  Use micros() and a variable to set-up the control structure for the timing.

I do not see any direct usage for the input data within the program... that is, the sketch seems to just send data to the SD card.  You "could" save processing time by writing sensor raw data to the SD and later post-processing the data for temperature conversion, svd, etc.

With optimizations, if the full-loop with SD write is still > 1mS you may wish to consider off-loading SD to a dedicated uC.  An example.  You can move the serial speed up to 19.2K BAUD or higher until serial latency is a non-concern.


Ray


Henradrie

One thing I've done for my project is use a serial datalogger called serialghost to take care of it. Data gets sent there using the UART serial port and the logger deals with it. This makes data collection quick and easy.

Go Up
 


Please enter a valid email to subscribe

Confirm your email address

We need to confirm your email address.
To complete the subscription, please click the link in the email we just sent you.

Thank you for subscribing!

Arduino
via Egeo 16
Torino, 10131
Italy