Using buffers to log data quickly

I am using an Arduino Uno and Adafruit datalogging shield. I need to be able to log all the data to an SD card every millisecond. How do i incorporate buffers into my code so I can write data to the card from one buffer while the data is constantly being collected to the other buffer?

Here is my code

#include <SD.h>
#include <Wire.h>
#include "RTClib.h"
#include "DHT.h"
// A simple data logger for the Arduino analog pins

// how many milliseconds between grabbing data and logging it. 1000 ms is once a second
#define LOG_INTERVAL 10 // mills between entries (ECG)

#define SYNC_INTERVAL (1) // mills between calls to flush() - to write data to the card
uint32_t syncTime = 0; // time of last sync()

#define ECHO_TO_SERIAL 0 // echo data to serial port
#define WAIT_TO_START 0 // Wait for serial input in setup()

// the digital pins that connect to the LEDs
#define DHTPin 3 // what pin we're connected to
#define DHTTYPE DHT22

// The analog pins that connect to the sensors
#define ecgPin 0
#define C02Pin 1

DHT dht(DHTPin, DHTTYPE);
RTC_DS1307 RTC; // define the Real Time Clock object

// for the data logging shield, we use digital pin 10 for the SD cs line
const int chipSelect = 10;

// the logging file
File logfile;

void error(char *str)
{
Serial.print("error: ");
Serial.println(str);

while(1);
}

void setup(void)
{
Serial.begin(9600);
Serial.println();
dht.begin();

#if WAIT_TO_START
Serial.println("Type any character to start");
while (!Serial.available());
#endif //WAIT_TO_START

// initialize the SD card
Serial.print("Initializing SD card...");
// make sure that the default chip select pin is set to
// output, even if you don't use it:
pinMode(10, OUTPUT);

// see if the card is present and can be initialized:
if (!SD.begin(chipSelect)) {
error("Card failed, or not present");
}
Serial.println("card initialized.");

// create a new file
char filename[] = "LOGGER00.CSV";
for (uint8_t i = 0; i < 100; i++) {
filename[6] = i/10 + '0';
filename[7] = i%10 + '0';
if (! SD.exists(filename)) {
// only open a new file if it doesn't exist
logfile = SD.open(filename, FILE_WRITE);
break; // leave the loop!
}
}

if (! logfile) {
error("couldnt create file");
}

Serial.print("Logging to: ");
Serial.println(filename);

// connect to RTC
Wire.begin();
if (!RTC.begin()) {
logfile.println("RTC failed");
#if ECHO_TO_SERIAL
Serial.println("RTC failed");
#endif //ECHO_TO_SERIAL
}

logfile.println("millis,stamp,datetime,ecg,C02,rh,temp_c,temp_f,sat_vapor_density,vapor_density");
#if ECHO_TO_SERIAL
Serial.println("millis,stamp,datetime,ecg,C02,rh,temp_c,temp_f,sat_vapor_density,vapor_density");
#endif //ECHO_TO_SERIAL

}

// If you want to set the aref to something other than 5v
//analogReference(EXTERNAL);

void loop(void)
{
DateTime now;

// delay for the amount of time we want between readings
delay((LOG_INTERVAL -1) - (millis() % LOG_INTERVAL));

String dataString = "";

// log milliseconds since starting
uint32_t m = millis();
logfile.print(m); // milliseconds since start
logfile.print(", ");
#if ECHO_TO_SERIAL
Serial.print(m); // milliseconds since start
Serial.print(", ");
#endif

// fetch the time
now = RTC.now();
// log time
logfile.print(now.unixtime()); // seconds since 1/1/1970
logfile.print(", ");
logfile.print('"');
logfile.print(now.year(), DEC);
logfile.print("/");
logfile.print(now.month(), DEC);
logfile.print("/");
logfile.print(now.day(), DEC);
logfile.print(" ");
logfile.print(now.hour(), DEC);
logfile.print(":");
logfile.print(now.minute(), DEC);
logfile.print(":");
logfile.print(now.second(), DEC);
logfile.print('"');
#if ECHO_TO_SERIAL
Serial.print(now.unixtime()); // seconds since 1/1/1970
Serial.print(", ");
Serial.print('"');
Serial.print(now.year(), DEC);
Serial.print("/");
Serial.print(now.month(), DEC);
Serial.print("/");
Serial.print(now.day(), DEC);
Serial.print(" ");
Serial.print(now.hour(), DEC);
Serial.print(":");
Serial.print(now.minute(), DEC);
Serial.print(":");
Serial.print(now.second(), DEC);
Serial.print('"');
#endif //ECHO_TO_SERIAL

analogRead(ecgPin);
delay(0);
int ecgReading = analogRead(ecgPin);

analogRead(C02Pin);
delay(0);
int C02Reading = analogRead(C02Pin);

logfile.print(", ");
logfile.print(ecgReading);
logfile.print(", ");
logfile.print(C02Reading);
logfile.print(", ");
#if ECHO_TO_SERIAL
logfile.print(", ");
logfile.print(ecgReading);
logfile.print(", ");
logfile.print(C02Reading);
logfile.print(", ");
#endif //ECHO_TO_SERIAL

// Reading temperature or humidity takes about 250 milliseconds!
// Sensor readings may also be up to 2 seconds 'old' (its a very slow sensor)
float h = dht.readHumidity(); // relative humidity, %
float t_c = dht.readTemperature(); // air temp, degC
float t_f = t_c*9.0/5.0 + 32.0; // air temp, degF
float svd; // saturation vapor density, g/m3
float vd; // vapor density, g/m3

// check if returns are valid, if they are NaN (not a number) then something went wrong!
if (isnan(t_c) || isnan(h)) {
Serial.println("Failed to read from DHT");
} else {
svd = 5.018 + 0.32321t_c + 8.1847e-3pow(t_c,2) + 3.1243e-4pow(t_c,3);
vd = h/100
svd;
logfile.print(",");
logfile.print(h, 2);
logfile.print(",");
logfile.print(t_c, 2);
logfile.print(",");
logfile.print(t_f, 2);
logfile.print(",");
logfile.print(svd, 4);
logfile.print(",");
logfile.print(vd, 4);
#if ECHO_TO_SERIAL
Serial.print(",");
Serial.print(h, 2);
Serial.print(",");
Serial.print(t_c, 2);
Serial.print(",");
Serial.print(t_f, 2);
Serial.print(",");
Serial.print(svd, 4);
Serial.print(",");
Serial.print(vd, 4);
#endif //ECHO_TO_SERIAL
}

logfile.println();
#if ECHO_TO_SERIAL
Serial.println();
#endif // ECHO_TO_SERIAL

// Now we write data to disk! Don't sync too often - requires 2048 bytes of I/O to SD card
// which uses a bunch of power and takes time
if ((millis() - syncTime) < SYNC_INTERVAL) return;
syncTime = millis();

logfile.flush();

}

Please follow the instructions in "How to use this forum" and post your code properly, with code tags.

The normal strategy would be to make the I/O interrupt driven.

Or perhaps the data gathering could be interrupt driven and the I/O normal?

You appear to be logging data every 10ms, yet flushing the SDcard every 1ms. That's sounds
very wrong.

I am using an Arduino Uno and Adafruit datalogging shield. I need to be able to log all the data to an SD card every millisecond. How do i incorporate buffers into my code so I can write data to the card from one buffer while the data is constantly being collected to the other buffer?

When asked about buffering (double-buffering), I usually point to the Adafruit GPS library as a study example.

But I'm disturbed by you presumption that the SD card write is your bottleneck. You should run your code full-speed and use micros() to time the start and end. This will profile your base code.

If your base code is over 1mS (or even very close) then you are going to need to put effort into refining/optimizing the main routine.

Generally speaking, any use of delay() in the main loop is not good practice. Use micros() and a variable to set-up the control structure for the timing.

I do not see any direct usage for the input data within the program... that is, the sketch seems to just send data to the SD card. You "could" save processing time by writing sensor raw data to the SD and later post-processing the data for temperature conversion, svd, etc.

With optimizations, if the full-loop with SD write is still > 1mS you may wish to consider off-loading SD to a dedicated uC. An example. You can move the serial speed up to 19.2K BAUD or higher until serial latency is a non-concern.

Ray

One thing I've done for my project is use a serial datalogger called serialghost to take care of it. Data gets sent there using the UART serial port and the logger deals with it. This makes data collection quick and easy.

Why is it that every sketch I see involving temperature has code for getting and displaying both centigrade and farenhheit? Why would you need both?

Some things are just easier to relate to in Fahrenheit.
For example, 68 to 72F is coolish to comfortable.
In C, that's like 20 to 22, something ridiculous like that? I can't relate to that.
And 0C doesn't even seem cold in the winter, while 0F is pretty darn cold!

But for engineering, degrees C can be convenient.

Fahrenheit is easier only for those who still use it. For other world is meaningless like feet, inch and gallons.

No, what I'm asking is why do they always want to log both. Either one or the other I understand, but every sketch I see with temp in it someone is printing or logging both. Why do you need it in 2 units. If F is easier for you then log F. If C is easier log C. But what's the point in having the same quantity in two units at the same time?

"For other world is meaningless like feet, inch and gallons."

Maybe, but it's feet, inch and gallons that got us to the moon and back, that works for me 8)

(This is Bar Sport, right? I can poke fun here?)

Printing F & C is useful because you can see example of using the two systems. Logging F & C is.... a mistake, I believe.

@Crossroads
Obviously yesss but.... what about Mars Climate Orbiter? :stuck_out_tongue_closed_eyes:
Anyway water pipes, tires and other things are still in inch in all the world.... :cry:

See, they mixed units there and look at what happened.