SD logging reliability

Hey guys,

I've been doing sensor data logging for some time now and recently faced some unexpected troubles. I am using adafruits rtc data logging shield with uno.

Case 1: Code for case 1 below. Had no problems what so ever.

//LED.
int db1 = 3;
int db2 = 5;

//RTC.
#include "Wire.h"
#include <RTClib.h>
RTC_DS1307 RTC;

//RHT.
int SHT_clockPin = 7;
int SHT_dataPin  = 4;

//SD.
const int chipSelect = 10; // Adafruit 10 - Arduino 4
#include <SPI.h>
#include <SD.h>

//S gen.
unsigned long starttime;
unsigned long sampletime_ms = 30000;

int S60 = 9;
float s60_avg = 0;
unsigned long S60_duration_1;
unsigned long S60_occupancy_1 = 0;
float S60_ratio_1 = 0;

float val[100];
int sample = 0;

void setup() { 
  Wire.begin();
  RTC.begin();
  Serial.begin(9600);
  pinMode(10, OUTPUT);
  pinMode(S60,INPUT);
  pinMode(db1, OUTPUT);
  pinMode(db2, OUTPUT);
  
  digitalWrite(db1, LOW);
  digitalWrite(db2, LOW);
  digitalWrite(db1, HIGH);
  delay(1000);
  digitalWrite(db1, LOW);
  
  //if (! RTC.isrunning()) {
   //RTC.adjust(DateTime(__DATE__, __TIME__));}
  
  if (!SD.begin(chipSelect)) {
    return;}
  File dataFile = SD.open("data.txt", FILE_WRITE);
    if (dataFile){
      dataFile.close();}
  delay(500);
  digitalWrite(db1, LOW);

  }

void loop() {

s60_avg = 0;
sample = 0;
float time = millis();

while ((millis()-time) < 290000){
  float arvo = getS60();
  digitalWrite(db1, HIGH);
  delay(500);
  digitalWrite(db1, LOW);
  val[sample] = arvo;
  sample++;}
for(int i=0; i<(sample+1); i++){
  s60_avg = s60_avg + val[i];}
s60_avg = s60_avg / sample;  

digitalWrite(db2, HIGH);
saveSD();
delay(500);
digitalWrite(db2, LOW);
}
  
float getS60(){
   starttime = millis();
   while ((millis()-starttime) < sampletime_ms){
      S60_duration_1 = pulseIn(S60, LOW);
      S60_occupancy_1 = S60_occupancy_1 + S60_duration_1;}
    
   S60_ratio_1 = S60_occupancy_1 / (sampletime_ms*10.0);
   S60_occupancy_1 = 0;
   return S60_ratio_1;}
    
void saveSD(){
  float temperature = getTemperature();
  float humidity = getHumidity();
  
  DateTime now = RTC.now();
  
  File dataFile = SD.open("data.txt", FILE_WRITE);
  if (dataFile){
    dataFile.print(s60_avg);
    dataFile.print("\t");
    dataFile.print(humidity);
    dataFile.print("\t");
    dataFile.print(temperature);
    dataFile.print("\t");
    dataFile.print(now.month(), DEC);
    dataFile.print('/');
    dataFile.print(now.day(), DEC);
    dataFile.print('/');
    dataFile.print(now.year(), DEC);
    dataFile.print(' ');
    dataFile.print(now.hour(), DEC);
    dataFile.print(':');
    dataFile.print(now.minute(), DEC);
    dataFile.print(':');
    dataFile.println(now.second(), DEC); 
    dataFile.close();}
    
  Serial.print(s60_avg);
  Serial.print("\t");
  Serial.print(humidity);
  Serial.print("\t");
  Serial.println(temperature);
  }

  
float getTemperature(){
  SHT_sendCommand(B00000011, SHT_dataPin, SHT_clockPin);
  SHT_waitForResult(SHT_dataPin);

  int val = SHT_getData(SHT_dataPin, SHT_clockPin);
  SHT_skipCrc(SHT_dataPin, SHT_clockPin);
  return (float)val * 0.01 - 40; //convert to celsius
}

float getHumidity(){
  SHT_sendCommand(B00000101, SHT_dataPin, SHT_clockPin);
  SHT_waitForResult(SHT_dataPin);
  int val = SHT_getData(SHT_dataPin, SHT_clockPin);
  SHT_skipCrc(SHT_dataPin, SHT_clockPin);
  return -4.0 + 0.0405 * val + -0.0000028 * val * val; 
}

.
.
.
}

Case 2: Code for case 2 below. Arduino had stalled data logging unexpectedly. It had logged for little over 2 days, then stopped. Everything else seemed ok otherwise. Also data was randomly missing dataFile.print("\t"); command. So the data looked like this:

4/13/2016 1:15:54	7.33	0.32	8.49	40.38	198.17
4/13/2016 1:31:126.89	0.14	8.35	40.41	203.83                            <--- error
4/13/2016 1:36:19	6.67	0.27	8.26	40.73	203.50

I was thinking could this have something to do with the time that writing on SD card takes? Or maybe outdoor humidity/temperature change?Basically the difference in codes 1 and 2 is the order of logging sensor data and timestamp. Funny thing is that everything seems to work when I test them at my lab but when the real measurements start everything goes fubar. Also, never have I ever before had arduino fail to start data logging at all. Also I tried with different cards and boards I don't think thats the problem.

//RTC.
#include "Wire.h"
#include <RTClib.h>
#include "Adafruit_HTU21DF.h"
Adafruit_HTU21DF htu = Adafruit_HTU21DF();
RTC_DS1307 RTC;

//SD.
const int chipSelect = 10; // Adafruit 10 - Arduino 4
#include <SPI.h>
#include <SD.h>

//RHT.
int SHT_clockPin = 7;
int SHT_dataPin  = 4;

//Sgen.
unsigned long starttime;
unsigned long sampletime_ms = 30000;

//S60.
int S60 = 8;
float s60_avg = 0;
unsigned long S60_duration_1;
unsigned long S60_occupancy_1 = 0;
float S60_ratio_1 = 0;

//S42.
int S42 = 9;
float s42_avg = 0;
unsigned long S42_duration_1;
unsigned long S42_occupancy_1 = 0;
float S42_ratio_1 = 0;

//Dp.
int dp_count = 30;
float dp_val[30];
float dp = 0;
float dpa = 0;
float paine = 0;

//Gen.
float val_S60[5];
float val_S42[5];
float temp = 0;
float hum = 0;

void setup() { 
  Wire.begin();
  RTC.begin();
  Serial.begin(9600);
  
  pinMode(10,OUTPUT);
  pinMode(S60,INPUT);
  pinMode(S42,INPUT);
  
  if (!htu.begin()) {
    Serial.println("Couldn't find HTU21D-F");
    while (1);
  }  
  
  if (!SD.begin(chipSelect)) {
    return;}
    
  File dataFile = SD.open("data.txt", FILE_WRITE);
    if (dataFile){
      dataFile.close();}
  }

void loop() {
  s60_avg = 0;
  s42_avg = 0;  
  
  for(int i=0; i<5; i++){
    val_S60[i] = getS60();
    val_S42[i] = getS42();}  
    
  for(int i=0; i<5; i++){
    s60_avg = s60_avg + val_S60[i];
    s42_avg = s42_avg + val_S42[i];}
  s60_avg = s60_avg / 5;
  s42_avg = s42_avg / 5;
  
temp = getTemperature();
hum = getHumidity();
paine = differentialPressure();


saveSD();
}



/////////////////////////////////
////////////FUNCTIONS////////////
/////////////////////////////////

float differentialPressure() {
  dp = 0;
  for(int i=0; i<dp_count; i++){
    dp_val[i] = analogRead(A3);
    delay(1);} 
 
  for(int i=0; i<dp_count; i++){
    dp = dp + dp_val[i];}
    
  dp = dp / dp_count;
  
return dp;}

float getS60(){
   starttime = millis();
   while ((millis()-starttime) < sampletime_ms){
      S60_duration_1 = pulseIn(S60, LOW);
      S60_occupancy_1 = S60_occupancy_1 + S60_duration_1;}
    
   S60_ratio_1 = S60_occupancy_1 / (sampletime_ms*10.0);
   S60_occupancy_1 = 0;
   return S60_ratio_1;}
   
float getS42(){
   starttime = millis();
   while ((millis()-starttime) < sampletime_ms){
      S42_duration_1 = pulseIn(S42, LOW);
      S42_occupancy_1 = S42_occupancy_1 + S42_duration_1;}
    
   S42_ratio_1 = S42_occupancy_1 / (sampletime_ms*10.0);
   S42_occupancy_1 = 0;
   return S42_ratio_1;}
    
void saveSD(){
  DateTime now = RTC.now();
  File dataFile = SD.open("data.txt", FILE_WRITE);
    if (dataFile){
      dataFile.print(now.month(), DEC);
      dataFile.print('/');
      dataFile.print(now.day(), DEC);
      dataFile.print('/');
      dataFile.print(now.year(), DEC);
      dataFile.print(' ');
      dataFile.print(now.hour(), DEC);
      dataFile.print(':');
      dataFile.print(now.minute(), DEC);
      dataFile.print(':');
      dataFile.print(now.second(), DEC);
      dataFile.print("\t"); 
      dataFile.print(s60_avg);
      dataFile.print("\t");
      dataFile.print(s42_avg);
      dataFile.print("\t"); 
      dataFile.print(temp);
      dataFile.print("\t");
      dataFile.print(hum);
      dataFile.print("\t");
      dataFile.println(paine);     
      dataFile.close();}
  }

  
float getTemperature(){
  SHT_sendCommand(B00000011, SHT_dataPin, SHT_clockPin);
  SHT_waitForResult(SHT_dataPin);

  int val = SHT_getData(SHT_dataPin, SHT_clockPin);
  SHT_skipCrc(SHT_dataPin, SHT_clockPin);
  return (float)val * 0.01 - 40; //convert to celsius
}

float getHumidity(){
  SHT_sendCommand(B00000101, SHT_dataPin, SHT_clockPin);
  SHT_waitForResult(SHT_dataPin);
  int val = SHT_getData(SHT_dataPin, SHT_clockPin);
  SHT_skipCrc(SHT_dataPin, SHT_clockPin);
  return -4.0 + 0.0405 * val + -0.0000028 * val * val; 
}

.
.
.
}

E: Had to take part of the code out for too long message.

Hey guys,

Any clues?

Does anyone know what might cause arduino to skip dataFile.print("\t); command? There's only normal space bar char instead of tab. I've been logging data for sometime now and I still get results like this.

4/29/2016 21:59:34 	16.50	0.22	493.37
4/29/2016 22:5:32	16.29	0.20	493.90
4/29/2016 22:11:28 14.85	0.40	494.43
4/29/2016 22:17:23	14.68	0.31	494.47
4/29/2016 22:23:19	15.40	0.18	494.33
4/29/2016 22:29:16	15.03	0.23	494.37
4/29/2016 22:35:11	13.82	0.28	494.37
4/29/2016 22:41:7 12.44	0.25	493.70
4/29/2016 22:47:5	12.56	0.23	493.93
4/29/2016 22:53:4	12.78	0.29	494.03
4/29/2016 22:59:1	12.20	0.24	494.13
4/29/2016 23:4:59	11.36	0.08	494.20
4/29/2016 23:10:56	10.05	0.17	494.03
4/29/2016 23:16:52 10.96	0.09	494.13
4/29/2016 23:22:48	11.03	0.25	494.23
4/29/2016 23:28:42	10.72	0.18	494.13
4/29/2016 23:34:39	10.16	0.31	494.53
4/29/2016 23:40:35 10.58	0.29	493.53
4/29/2016 23:46:33 10.37	0.22	493.80
4/29/2016 23:52:29	10.15	0.15	493.43
4/29/2016 23:58:25	10.25	0.32	493.67
4/30/2016 0:4:21	9.50	0.14	493.87
4/30/2016 0:10:17	9.49	0.25	494.57
4/30/2016 0:16:12	8.93	0.06	493.57
4/30/2016 0:22:9	9.43	0.14	494.03
4/30/2016 0:28:6 9.71	0.15	493.20
4/30/2016 0:34:5	9.39	0.12	493.80
4/30/2016 0:40:2 8.60	0.10	493.80
4/30/2016 0:45:59	8.72	0.09	493.43
4/30/2016 0:51:54	8.72	0.17	493.90
4/30/2016 0:57:51	8.98	0.09	494.50
void saveSD(){
  DateTime now = RTC.now();
  File dataFile = SD.open("data.txt", FILE_WRITE);
    if (dataFile){
      dataFile.print(now.month(), DEC);
      dataFile.print('/');
      dataFile.print(now.day(), DEC);
      dataFile.print('/');
      dataFile.print(now.year(), DEC);
      dataFile.print(' ');
      dataFile.print(now.hour(), DEC);
      dataFile.print(':');
      dataFile.print(now.minute(), DEC);
      dataFile.print(':');
      dataFile.print(now.second(), DEC);
      dataFile.print("\t"); 
      dataFile.print(s60_avg);
      dataFile.print("\t");
      dataFile.print(s42_avg);
      dataFile.print("\t");
      dataFile.println(paine);     
      dataFile.close();}
  }

Arduino stalling might have something to do with rain since it's working ok now with clear skies.

You might expect characters to go missing on a serial link but if this is talking to an SD card in the same box, there is no programming explanation for the missing /t.

I solved the problem.

If the time stamp was very short (i.e. 1/1/2016 1:1:1) then the \t command moved the column to start earlier. If the time stamp was longer (like in most cases) the columns started evenly from the same point.
The problem was fixed by adding additional dataFile.print(" "); before the \t command so that even if the time stamp stays short, it will still be long enough so that all the columns are even.