My datalogger setup keeps overwriting (not appending) the datalogging file

Greetings, my name is Faishal. I'm new to this forum. So, pardon me if I made this in the wrong section or made other noobly mistakes. :slight_smile:

intro
I'm currently working on a project of creating a device to monitor Indoor Air Quality. This project was supposed to be an IoT-based in regards to datalogging the measured data. So I use ESP8266 (Wemos D1 Mini board) as the board of choice. I've got all things sorted out and sucessfully made the device. Sadly, the project had kind of skewed (re-purposed) to not having an "IoT properties". By that, the datalogging should be done into an SD Card.
problem
Rather than using another board for the device, I still have to use Wemos D1 Mini since I have made the custom PCB and the 3D printed enclosure for the device specifically for the board. So I bought a Wemos D1 Mini datalogger shield which contain DS1307 RTC Chip and a micro SD Card reader/writer. I was able to make the RTC work and able to make the board to create a file (datalog.txt) to datalog into. However, the board is not able to write any lines of data to the file BUT the last line.

I tried to troubleshoot by making a file (with the same name as coded into the board;datalog.txt) using my computer and wrote a random word in the first line of the file. By this, the first line of the datalog.txt should've been the random word even after I let the board to write into the file. However, after I let the board datalog, the random word is not there anymore and seems to be overwritten by the last line that the board write (or the whole file is overwritten).

Refering to Arduino's SD.Open syntax explanation, the
</sup></sub> <sub><sup>FILE_WRITE</sup></sub> <sub><sup>
argument suppose to do "reading and writing, starting at EOF". In this case, it seems that it haven't.

So, how can I assess this situation ?

I've been searching to a lot of sites, asking friends, and haven't found the solution. Please, help :frowning:

I'm using [tt][sub][sup]SD.h[/sup][/sub][/tt] library and below are fragments of my code.

void printToSD()
{ 
  File KitPlane = SD.open("datalog.txt", FILE_WRITE);
  //delay(5000);
  now = Rtc.GetDateTime();
  if (KitPlane)
  {
    KitPlane.print("[MM/DD/YYYY hh:mm:ss - CO2 RH T O2 TVOC PM2.5]");
    KitPlane.print(",");
    KitPlane.print(now.Month(), DEC);
    KitPlane.print("/");
    KitPlane.print(now.Day(), DEC);
    KitPlane.print("/");
    KitPlane.print(now.Year(), DEC);
    KitPlane.print(" ");
    KitPlane.print(now.Hour(), DEC);
    KitPlane.print(":");
    KitPlane.print(now.Minute(), DEC);
    KitPlane.print(":");
    KitPlane.print(now.Second(), DEC);
    KitPlane.print(",");
    KitPlane.print(F_CO2);
    KitPlane.print(",");
    KitPlane.print(F_RH);
    KitPlane.print(",");
    KitPlane.print(F_Temp);
    KitPlane.print(",");
    KitPlane.print(F_O2);
    KitPlane.print(",");
    KitPlane.print(F_TVOC);
    KitPlane.print(",");
    KitPlane.print(F_PM);
    KitPlane.println(",");
    

    Serial.print("[MM/DD/YYYY hh:mm:ss - CO2 RH T O2 TVOC PM2.5]");
    Serial.print(",");
    Serial.print(now.Month(), DEC);
    Serial.print("/");
    Serial.print(now.Day(), DEC);
    Serial.print("/");
    Serial.print(now.Year(), DEC);
    Serial.print(" ");
    Serial.print(now.Hour(), DEC);
    Serial.print(":");
    Serial.print(now.Minute(), DEC);
    Serial.print(":");
    Serial.print(now.Second(), DEC);
    Serial.print(",");
    Serial.print("[MM/DD/YYYY HH:MM:SS CO2 RH T O2 TVOC PM2.5],");
    Serial.print(",");
    Serial.print(F_CO2);
    Serial.print(",");
    Serial.print(F_RH);
    Serial.print(",");
    Serial.print(F_Temp);
    Serial.print(",");
    Serial.print(F_O2);
    Serial.print(",");
    Serial.print(F_TVOC);
    Serial.print(",");
    Serial.println(F_PM);
  }
  else {
    Serial.println("DataLogging-ERROR");
  }
  KitPlane.close();
  delay(5000);
}

TL;DR :
My setup can't seem to "append" a line of string into an existing file in an SD Card. It rather overwrite or make a new file with only one line inside (the last line written). How to assess ?

Thank you :slight_smile:

Best regards,
Faishal

It seems I posted on a wrong section. I don't understand how to move the post. Can someone help ?:))

Please post the code in full. It will make it easier for others to assist you.

fnb111:
Please post the code in full. It will make it easier for others to assist you.

Well, I'm not posting full code because there are so many line, but here it is.. hope it helps finding the solution..

/* RTC */
#include <Wire.h> // must be included here so that Arduino library object file references work
#include <RtcDS1307.h>
RtcDS1307<TwoWire> Rtc(Wire);
RtcDateTime now;

/* DATALoggerLib */
#include <SPI.h>
#include <SD.h>
const int chipSelect = D8;
File KitPlane;

/* CO2 */
#include "SparkFun_SCD30_Arduino_Library.h"
SCD30 airSensor;

/* TVOC */
#include "SparkFun_SGP30_Arduino_Library.h"
SGP30 mySensor;

/* PM */
#include <Arduino.h>
#include <SoftwareSerial.h>
SoftwareSerial PMSerial(D4, D3); //NEEDS TO BE CHANGED I GUESS ???
#define LENG 31
unsigned char buf[LENG];
int PM01Value=0; 
int PM2_5Value=0; 
int PM10Value=0; 

/* O2 */
float sensorValue;
float sensorVoltage;
float Value_O2;

/*SensorsVariables*/
//LogVar
int F_CO2=6,F_TVOC=7,F_PM=8;
float F_O2=11, F_RH=9, F_Temp=10;

void initRTC()
{
  Rtc.Begin();
  RtcDateTime compiled = RtcDateTime(__DATE__, __TIME__);
  Serial.println();
  delay(1000);
}

void initSensors()
{
  //SGP30
  mySensor.begin();
  mySensor.initAirQuality();
  //SCD30
  airSensor.begin();
}

void initSD()
{
  if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    
    return;
  }
  Serial.println("card initialized.");
}

void setup () 
{
    Serial.begin(9600);
    initRTC();
    //initSensors();
    initSD();
}

void getSensorData()
{
  co2RHT();
  o2();
  tvoc();
  PM();
}

void o2()
{
   
}

void tvoc()
{
  
}

void PM()
{
  
}

void co2RHT()
{
  if (airSensor.dataAvailable())
  {
    Serial.print("co2(ppm):");
    Serial.print(airSensor.getCO2());
    F_CO2=airSensor.getCO2();
    //Blynk.virtualWrite(B_CO2, F_CO2);
    Serial.print(" temp(C):");
    Serial.print(airSensor.getTemperature(), 1);
    F_Temp=airSensor.getTemperature();
    //Blynk.virtualWrite(B_Temp, F_Temp);
    Serial.print(" humidity(%):");
    Serial.print(airSensor.getHumidity(), 1);
    F_RH=airSensor.getHumidity();
    //Blynk.virtualWrite(B_RH, F_RH);
    Serial.println();
  }
  else
  {  F_CO2=0;
     F_RH=0;
     F_Temp=0;
    Serial.println("No data");
  }
  delay(1000);
}

void printToSD()
{ 
  KitPlane = SD.open("datalog.txt", FILE_WRITE);
  delay(5000);
  now = Rtc.GetDateTime();
  if (KitPlane)
  {
    KitPlane.print("[MM/DD/YYYY hh:mm:ss - CO2 RH T O2 TVOC PM2.5]");
    KitPlane.print(",");
    KitPlane.print(now.Month(), DEC);
    KitPlane.print("/");
    KitPlane.print(now.Day(), DEC);
    KitPlane.print("/");
    KitPlane.print(now.Year(), DEC);
    KitPlane.print(" ");
    KitPlane.print(now.Hour(), DEC);
    KitPlane.print(":");
    KitPlane.print(now.Minute(), DEC);
    KitPlane.print(":");
    KitPlane.print(now.Second(), DEC);
    KitPlane.print(",");
    KitPlane.print(F_CO2);
    KitPlane.print(",");
    KitPlane.print(F_RH);
    KitPlane.print(",");
    KitPlane.print(F_Temp);
    KitPlane.print(",");
    KitPlane.print(F_O2);
    KitPlane.print(",");
    KitPlane.print(F_TVOC);
    KitPlane.print(",");
    KitPlane.print(F_PM);
    KitPlane.println(",");
    

    Serial.print("[MM/DD/YYYY hh:mm:ss - CO2 RH T O2 TVOC PM2.5]");
    Serial.print(",");
    Serial.print(now.Month(), DEC);
    Serial.print("/");
    Serial.print(now.Day(), DEC);
    Serial.print("/");
    Serial.print(now.Year(), DEC);
    Serial.print(" ");
    Serial.print(now.Hour(), DEC);
    Serial.print(":");
    Serial.print(now.Minute(), DEC);
    Serial.print(":");
    Serial.print(now.Second(), DEC);
    Serial.print(",");
    Serial.print("[MM/DD/YYYY HH:MM:SS CO2 RH T O2 TVOC PM2.5],");
    Serial.print(",");
    Serial.print(F_CO2);
    Serial.print(",");
    Serial.print(F_RH);
    Serial.print(",");
    Serial.print(F_Temp);
    Serial.print(",");
    Serial.print(F_O2);
    Serial.print(",");
    Serial.print(F_TVOC);
    Serial.print(",");
    Serial.println(F_PM);
  }
  else {
    Serial.println("DataLogging-ERROR");
  }
  KitPlane.close();
  delay(5000);
}

void loop () 
{
    //getSensorData();
    printToSD();
    delay(2000); 
}

#define countof(a) (sizeof(a) / sizeof(a[0]))

void printDateTime(const RtcDateTime& dt)
{
    char datestring[20];

    snprintf_P(datestring, 
            countof(datestring),
            PSTR("%02u/%02u/%04u %02u:%02u:%02u"),
            dt.Month(),
            dt.Day(),
            dt.Year(),
            dt.Hour(),
            dt.Minute(),
            dt.Second() );
    Serial.print(datestring);
}

Try adding the pinMode() lines for the actual chipSelect pin and the pin 10 required by the library. These pins must be set to outputs.

void initSD()
{
   pinMode(10, OUTPUT);
   pinMode(chipSelect, OUTPUT);

   if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    
    return;
  }

cattledog:
Try adding the pinMode() lines for the actual chipSelect pin and the pin 10 required by the library. These pins must be set to outputs.

Okay. I will try. But one quick question. Why is it that pin 10 needs to be stated as an output ? Please elaborate "required". Thanks.

What type of formatting does your SD card have? I've read that some SD cards made for cameras have a different formatting than fat32.

Why is it that pin 10 needs to be stated as an output ? Please elaborate "required". Thanks.

It relates to the SPI master/slave arrangement and the SPI library.

See
https://www.arduino.cc/en/reference/SPI

Note about Slave Select (SS) pin on AVR based boards
All AVR based boards have an SS pin that is useful when they act as a slave controlled by an external master. Since this library supports only master mode, this pin should be set always as OUTPUT otherwise the SPI interface could be put automatically into slave mode by hardware, rendering the library inoperative

zoomkat:
What type of formatting does your SD card have? I've read that some SD cards made for cameras have a different formatting than fat32.

It's FAT32. I already try changing the formatting to NTFS and it straight up didn't work at all. I changed it back to FAT32

cattledog:
Try adding the pinMode() lines for the actual chipSelect pin and the pin 10 required by the library. These pins must be set to outputs.

void initSD()

{
  pinMode(10, OUTPUT);
  pinMode(chipSelect, OUTPUT);

if (!SD.begin(chipSelect)) {
   Serial.println("Card failed, or not present");
   
   return;
 }

I already try adding pinMode() line but still didn't work...

jfaishal:
TL;DR :
My setup can't seem to "append" a line of string into an existing file in an SD Card. It rather overwrite or make a new file with only one line inside (the last line written). How to assess ?

Thanks for all the answer. I finally sort it out. Rather than working out on how to not "overwrite" the datalogging file, I try to make a new file (with prefix and incremental numbers) every time the board started. I also use "SdFat.h" rather than <SD.h>. I guess it's some kind of a hassle and storage munching, but anyway it works.

If someone having the same problem as mine, please refer to examples -> ESP8266SdFat -> dataLogger (ESP8266 Board needs to be installed). There you could find the "prefix with incremental numbering" code under void setup (specifically under if(sd.exists(filename))). From there you could trace/reverse engineer the code and re-purpose it for yours.

If needed, here's my final code:

/* RTC */
#include <Wire.h> // must be included here so that Arduino library object file references work
#include <RtcDS1307.h>
RtcDS1307<TwoWire> Rtc(Wire);
RtcDateTime now;

/* DATALoggerLib */
#include <SPI.h>
//#include <SD.h>
#include "SdFat.h"
const int chipSelect = 15;
#define baseName "KitP"
SdFat sd;
SdFile file;
char fileName [13] = baseName "00.csv";
const uint8_t baseSize = sizeof(baseName) - 1;

/* CO2 */
#include "SparkFun_SCD30_Arduino_Library.h"
SCD30 airSensor;

/* TVOC */
#include "SparkFun_SGP30_Arduino_Library.h"
SGP30 mySensor;

/* PM */
#include <Arduino.h>
#include <SoftwareSerial.h>
SoftwareSerial PMSerial(D4, D3); //NEEDS TO BE CHANGED I GUESS ???
#define LENG 31
unsigned char buf[LENG];
int PM01Value=0; 
int PM2_5Value=0; 
int PM10Value=0; 

/* O2 */
float sensorValue;
float sensorVoltage;
float Value_O2;

/*SensorsVariables*/
//LogVar
int F_CO2=6,F_TVOC=7,F_PM=8;
float F_O2=11, F_RH=9, F_Temp=10;

void initRTC()
{
  Rtc.Begin();
  RtcDateTime compiled = RtcDateTime(__DATE__, __TIME__);
  Serial.println("RTC Initialized...");
  delay(1000);
}

void initSensors()
{
  //SGP30
  mySensor.begin();
  mySensor.initAirQuality();
  //SCD30
  airSensor.begin();
  Serial.println("Sensors Initialized...");
}

void initSD()
{
  if(!sd.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    return;
  }
  Serial.println("card initialized.");

  while (sd.exists(fileName)) {
    if (fileName[baseSize + 1] != '9') {
      fileName[baseSize + 1]++;
    } else if (fileName[baseSize] != '9') {
      fileName[baseSize + 1] = '0';
      fileName[baseSize]++;
    } else {
      Serial.println("Can't create file name");
    }
  }

  if (!file.open(fileName, O_WRONLY | O_CREAT | O_EXCL)) {
    Serial.println("error('file.open')");
  }
  Serial.print("logging to : ");
  Serial.println(fileName);
  Serial.println("Creating Header...");
  file.write("-,[MM/DD/YYYY hh:mm:ss],CO2,RH,T,O2,TVOC,PM2.5");
  file.println("");
  Serial.println("-,[MM/DD/YYYY hh:mm:ss],CO2,RH,T,O2,TVOC,PM2.5");
  Serial.println("Header Created");

}

void setup () 
{
    Serial.begin(9600);
    //initSensors();
    initRTC();
    initSD();
}

void getSensorData()
{
  co2RHT();
  o2();
  tvoc();
  PM();
}

void o2()
{
   
}

void tvoc()
{
  
}

void PM()
{
  
}

void co2RHT()
{
  if (airSensor.dataAvailable())
  {
    Serial.print("co2(ppm):");
    Serial.print(airSensor.getCO2());
    F_CO2=airSensor.getCO2();
    //Blynk.virtualWrite(B_CO2, F_CO2);
    Serial.print(" temp(C):");
    Serial.print(airSensor.getTemperature(), 1);
    F_Temp=airSensor.getTemperature();
    //Blynk.virtualWrite(B_Temp, F_Temp);
    Serial.print(" humidity(%):");
    Serial.print(airSensor.getHumidity(), 1);
    F_RH=airSensor.getHumidity();
    //Blynk.virtualWrite(B_RH, F_RH);
    Serial.println();
  }
  else
  {  F_CO2=0;
     F_RH=0;
     F_Temp=0;
    Serial.println("No data");
  }
  delay(1000);
}

void printToSD()
{ 
    file.print(",");
    file.print(now.Month(), DEC);
    file.print("/");
    file.print(now.Day(), DEC);
    file.print("/");
    file.print(now.Year(), DEC);
    file.print(" ");
    file.print(now.Hour(), DEC);
    file.print(":");
    file.print(now.Minute(), DEC);
    file.print(":");
    file.print(now.Second(), DEC);
    file.print(",");
    file.print(F_CO2);
    file.print(",");
    file.print(F_RH);
    file.print(",");
    file.print(F_Temp);
    file.print(",");
    file.print(F_O2);
    file.print(",");
    file.print(F_TVOC);
    file.print(",");
    file.print(F_PM);
    file.println(",");
    file.flush();

    Serial.print("[MM/DD/YYYY hh:mm:ss - CO2 RH T O2 TVOC PM2.5]");
    Serial.print(",");
    Serial.print(now.Month(), DEC);
    Serial.print("/");
    Serial.print(now.Day(), DEC);
    Serial.print("/");
    Serial.print(now.Year(), DEC);
    Serial.print(" ");
    Serial.print(now.Hour(), DEC);
    Serial.print(":");
    Serial.print(now.Minute(), DEC);
    Serial.print(":");
    Serial.print(now.Second(), DEC);
    Serial.print(",");
    Serial.print("[MM/DD/YYYY HH:MM:SS CO2 RH T O2 TVOC PM2.5],");
    Serial.print(",");
    Serial.print(F_CO2);
    Serial.print(",");
    Serial.print(F_RH);
    Serial.print(",");
    Serial.print(F_Temp);
    Serial.print(",");
    Serial.print(F_O2);
    Serial.print(",");
    Serial.print(F_TVOC);
    Serial.print(",");
    Serial.println(F_PM);
  
  
  delay(1000);

}

void loop () 
{
    //getSensorData();
    now = Rtc.GetDateTime();
    printToSD();
    delay(1000); 
}

#define countof(a) (sizeof(a) / sizeof(a[0]))

void printDateTime(const RtcDateTime& dt)
{
    char datestring[20];

    snprintf_P(datestring, 
            countof(datestring),
            PSTR("%02u/%02u/%04u %02u:%02u:%02u"),
            dt.Month(),
            dt.Day(),
            dt.Year(),
            dt.Hour(),
            dt.Minute(),
            dt.Second() );
    Serial.print(datestring);
}

Cheers and thanks again.
Faishal