Nodemcu ESP8266 SD Card File Append Issues

Carpoc · May 3, 2024, 12:32pm

Hello Everyone!

Project: Weather station
Components: DHT22, BMP388, SD card module, Nodemcu ESP8266 with integrated .9" OLED display. 16GB SD card formatted to FAT32, Arduino IDE

Everything works except appending the datafile on the SD card. I know i can send the data to the cloud and that is my intention but i wanted a back up copy in case internet goes down., hence, the SD card.

When creating the file in VOID setup(), the file is created and writes the data headers in the file. If the file exists already, the file creation and headers write step is bypassed.

Here's the code:

#include <Arduino.h>
#include <Wire.h>
#include <U8g2lib.h>
#include <Adafruit_Sensor.h>
#include "Adafruit_BMP3XX.h"
#include "DHT.h" // Include the temp/humidity sensor library
// Libraries to get time from NTP Server
#include <ESP8266WiFi.h>
#include <NTPClient.h>
#include <WiFiUdp.h>
//Libraries for SD Card
#include <SD.h>
#include <SPI.h>

// Network Credentials
const char *ssid = "Trumped 2.4";
const char *password = "T1h2e3E4n5d!";

// Define SPI CS pin for the SD card module

#define SD_CS 0

// Save reading number on RTC memory

//Define the reading ID for every line of data
uint32_t readingID;

//define the variable dataMessage to hold all data in one string
String dataMessage;

//Define NTP Client to get time

WiFiUDP ntpUDP;
NTPClient timeClient(ntpUDP,"pool.ntp.org");

//Variables to save date and time

String formattedDate;
String dayStamp;
String timeStamp;

#define DHTPIN 2
    // what pin on NodeMCU we're connected to for the DHT22 (GPIO2) D4
#define DHTTYPE DHT22   // DHT 22  (AM2302)
DHT dht(DHTPIN, DHTTYPE); // Initialize DHT Sensor

//Set up SPI pins if using SPI for BMP3XX sensor
//#define BMP_SCK 14
//#define BMP_MISO 12
//#define BMP_MOSI 13
//#define BMP_CS 15

float Temp; // Temperature C
float Press; // Absolute Pressure mB
float Alt; // Altitude m
float h; //Relative Humidity %

//Define sea level pressure
#define SEALEVELPRESSURE_HPA (1013.25)

Adafruit_BMP3XX bmp;

U8G2_SSD1306_128X64_NONAME_F_SW_I2C
u8g2(U8G2_R0,14,12,U8X8_PIN_NONE);

//Define dataFile variable for writing data to data.txt
File dataFile;

void setup() {

//set up dataFile for write data.txt file to SD card
dataFile = SD.open("data.txt", FILE_WRITE);

//Initiate Time Client
timeClient.begin();
//Set time offset from GMT
timeClient.setTimeOffset(-21600);

// Open serial communications and wait for port to open:
  Serial.begin(9600);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  } 

 // Connect to Wi-Fi network with SSID and password
  Serial.print("Connecting to ");
  Serial.println(ssid);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected.");

  // Initialize SD card
  Serial.print("Initializing SD card...");
  pinMode(SD_CS,OUTPUT);
  if (!SD.begin(SD_CS)) {
    Serial.println("initialization failed!");
    while (1);
  }
  Serial.println("initialization done.");

  // If the data.txt file doesn't exist  
  // Create a file on the SD card and write the data labels
               
  if (SD.exists("data.txt")) {
    Serial.println("Data file already exists"); 
  }
  else {
    Serial.println("File doesn't exist");
    Serial.println("Creating file...");
    dataFile.println("Reading ID, Date, Time, Temp, Humidity, Pressure");
    dataFile.close();
  }
    

  
  //Start DHT Sensor
  dht.begin();
  
  //Start the OLED display
  u8g2.begin();

  //Start serial communication with BMP388
  while (!Serial);
  Serial.println("Adafruit BMP388 / BMP390 test");

  //Begin I2C communication
 if (!bmp.begin_I2C()) {   // hardware I2C mode, can pass in address & alt Wire
  //if (! bmp.begin_SPI(BMP_CS)) {  // hardware SPI mode  
  //if (! bmp.begin_SPI(BMP_CS, BMP_SCK, BMP_MISO, BMP_MOSI)) {  // software SPI mode
    Serial.println("Could not find a valid BMP3 sensor, check wiring!");
    while (1);
  }

  // Set up oversampling and filter initialization
  bmp.setTemperatureOversampling(BMP3_OVERSAMPLING_8X);
  bmp.setPressureOversampling(BMP3_OVERSAMPLING_4X);
  bmp.setIIRFilterCoeff(BMP3_IIR_FILTER_COEFF_3);
  bmp.setOutputDataRate(BMP3_ODR_50_HZ);

}

void loop() {

 timeClient.update();

 //Read Humidity
 h = dht.readHumidity();

 // increment readingID
 readingID = readingID + 1;

 // Check if any reads failed and exit early (to try again).
    if (isnan(h)){
      
    }
   
  if (! bmp.performReading()) {
    u8g2.drawStr(0,10,"Failed to perform reading :(");
    return;
  }
  //Calculate Temperature, pressure and altitude
  Temp = bmp.temperature;
  Press = bmp.pressure/100;
  Alt = bmp.readAltitude(SEALEVELPRESSURE_HPA);

  // Calculate corrected pressure for station down to sea level:  
  // Z = Elevation above sea level (m) 
  float a = 16000 + 64*Temp;
  float Z = 670.8648;
  float correctedP = Press * (a + Z)/(a - Z);

  u8g2.clearBuffer();
  u8g2.setFont(u8g2_font_8x13_tf);
  u8g2.drawStr(0,10,"T(C)= ");
  u8g2.setCursor(50,10);
  u8g2.print(Temp);
  
  u8g2.drawStr(0,25,"H(%)= ");
  u8g2.setCursor(50,25);
  u8g2.print(h);
  
  u8g2.drawStr(0,40,"P(mB)= ");
  u8g2.setCursor(50,40);
  u8g2.print(correctedP);
 
  u8g2.drawStr(0,55,"Alt(m)= ");
  u8g2.setCursor(57,55);
  u8g2.print(Alt);
  u8g2.sendBuffer(); 
  
//Retrieve time and date from NTP Server

  time_t epochTime = timeClient.getEpochTime();
  //Get a time structure
  struct tm *ptm = gmtime ((time_t *)&epochTime);
 
  String formattedTime = timeClient.getFormattedTime(); // Get formatted Time from NTP Server
  int monthDay = ptm->tm_mday; // Get day of the month(integer)
  int currentMonth = ptm->tm_mon+1; //Get current month (integer)
  int currentYear = ptm->tm_year+1900; //Get current year
  // Complete Date
  String currentDate = String(currentMonth) + "/" + String(monthDay) + "/" + String(currentYear);

  // Create the data reading string
  dataMessage = String(readingID) + "," + String(currentDate) + "," + String(formattedTime) + "," + String(Temp) +  "," +String(h) + "," + String(correctedP) +"\r\n";
    
  // Open the file on SD card
  // if the file is available, write to it:
  dataFile = SD.open("data.txt", FILE_WRITE);
  if (SD.exists("data.txt")) {
    Serial.println("Writing Data...");
    dataFile.println(dataMessage);
    dataFile.close();
    Serial.println("Write Complete...");
    Serial.println(dataMessage);
  }
  // if the file isn't open, pop up an error:
  else {
    Serial.println("error opening data.txt");
  }
  delay(5000);

}

Here's the Serial output:

06:03:06.659 -> .............................................................................................................................................................................

06:03:10.716 -> WiFi connected.

06:03:10.750 -> Initializing SD card...initialization done.

06:03:10.783 -> Data file already exists

06:03:11.246 -> Adafruit BMP388 / BMP390 test

06:03:11.560 -> Writing Data...

06:03:12.153 -> Write Complete...

06:03:12.153 -> 1,5/3/2024,06:03:09,24.85,nan,892.09

06:03:12.189 ->

06:03:17.762 -> Writing Data...

06:03:18.397 -> Write Complete...

06:03:18.397 -> 2,5/3/2024,06:03:15,23.10,nan,1011.24

06:03:18.441 ->

06:03:23.584 -> Writing Data...

06:03:24.246 -> Write Complete...

06:03:24.246 -> 3,5/3/2024,06:03:21,23.10,37.60,1011.24

06:03:24.292 ->

06:03:29.447 -> Writing Data...

06:03:30.077 -> Write Complete...

06:03:30.077 -> 4,5/3/2024,06:03:27,23.10,38.50,1011.24

06:03:30.077 ->

06:03:35.270 -> Writing Data...

06:03:35.867 -> Write Complete...

06:03:35.867 -> 5,5/3/2024,06:03:33,23.09,38.50,1011.24

06:03:35.900 ->

06:03:41.055 -> Writing Data...

06:03:41.725 -> Write Complete...

06:03:41.725 -> 6,5/3/2024,06:03:38,23.09,37.30,1011.24

06:03:41.725 ->

06:03:46.897 -> Writing Data...

06:03:47.549 -> Write Complete...

06:03:47.549 -> 7,5/3/2024,06:03:44,23.08,36.50,1011.24

06:03:47.549 ->

Here's the data file that is created:

You can see that no data lines have been written other than the headers. Funny thing as well, if I remove the SD card while the program is running, the serial output still says the data is writing successfully. Hope someone can help!

ShermanP · May 4, 2024, 2:49pm

What is the exact length of that Data file? And what does it look like in a hex editor?

Carpoc · May 6, 2024, 12:43pm

ShermanP · May 6, 2024, 5:29pm

Ok, either it isn't writing to the card at all, or it isn't closing correctly to update the file length in the directory entry. I don't know why. But...

The first line of setup() is opening the file, which is before SD.begin is done. So I don't know how that could work. It seems this line should be just before line 113.

Also, I wonder what happens if you delete the file from the card, then run this code again. Does it create the file and write the headers? If so , then why would it not work for the data logging? Maybe it has something to do with using String functions in the data logging.

Well maybe an ESP8266 expert will have better ideas. I assume you've run the ReadWrite example in the SD library successfully, so the wiring and basic functions are working.

Carpoc · May 6, 2024, 6:33pm

Yeah i saw that... thanks for pointing that out. I made some changes to the code:

#include <Arduino.h>
#include <Wire.h>
#include <U8g2lib.h>
#include <Adafruit_Sensor.h>
#include "Adafruit_BMP3XX.h"
#include "DHT.h" // Include the temp/humidity sensor library
// Libraries to get time from NTP Server

#include <ESP8266WiFi.h>
#include <NTPClient.h>
#include <WiFiUdp.h>
//Libraries for SD Card
#include <SD.h>
#include <SPI.h>

// Network Credentials
const char *ssid = "Trumped 2.4";
const char *password = "T1h2e3E4n5d!";

// Define SPI CS pin for the SD card module

#define SD_CS 0

// Save reading number on RTC memory

//Define the reading ID for every line of data
uint32_t readingID;

//define the variable dataMessage to hold all data in one string
String dataMessage;

//Define NTP Client to get time

WiFiUDP ntpUDP;
NTPClient timeClient(ntpUDP,"pool.ntp.org");

//Variables to save date and time

String formattedDate;
String dayStamp;
String timeStamp;

#define DHTPIN 2
    // what pin on NodeMCU we're connected to for the DHT22 (GPIO2) D4
#define DHTTYPE DHT22   // DHT 22  (AM2302)
DHT dht(DHTPIN, DHTTYPE); // Initialize DHT Sensor

//Set up SPI pins if using SPI for BMP3XX sensor
//#define BMP_SCK 14
//#define BMP_MISO 12
//#define BMP_MOSI 13
//#define BMP_CS 15

float Temp; // Temperature C
float Press; // Absolute Pressure mB
float Alt; // Altitude m
float h; //Relative Humidity %

//Define sea level pressure
#define SEALEVELPRESSURE_HPA (1013.25)

Adafruit_BMP3XX bmp;

U8G2_SSD1306_128X64_NONAME_F_SW_I2C
u8g2(U8G2_R0,14,12,U8X8_PIN_NONE);

void setup() {

//Initiate Time Client
timeClient.begin();
//Set time offset from GMT
timeClient.setTimeOffset(-21600);

// Open serial communications and wait for port to open:
  Serial.begin(9600);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  } 

 // Connect to Wi-Fi network with SSID and password
  Serial.print("Connecting to ");
  Serial.println(ssid);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected.");

  // Initialize SD card
  Serial.print("Initializing SD card...");
  pinMode(SD_CS,OUTPUT);
  if (!SD.begin(SD_CS)) {
    Serial.println("initialization failed!");
    while (1);
  }
  Serial.println("initialization done.");

  /*Attempt to open data.txt. If the data.txt file doesn't exist, 
    Create a file on the SD card and write the data column labels*/
  
    if (!SD.exists("data.txt")) {
    File dataFile = SD.open("data.txt",FILE_WRITE);
    Serial.println("File doesn't exist");
    Serial.println("Creating file...");
    dataFile.println("Reading ID, Date, Time, Temp, Humidity, Pressure");
    dataFile.close();

  } 
  else {
    
    Serial.println("Data file already exists");

  }
   
  //Start DHT Sensor
  dht.begin();
  
  //Start the OLED display
  u8g2.begin();

  //Start serial communication with BMP388
  while (!Serial);
  Serial.println("Adafruit BMP388 / BMP390 test");

  //Begin I2C communication
 if (!bmp.begin_I2C()) {   // hardware I2C mode, can pass in address & alt Wire
  //if (! bmp.begin_SPI(BMP_CS)) {  // hardware SPI mode  
  //if (! bmp.begin_SPI(BMP_CS, BMP_SCK, BMP_MISO, BMP_MOSI)) {  // software SPI mode
    Serial.println("Could not find a valid BMP3 sensor, check wiring!");
    while (1);
  }

  // Set up oversampling and filter initialization
  bmp.setTemperatureOversampling(BMP3_OVERSAMPLING_8X);
  bmp.setPressureOversampling(BMP3_OVERSAMPLING_4X);
  bmp.setIIRFilterCoeff(BMP3_IIR_FILTER_COEFF_3);
  bmp.setOutputDataRate(BMP3_ODR_50_HZ);

}

void loop() {

 timeClient.update();

 //Read Humidity
 h = dht.readHumidity();

 // increment readingID
 readingID = readingID + 1;

 // Check if any reads failed and exit early (to try again).
    if (isnan(h)){
      
    }
   
  if (! bmp.performReading()) {
    u8g2.drawStr(0,10,"Failed to perform reading :(");
    return;
  }
  //Calculate Temperature, pressure and altitude
  Temp = bmp.temperature;
  Press = bmp.pressure/100;
  Alt = bmp.readAltitude(SEALEVELPRESSURE_HPA);

  // Calculate corrected pressure for station down to sea level:  
  // Z = Elevation above sea level (m) 
  float a = 16000 + 64*Temp;
  float Z = 670.8648;
  float correctedP = Press * (a + Z)/(a - Z);


  //Print data to OLED
  u8g2.clearBuffer();
  u8g2.setFont(u8g2_font_8x13_tf);
  u8g2.drawStr(0,10,"T(C)= ");
  u8g2.setCursor(50,10);
  u8g2.print(Temp);
  
  u8g2.drawStr(0,25,"H(%)= ");
  u8g2.setCursor(50,25);
  u8g2.print(h);
  
  u8g2.drawStr(0,40,"P(mB)= ");
  u8g2.setCursor(50,40);
  u8g2.print(correctedP);
 
  u8g2.drawStr(0,55,"Alt(m)= ");
  u8g2.setCursor(57,55);
  u8g2.print(Alt);
  u8g2.sendBuffer(); 
  
  //Retrieve time and date from NTP Server

  time_t epochTime = timeClient.getEpochTime();
  //Get a time structure
  struct tm *ptm = gmtime ((time_t *)&epochTime);
 
  String formattedTime = timeClient.getFormattedTime(); // Get formatted Time from NTP Server
  int monthDay = ptm->tm_mday; // Get day of the month(integer)
  int currentMonth = ptm->tm_mon+1; //Get current month (integer)
  int currentYear = ptm->tm_year+1900; //Get current year
  // Complete Date
  String currentDate = String(currentMonth) + "/" + String(monthDay) + "/" + String(currentYear);

  // Create the data reading string
  dataMessage = String(readingID) + "," + String(currentDate) + "," + String(formattedTime) + "," + String(Temp) +  "," + String(h) + "," + String(correctedP) +"\r\n";
    
  // Open the file on SD card
  // if the file is available, write to it:
  File dataFile = SD.open("data.txt",FILE_WRITE);
  
    Serial.println("Writing Data...");
    //dataFile.seek(EOF);
    dataFile.println(String(dataMessage));
    dataFile.close();
    Serial.println("Write Complete...");
    Serial.println(dataMessage);
  
  delay(5000);
}

I moved the File open statement to after the "if" statement in setup. And yes the file is created with headers when I deleted the file from the card.

It also bypasses the file create when it sees the file exists:

It's weird that it writes in setup but not in the loop. I also removed the "if" condition for the loop write to the SD. It still doesn't write in the loop.

I thought about the string issue... but it prints fine in the serial monitor... I'm going to try writing plain text to the file in the loop and see if there is any difference.

Thanks!

oldcurmudgeon · May 6, 2024, 8:28pm

I think this defines a new empty string. Try

dataFile.println(dataMessage);

If you check the return code from the

dataFile.println()

you can see how many bytes were actually written

Also you never check to see that the file open actually worked.

Carpoc · May 7, 2024, 1:10pm

Hi Thanks for the reply!
Yes tried dataFile.println(dataMessage);... still nothing. I even replaced dataMessage with straight text and does not write - dataFile.println("Goodbye Cruel World") lol. I'm guessing it is not writing in the loop because there may be some conflict with the OLED display or the bmp388 because pins are limited. I'll send a pin out diagram soon

This is the set up:

oldcurmudgeon · May 7, 2024, 2:59pm

You are defining the display with an I2C interface but then specifying SPI pins for clock and data - this will not end well.

It probably works in setup() because the u8g2.begin() is after the SD processing.

You should check that the SD open() actually works.

Carpoc · May 7, 2024, 5:02pm

Thanks for your help!
Unfortunately I believe the display on the board is hard wired to pins 12, 14 which is the MISO and CLK pins for SPI. So the limitations of the board have hooped me lol. Looks like google sheets maybe the answer for this board or a similar OLED ESP32 board

oldcurmudgeon · May 7, 2024, 7:04pm

I found some limited documentation and you are correct - they hardwired the I2C interface to the SPI pins. This is a stupid implementation. The default is to put the I2C pins on D1/GPI05 and D2/GPI04 - I2C is implemented in software so can be assigned to other pins.

oldcurmudgeon · May 7, 2024, 7:21pm

If you are NOT totally done with Ideaspark, they have an esp32 variant where they use the default I2C pins. No idea if will work for you application.

EDIT: added NOT

Carpoc · May 7, 2024, 7:29pm

Idea Spark is Idea Crap! lol Thank you for the confirmation. Problem solved - I used a nodemcu without the OLED and went old school with an LCD using I2C... no more conflicts!

I will look into the esp32 variant!
Thanks again for your help!

F5OPV · June 9, 2024, 7:14am

Replace
dataFile = SD.open("data.txt", FILE_WRITE);
by

dataFile = SD.open("data.txt", FILE_APPEND);

system · December 6, 2024, 7:14am

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.

Topic		Replies	Views
SD card - datalogger with Nodemcu v3 and Max 31865 - not appending..... Programming	4	1232	May 6, 2021
SD card write very random, mostly not working Storage	16	9427	May 6, 2021
esp8266 + sd card+ send Programming	22	3255	May 5, 2021
SD Card not writing file despite successful initialization Programming	5	4058	May 5, 2021
ESP32 server.on("/get"... Save to SD card problem Storage	8	280	October 6, 2024

Nodemcu ESP8266 SD Card File Append Issues

Related topics