SD.open created a file but failed to open.

Hello,

I'm trying to write a program (UNO) that collects data from two different sensors and stores the data in an SD card (adafruit datalogging shield). I was able to get the two sensors to work separately and successfully ran for more than two days. However, when I modified the program to work with both sensors, the IDE issued a warning and the SD.open() kept failing. Though it did create a file.

Here is the warning when complied: "Sketch uses 27132 bytes (84%) of program storage space. Maximum is 32256 bytes.
Global variables use 1706 bytes (83%) of dynamic memory, leaving 342 bytes for local variables. Maximum is 2048 bytes.
Low memory available, stability problems may occur."

Any input on optimizing memory is also appreciated! :smiley:

Thanks!!

#include<SoftwareSerial.h>
#include<SD.h>
#include<SPI.h>
#include "RTClib.h"
#include <Wire.h>
#include <Adafruit_Sensor.h>
#include "Adafruit_BME680.h"

#define BME_SCK 7
#define BME_MISO 6
#define BME_MOSI 5
#define BME_CS 4

#define SEALEVELPRESSURE_HPA (1013.25)
RTC_Millis rtc;

SoftwareSerial dustSensorSerial(2, 3); //RX,TX
Sd2Card card;
SdVolume volume;
SdFile root;
const int chipSelect = 10;
char dataFileName[] = "one.txt";
bool dustSensorStatus = false;

byte dustSensorGetReading[] = {0x42, 0x4D, 0xE2, 0x00, 0x00, 0x01, 0x71};

Adafruit_BME680 bme(BME_CS, BME_MOSI, BME_MISO,  BME_SCK); // hardware SPI

File myFile;

void setup() {
  Serial.begin(9600);
  //Initialize dust sensor
  dustSensorSerial.begin(9600);
  byte dustSensorSwitchMode[] = {0x42, 0x4D, 0xE1, 0x00, 0x00, 0x01, 0x70};
  dustSensorSerial.write(dustSensorSwitchMode, 7);
  while (!dustSensorModeStatus(&dustSensorSerial)) {
    //Serial.println(F("in dust loop"));
    //delay(1000);
  }
  //initialize bme680 sensor
  while (!bme.begin()) {
    //Serial.println(F("Could not find a valid BME680 sensor, check wiring!"));
    //delay(1000);
  }
  // Set up oversampling and filter initialization
  bme.setTemperatureOversampling(BME680_OS_8X);
  bme.setHumidityOversampling(BME680_OS_2X);
  bme.setPressureOversampling(BME680_OS_4X);
  bme.setIIRFilterSize(BME680_FILTER_SIZE_3);
  bme.setGasHeater(320, 150); // 320*C for 150 ms

  //Initialize da timer
  rtc.begin(DateTime(F(__DATE__), F(__TIME__)));

  //initialize SD card
  while (!initializeSD(chipSelect)) {
  }
  myFile = SD.open(dataFileName, FILE_WRITE);
  if (myFile) {
    DateTime now = rtc.now();
    myFile.print(now.month(), DEC); myFile.print("-");
    myFile.print(now.day(), DEC); myFile.print("-");
    myFile.println(now.year(), DEC);
    myFile.println("Time,dustDataPM10,dustDataPM25,dustDataPM100,Temperature,Pressure,Humidity,Gas,Altitude");
    Serial.println(F("Initial Saving Success"));
  }
  else
  {
    while (!(myFile = SD.open(dataFileName, FILE_WRITE))) {
      Serial.println(F("FileFailed"));
      delay(1000);
    }
    DateTime now = rtc.now();
    myFile.print(now.month(), DEC); myFile.print("-");
    myFile.print(now.day(), DEC); myFile.print("-");
    myFile.println(now.year(), DEC);
    myFile.println("Time,dustDataPM10,dustDataPM25,dustDataPM100,Temperature,Pressure,Humidity,Gas,Altitude");
    Serial.println(F("Initial Saving Success"));
  }
  myFile.close();
}

struct dustSensorData {
  uint16_t framelen;
  uint16_t pm10_standard, pm25_standard, pm100_standard;
  uint16_t pm10_env, pm25_env, pm100_env;
  uint16_t particles_03um, particles_05um, particles_10um, particles_25um, particles_50um, particles_100um;
  uint16_t unused;
  uint16_t checksum;
};

struct dustSensorData dustData;

void loop() {
  serialFlush(&dustSensorSerial);
  myFile = SD.open(dataFileName, FILE_WRITE);
  if (myFile) {
    //Serial.println("myFile True");
    DateTime now = rtc.now();
    myFile.print(now.unixtime(), DEC); myFile.print(",");
    dustSensorSerial.write(dustSensorGetReading, sizeof(dustSensorGetReading));
    dustSensorStatus = readDustSensordata(&dustSensorSerial);
    if (dustSensorStatus) {
      myFile.print(dustData.pm10_standard); myFile.print(",");
      myFile.print(dustData.pm25_standard); myFile.print(",");
      myFile.print(dustData.pm100_standard); myFile.print(",");
      Serial.println(F("in dust"));
    }
    else {
      while (!dustSensorStatus) {
        dustSensorStatus = readDustSensordata(&dustSensorSerial);
        if (dustSensorStatus) {
          myFile.print(dustData.pm10_standard); myFile.print(",");
          myFile.print(dustData.pm25_standard); myFile.print(",");
          myFile.print(dustData.pm100_standard); myFile.print(",");
          Serial.println(F("in dust"));
        }
      }
    }
    if (bme.performReading()) {
      myFile.print(bme.temperature); myFile.print(",");
      myFile.print(bme.pressure); myFile.print(",");
      myFile.print(bme.humidity); myFile.print(",");
      myFile.print(bme.gas_resistance / 1000); myFile.print(",");
      myFile.println(bme.readAltitude(SEALEVELPRESSURE_HPA));

      /* Serial.print("Temperature = ");
        Serial.print(bme.temperature);
        Serial.println(" *C");

        Serial.print("Pressure = ");
        Serial.print(bme.pressure / 100.0);
        Serial.println(" hPa");

        Serial.print("Humidity = ");
        Serial.print(bme.humidity);
        Serial.println(" %");

        Serial.print("Gas = ");
        Serial.print(bme.gas_resistance / 1000.0);
        Serial.println(" KOhms");

        Serial.print("Approx. Altitude = ");
        Serial.print(bme.readAltitude(SEALEVELPRESSURE_HPA));
        Serial.println(" m");*/
    }
    else {
      Serial.println(F("BME 680 Failed"));
      myFile.print("0,");
      myFile.print("0,");
      myFile.print("0,");
      myFile.print("0,");
      myFile.println("0");
    }
    myFile.close();
  }
  else {
    Serial.println(F("myFile False"));
  }
  delay(60000);
}
////////////////////////////////////////////////////////////////////////////
//                  Dust Sensor Functions                                //
//////////////////////////////////////////////////////////////////////////
void serialFlush(Stream *s) {
  while (s->available() > 0) {
    char t = s->read();
  }
}
boolean dustSensorModeStatus(Stream *s) {
  if (! s->available()) {
    return false;
  }
  // Read a byte at a time until we get to the special '0x42' start-byte
  if (s->peek() != 0x42) {
    s->read();
    return false;
  }

  // Now read all 32 bytes
  if (s->available() < 8) {
    return false;
  }
  uint8_t buffer[8];
  uint16_t sum = 0;
  s->readBytes(buffer, 8);

  // get checksum ready
  for (uint8_t i = 0; i < 6; i++) {
    sum += buffer[i];
  }
  uint16_t checksum = buffer[6] << 8 | buffer[7];
  if (sum != checksum) {
    Serial.println("Checksum failure");
    return false;
  }
  // success!
  return true;
}
boolean readDustSensordata(Stream *s) {
  if (! s->available()) {
    return false;
  }
  // Read a byte at a time until we get to the special '0x42' start-byte
  if (s->peek() != 0x42) {
    s->read();
    return false;
  }

  // Now read all 32 bytes
  if (s->available() < 32) {
    return false;
  }
  uint8_t buffer[32];
  uint16_t sum = 0;
  s->readBytes(buffer, 32);

  // get checksum ready
  for (uint8_t i = 0; i < 30; i++) {
    sum += buffer[i];
  }

  // The data comes in endian'd, this solves it so it works on all platforms
  uint16_t buffer_u16[15];
  for (uint8_t i = 0; i < 15; i++) {
    buffer_u16[i] = buffer[2 + i * 2 + 1];
    buffer_u16[i] += (buffer[2 + i * 2] << 8);
  }

  // put it into a nice struct :)
  memcpy((void *)&dustData, (void *)buffer_u16, 30);

  if (sum != dustData.checksum) {
    Serial.println("Checksum failure");
    return false;
  }
  // success!
  return true;
}

////////////////////////////////////////////////////////////////////////////
//                  SD Card  Functions                                   //
//////////////////////////////////////////////////////////////////////////
bool initializeSD(int chipSelect) {
  if (SD.begin(chipSelect)) {
    Serial.println(F("SD Initialization Success"));
    return true;
  }
  else {
    Serial.println(F("SD Initialization Failed"));
    return false;
  }
}

Are you sure that the file is closed before trying to open it again ?

UKHeliBob:
Are you sure that the file is closed before trying to open it again ?

Hello,

I think so. Basically, I wrote a script to check the SD card content, open file to read, then close at the end. My program works now after I removed the timer program and brought the memory usage down to 80%. Though, it is an important parameter that I must include :(.

The SD library is notorious for its use of memory for its buffer

I'm currently looking into Petit FAT as an alternative. Crossing my fingers that it will solve the issue :slight_smile: .

darkmatter820:
I'm currently looking into Petit FAT as an alternative. Crossing my fingers that it will solve the issue :slight_smile: .

Have you tried the same open/write code in a sketch that is NOT using 80% of memory?

Hi PaulS,

Yes, I have two separate programs for the two sensors and each can write to the SD card. I also have a open/read program that I run once a while to make sure the file is open and close properly.

Yes, I have two separate programs for the two sensors and each can write to the SD card. I also have a open/read program that I run once a while to make sure the file is open and close properly.

So, it's pretty clear, to me, at least, that you are running out of memory. Each time you open a file, you need to have 512 or more bytes of memory available. You don't have that many when there are no files open.

You might want to try the SdFat library to see if it is more stingy with memory.

darkmatter820:
I'm currently looking into Petit FAT as an alternative. Crossing my fingers that it will solve the issue

I get the impression that any data-logging programme is better done on a Mega. It is common to run out of memory and, if you tinker round the edges to fix today's problem, you may still run out because of something you didn't know about today but will add next week. Having a sensor being the straw that breaks the camel's back is a bit of a surprise, but nonetheless indicative of how marginal things can be with Uno.