Arrays and RTC_DATA_ATTR with DeepSleep (ESP32C6)

I'm making a low-power CO2 monitor, with an eink display, using Deep Sleep to keep the battery consumption minimal with solar charging (using a DFRobot Firebeetle 2 ESP32-C3).

It's working fine for wakeup each 60 seconds, and with retaining the value of bootCount declared with RTC_DATA_ATTR.

I want to use two arrays, also with RTC_DATA_ATTR, to populate with previous readings from which ten-minute and one-hour averages can be calculated. But these are not being retained, on wakeup they are being filled with zeroes. (Or, if I remove the value from the declaration, they contain random data, as would be expected from a newly-declared variable.)

Is the issue that arrays cannot be placed in RTC memory in this way - and if so, is there another way to retain this data through deep sleep? Or is it another problem entirely?

#include <GxEPD2_BW.h>
#include <DFRobot_SCD4X.h>
#include <Fonts/FreeSansBold24pt7b.h>
#include <Fonts/FreeSansBold12pt7b.h>

#define BATT_V_PIN    0

#define uS_TO_S_FACTOR 1000000ULL  /* Conversion factor for micro seconds to seconds */
#define TIME_TO_SLEEP  60        /* Time ESP32 will go to sleep (in seconds) */
#define TEN_MINUTE_READINGS 10      // Adjust these to match TIME_TO_SLEEP
#define ONE_HOUR_READINGS 60    //      "        "      "       "

GxEPD2_BW<GxEPD2_420_GDEY042T81, GxEPD2_420_GDEY042T81::HEIGHT> display(GxEPD2_420_GDEY042T81(14, 8, 16, 17)); // 400x300, SSD1683

DFRobot_SCD4X SCD4X(&Wire, SCD4X_I2C_ADDR);
DFRobot_SCD4X::sSensorMeasurement_t data;

char displayText[12];

struct readings {
  uint16_t CO2;
  float temp;
  float humid;
  int tempInt;
  int humidInt;
  float battery;
};

readings current;

RTC_DATA_ATTR int CO2_ten_minutes[TEN_MINUTE_READINGS] = {0};
RTC_DATA_ATTR int CO2_one_hour[ONE_HOUR_READINGS] = {0};

int CO2_ten_minute_average = 0;
int CO2_one_hour_average = 0;
RTC_DATA_ATTR int bootCount = 0;
int i;

void setup() {
  Serial.begin(115200);

  pinMode(BATT_V_PIN, INPUT);

  while( !SCD4X.begin() ){
    Serial.println("Communication with device failed, please check connection");
    delay(3000);
  }
  Serial.println("SCD4X initialised OK");

  bootCount++;
  Serial.println("Boot count: " + String(bootCount));

  SCD4X.enablePeriodMeasure(SCD4X_STOP_PERIODIC_MEASURE);

  display.init(115200,true,50,false);
  
  Serial.println("Display init OK");
  display.setTextColor(GxEPD_BLACK);

}

void loop() {

  display.fillScreen(GxEPD_WHITE);

  SCD4X.measureSingleShot(SCD4X_MEASURE_SINGLE_SHOT);
  Serial.print("Waiting for readings to be available...");
  while (!SCD4X.getDataReadyStatus()) 
  {
    Serial.print(F("."));
    delay(500);
  }
  Serial.println("taking readings now.");
  
  // Discard the first set of data, because the chip datasheet indicates the first reading obtained after waking up is invalid
  SCD4X.measureSingleShot(SCD4X_MEASURE_SINGLE_SHOT);
  while(!SCD4X.getDataReadyStatus()) {
    delay(100);
  }
  SCD4X.readMeasurement(&data);

  current.CO2 = data.CO2ppm;
  current.temp = data.temp;
  current.humid = data.humidity;
  current.tempInt = rint(data.temp);
  current.humidInt = rint(data.humidity);

  Serial.print("Carbon dioxide concentration : ");
  Serial.print(current.CO2);
  Serial.println(" ppm");

  for (i = 0; i++; i < (TEN_MINUTE_READINGS - 1) ) {
    CO2_ten_minutes[i] = CO2_ten_minutes[i+1];
    CO2_ten_minute_average += CO2_ten_minutes[i];
  }
  CO2_ten_minutes[i] = current.CO2;
  CO2_ten_minute_average += CO2_ten_minutes[i];
  CO2_ten_minute_average /= TEN_MINUTE_READINGS;

  if ( bootCount >= TEN_MINUTE_READINGS ) {
    Serial.print("Ten minutes CO2 average: ");
    Serial.println(CO2_ten_minute_average);
  }
  else {
    Serial.println("Awaiting ten-minute average");
  }

  for (i = 0; i++; i < (ONE_HOUR_READINGS - 1) ) {
    CO2_one_hour[i] = CO2_one_hour[i+1];
    CO2_one_hour_average += CO2_one_hour[i];
  }
  CO2_one_hour[i] = current.CO2;
  CO2_one_hour_average += CO2_one_hour[i];
  CO2_one_hour_average /= ONE_HOUR_READINGS;

  if ( bootCount >= ONE_HOUR_READINGS ) {
    Serial.print("One hour CO2 average: ");
    Serial.println(CO2_one_hour_average);
  }
  else {
    Serial.println("Awaiting one-hour average");
  }  

  Serial.print("Environment temperature : ");
  Serial.print(current.temp);
  Serial.println(" C");

  Serial.print("Relative humidity : ");
  Serial.print(current.humid);
  Serial.println("%");

  display.setFont(&FreeSansBold24pt7b);
  display.setTextSize(2);
  sprintf(displayText, "%d", current.CO2);
  drawCentreString(displayText, 200, 100);
  
  display.setTextSize(1);
  sprintf(displayText, "%dC", current.tempInt);
  drawCentreString(displayText, 100, 200);

  sprintf(displayText, "%d%%", current.humidInt);
  drawCentreString(displayText, 300, 200);

  display.setFont(&FreeSansBold12pt7b);
  display.setTextSize(1);
  current.battery = (analogReadMilliVolts(BATT_V_PIN)*0.002);
  Serial.print("Battery reading via ADC on pin 0: ");
  Serial.print(current.battery, 2);
  Serial.println("V");
  sprintf(displayText, "Battery: %.2fV", current.battery);
  drawCentreString(displayText, 200, 260);

  display.nextPage();

  Serial.println();

  display.hibernate();
  SCD4X.setSleepMode(SCD4X_POWER_DOWN);
  esp_sleep_enable_timer_wakeup( ( TIME_TO_SLEEP - millis()/1000 ) * uS_TO_S_FACTOR);
  Serial.println("Setup ESP32 to sleep for " + String(TIME_TO_SLEEP) +
  " Seconds");  
  Serial.println("Going to sleep now");
  Serial.flush(); 
  esp_deep_sleep_start();
  Serial.println("This will never be printed");  
}

void drawCentreString(const char *buf, int x, int y)
{
    int16_t x1, y1;
    uint16_t w, h;
    display.getTextBounds(buf, x, y, &x1, &y1, &w, &h);
    display.setCursor(x - w / 2, y);
    display.print(buf);
}

Do you see that line printed in the log?
Why do you think that is?

I don't see any difference between the three variables with the RTC_DATA_ATTR attribute. I would get rid of boot count and see what happens.

BTW, unless you have a specific reason to count boots, the boot count was just a trivial example; it isn't needed, so I would replace it with whatever data you want to store.

No, I don't see that line. Everything else works as intended.

Although the bootCount is from the example, I've been using it so as to not print the rolling averages until the correct amount of data is available, i.e. the arrays are fully populated with readings. Hence the if ( bootCount >= TEN_MINUTE_READINGS ) and similar for ONE_HOUR_READINGS tests.

Removing those if/else conditions doesn't change things. I'm still getting a zeroed array on each wakeup, and serial output where the sum of all those zeroes plus the new reading is being divided by TEN_MINUTE_READINGS/ONE_HOUR_READINGS, looking like this with a different reading on each wakeup:

Waiting for readings to be available...taking readings now.
Carbon dioxide concentration : 594 ppm
Ten minutes CO2 average: 59
One hour CO2 average: 9

How is a sequential counter like boot count related to a 10 min timer or 60 min timer. get rid of boot count and change the logical operation of is it 10 mins since last report to use either a timer or just use millis(). Same for the 60 min.
The reason you never see the line is the esp32 REBOOTS that means the enbtire sketc h starts over.
The mystery is why the two additional RTC_DATA_ATTR attributes don;t seem to work, but that may be due to invalid logic. Get the basic code right first than I think your problem goes away. You will need to run it for 10 mins and 60 minutes I think, YES / NO?

OK, maybe I've not explained myself well.

It's intended to reboot every 60 seconds, and does so. Take one new reading, store it in the arrays along with the ones from the previous ten/sixty wakeups and rotate out the oldest of those, and use the sum of each array to calculate the new rolling averages.

It's awake for only a few seconds, and then shuts down both the sensor and display so that it's drawing ~2mA while in deep sleep, sustainable with a small solar panel on the board's charge circuit.

Nothing wrong with your description other than the boot count is not needed at all. You need a time as in has 10 mins elapsed and has 60 minutes elapsed.

When the board wakes from sleep, ALL variables will be re-initialized except those marked RTC_DATA_ATTR

Rewrite your 10-minute and 60-minute code so it is the same as your description.

I just noticed one of your key variables will be reset to 0 every 60 secs.

Here are a few questions re this code

 for (i = 0; i++; i < (TEN_MINUTE_READINGS - 1) ) {
    CO2_ten_minutes[i] = CO2_ten_minutes[i+1];
    CO2_ten_minute_average += CO2_ten_minutes[i];
  }
  CO2_ten_minutes[i] = current.CO2;
  CO2_ten_minute_average += CO2_ten_minutes[i];
  CO2_ten_minute_average /= TEN_MINUTE_READINGS;
  1. On line 5 what is the value of i given it is being used outside the for loop. The optimization settings will influence the absolute value, but what did you expect it to be.
  2. Line 3 and 6 appear to be identical hbut 6 has the same i issue.

My 50+ years of experience is nagging at me that this 'algorithm' is at the least less understandable than it could be and may be flat out wrong.

Just noticed something else.
How is the array of 10min values organized? In other word where is the most recent value stored? Same for 60 I imagine.

I have a rewritten version of the 10 min algorithm with the errors fixed but I will let you take a crack at it first.

Ta - am trying that now....

I am second guessing myself, don't take my comments as valid, just use them as hints.

I think if you place the current reading in the array size -1 which is 9 and 59. The next line can be blah=current or leave it but it's my personal choice.
The var CO2_ten_minute_average must be set to 0 prior to the loop and must be marked RTC_DATA_ATTR

For anyone playing along at home, I'm now trying to figure out what's wrong with the average algorithm, with this slimmed down code. (No sleep, just a sped-up loop.)

#include <DFRobot_SCD4X.h>

DFRobot_SCD4X SCD4X(&Wire, SCD4X_I2C_ADDR);
DFRobot_SCD4X::sSensorMeasurement_t data;

struct readings {
  uint16_t CO2;
};

readings current;

int CO2_ten_minutes[10] = {0};

int CO2_ten_minute_average = 0;
int i;

void setup() {
  Serial.begin(115200);
  SCD4X.begin();
  SCD4X.enablePeriodMeasure(SCD4X_STOP_PERIODIC_MEASURE);
}

void loop() {

  SCD4X.measureSingleShot(SCD4X_MEASURE_SINGLE_SHOT);
  while (!SCD4X.getDataReadyStatus()) 
  {
    delay(100);
  }
  SCD4X.readMeasurement(&data);

  current.CO2 = data.CO2ppm;

  Serial.print("Carbon dioxide concentration : ");
  Serial.print(current.CO2);
  Serial.println(" ppm");

  CO2_ten_minute_average = 0;

  for (i = 0; i++; i < 9 ) {
    CO2_ten_minutes[i] = CO2_ten_minutes[i+1];
    CO2_ten_minute_average += CO2_ten_minutes[i];
  }
  CO2_ten_minutes[9] = current.CO2;
  CO2_ten_minute_average += CO2_ten_minutes[9];
  CO2_ten_minute_average /= 10;

  Serial.print("Ten minutes CO2 average: ");
  Serial.println(CO2_ten_minute_average);

  Serial.println();
  delay(5);
}

If you have test data as in 1,4,7,2 then manually calculate and compare to your code result, dope it out from there. OR print the input values and limit the number of data points to say 12.

It looks right to me. Show us your test data input and the code output.

I can't tell if data and current are compatible.

Oh ffs Owain. Get your for-loop syntax in the right order!! No wonder the algorithm looked a bit odd. :dotted_line_face: Having i++ as the condition was always returning false!

Correct code here, working with deep sleep and RTC_DATA_ATTR as intended.

#include <GxEPD2_BW.h>
#include <DFRobot_SCD4X.h>
#include <Fonts/FreeSansBold24pt7b.h>
#include <Fonts/FreeSansBold12pt7b.h>

#define BATT_V_PIN    0

#define uS_TO_S_FACTOR 1000000ULL  /* Conversion factor for micro seconds to seconds */
#define TIME_TO_SLEEP  60        /* Time ESP32 will go to sleep (in seconds) */
#define TEN_MINUTE_READINGS 10      // Adjust these to match TIME_TO_SLEEP
#define ONE_HOUR_READINGS 60    //      "        "      "       "

GxEPD2_BW<GxEPD2_420_GDEY042T81, GxEPD2_420_GDEY042T81::HEIGHT> display(GxEPD2_420_GDEY042T81(14, 8, 16, 17)); // 400x300, SSD1683

DFRobot_SCD4X SCD4X(&Wire, SCD4X_I2C_ADDR);
DFRobot_SCD4X::sSensorMeasurement_t data;

char displayText[12];

struct readings {
  uint16_t CO2;
  float temp;
  float humid;
  int tempInt;
  int humidInt;
  float battery;
};

readings current;

RTC_DATA_ATTR int CO2_ten_minutes[TEN_MINUTE_READINGS] = {0};
RTC_DATA_ATTR int CO2_one_hour[ONE_HOUR_READINGS] = {0};

int CO2_ten_minute_average = 0;
int CO2_one_hour_average = 0;
RTC_DATA_ATTR int bootCount = 0;
int i;

void setup() {
  Serial.begin(115200);

  pinMode(BATT_V_PIN, INPUT);

  while( !SCD4X.begin() ){
    Serial.println("Communication with device failed, please check connection");
    delay(3000);
  }
  Serial.println("SCD4X initialised OK");

  bootCount++;
  Serial.println("Boot count: " + String(bootCount));

  SCD4X.enablePeriodMeasure(SCD4X_STOP_PERIODIC_MEASURE);

  display.init(115200,true,50,false);
  
  Serial.println("Display init OK");
  display.setTextColor(GxEPD_BLACK);

}

void loop() {

  CO2_ten_minute_average = 0;
  CO2_one_hour_average = 0;
 
  display.fillScreen(GxEPD_WHITE);

  SCD4X.measureSingleShot(SCD4X_MEASURE_SINGLE_SHOT);
  Serial.print("Waiting for readings to be available...");
  while (!SCD4X.getDataReadyStatus()) 
  {
    Serial.print(F("."));
    delay(500);
  }
  Serial.println("taking readings now.");
  
  // Discard the first set of data, because the chip datasheet indicates the first reading obtained after waking up is invalid
  SCD4X.measureSingleShot(SCD4X_MEASURE_SINGLE_SHOT);
  while(!SCD4X.getDataReadyStatus()) {
    delay(100);
  }
  SCD4X.readMeasurement(&data);

  current.CO2 = data.CO2ppm;
  current.temp = data.temp;
  current.humid = data.humidity;
  current.tempInt = rint(data.temp);
  current.humidInt = rint(data.humidity);

  Serial.print("Carbon dioxide concentration : ");
  Serial.print(current.CO2);
  Serial.println(" ppm");

  for (i = 0; i < (TEN_MINUTE_READINGS - 1); i++) {
    CO2_ten_minutes[i] = CO2_ten_minutes[i+1];
    CO2_ten_minute_average += CO2_ten_minutes[i];
  }
  CO2_ten_minutes[i] = current.CO2;
  CO2_ten_minute_average += CO2_ten_minutes[i];
  CO2_ten_minute_average /= TEN_MINUTE_READINGS;

  if ( bootCount >= TEN_MINUTE_READINGS ) {
    Serial.print("Ten minutes CO2 average: ");
    Serial.println(CO2_ten_minute_average);
  }
  else {
    Serial.println("Awaiting ten-minute average");
  }

  for (i = 0; i < (ONE_HOUR_READINGS - 1); i++ ) {
    CO2_one_hour[i] = CO2_one_hour[i+1];
    CO2_one_hour_average += CO2_one_hour[i];
  }
  CO2_one_hour[i] = current.CO2;
  CO2_one_hour_average += CO2_one_hour[i];
  CO2_one_hour_average /= ONE_HOUR_READINGS;

  if ( bootCount >= ONE_HOUR_READINGS ) {
    Serial.print("One hour CO2 average: ");
    Serial.println(CO2_one_hour_average);
  }
  else {
    Serial.println("Awaiting one-hour average");
  }  

  Serial.print("Environment temperature : ");
  Serial.print(current.temp);
  Serial.println(" C");

  Serial.print("Relative humidity : ");
  Serial.print(current.humid);
  Serial.println("%");

  display.setFont(&FreeSansBold24pt7b);
  display.setTextSize(2);
  sprintf(displayText, "%d", current.CO2);
  drawCentreString(displayText, 200, 100);
  
  display.setTextSize(1);
  sprintf(displayText, "%dC", current.tempInt);
  drawCentreString(displayText, 100, 200);

  sprintf(displayText, "%d%%", current.humidInt);
  drawCentreString(displayText, 300, 200);

  display.setFont(&FreeSansBold12pt7b);
  display.setTextSize(1);
  current.battery = (analogReadMilliVolts(BATT_V_PIN)*0.002);
  Serial.print("Battery reading via ADC on pin 0: ");
  Serial.print(current.battery, 2);
  Serial.println("V");
  sprintf(displayText, "Battery: %.2fV", current.battery);
  drawCentreString(displayText, 200, 260);

  display.nextPage();
  Serial.println();

  display.hibernate();
  SCD4X.setSleepMode(SCD4X_POWER_DOWN);
  esp_sleep_enable_timer_wakeup( ( TIME_TO_SLEEP - millis()/1000 ) * uS_TO_S_FACTOR);
  Serial.println("Setup ESP32 to sleep for " + String(TIME_TO_SLEEP) +
  " Seconds");  
  Serial.println("Going to sleep now");
  Serial.flush(); 
  esp_deep_sleep_start();
  Serial.println("This will never be printed"); 
}

void drawCentreString(const char *buf, int x, int y)
{
    int16_t x1, y1;
    uint16_t w, h;
    display.getTextBounds(buf, x, y, &x1, &y1, &w, &h);
    display.setCursor(x - w / 2, y);
    display.print(buf);
}

Serial output from where it ticks over to the tenth boot and starts outputing the rolling average:

SCD4X initialised OK
Boot count: 9
Display init OK
Waiting for readings to be available...taking readings now.
Carbon dioxide concentration : 492 ppm
Awaiting ten-minute average
Awaiting one-hour average
Environment temperature : 20.21 C
Relative humidity : 50.31%
Battery reading via ADC on pin 0: 4.23V
_Update_Full : 2945013
_Update_Full : 2945010

Setup ESP32 to sleep for 60 Seconds
Going to sleep now
SCD4X initialised OK
Boot count: 10
Display init OK
Waiting for readings to be available...taking readings now.
Carbon dioxide concentration : 536 ppm
Ten minutes CO2 average: 487
Awaiting one-hour average
Environment temperature : 20.18 C
Relative humidity : 50.39%
Battery reading via ADC on pin 0: 4.23V
_Update_Full : 2945013
_Update_Full : 2945010

Setup ESP32 to sleep f

I would still implement a proper 10-minute and 60-minute time rather than a boot count. boot count is NOT a time variable; it's just the number of times the ESP32 has reset from the sleep timer. Just because it happens to work doesn't make it right, and what happens when you change the sleep value to 14 seconds?

I don't understand the benefit, though? All the bootCount is doing is stopping it from outputting an arithmetical mean before there's sufficient data. Once the arrays are populated, they're just working on a last-in-first-out basis.

The sleep-timer might not end up with it being an average over precisely 10 & 60 minute periods, but it's close enough for my purposes.

I guess for others to be playing with the code, the second and third of these three lines should be calculated from the first rather than stated separately. (And commented better, I know!)

#define TIME_TO_SLEEP  60        /* Time ESP32 will go to sleep (in seconds) */
#define TEN_MINUTE_READINGS 10      // Adjust these to match TIME_TO_SLEEP
#define ONE_HOUR_READINGS 60    //      "        "      "       "