EEPROM to store data? -- Tipping Bucket Rain Gauge

I’m not really having a problem with this working, but I was hoping for general comments on my methods, particularly handling the arrays.

My concern is a power interruption, and I lose 5 days data. Without the trouble of adding an SD card, could I use EEPROM to hold the rain history?

This is talking to my home automation device and it relies on the hourly updates of accumulated rain over the past (up to) 5 days.

“Are my methods unsound?” Col. Kurtz
“I don’t see any method, at all.” Capt. Willard

/*
Arduino Tipping Bucket Rain Guage
 
by Jim (BulldogLowell@gmail.com)
 
*/
#include <EEPROM.h>
#include <SPI.h>
#include <EEPROM.h>  
#include <RF24.h>
#include <Sensor.h>
//
Sensor gw;
boolean metric = false;
//
int tipSensorPin=3;
unsigned long dataMillis;
unsigned long serialInterval = 10000UL;// you should comment out the serial stuff when you are done calibrating
const unsigned long oneHour = 30000UL;  // change this to a lower number for debugging
unsigned long lastTipTime;
unsigned long lastBucketInterval;
unsigned long startMillis;
word dayBuckets [5] [2] = {
  { V_VAR1, 24 },
  { V_VAR2, 48 },
  { V_VAR3, 72 },
  { V_VAR4, 96 },
  { V_VAR5, 120},
};
volatile int rainBucket [120] ; // 5 days of data
const unsigned long calibrateFactor = 2UL; //Calibration in tenths of milimeters of rain per single tip
unsigned long rainRate;
float currentRain = 0;
boolean wasTipped = false;
boolean updateVera;
int rainCount;
volatile int tipBuffer = 0;
int rainWindow = 24;//default rain window in hours
int rainSensorThreshold = 12;//default rain sensor sensitivity in mm
byte hourCount = 24;
byte state = 0;
byte oldState = -1;
//
void setup()  
{ 
  Serial.begin(115200);
  gw.begin(); 
  pinMode(tipSensorPin, OUTPUT);
  attachInterrupt (1, sensorTipped, CHANGE);
  for (int i = 0; i < 120; i++)
  {
    rainBucket[i] = 0;
  }
  gw.sendSketchInfo("Rain Guage", "0.9a"); 
  gw.sendSensorPresentation(1, S_RAIN);
  gw.sendSensorPresentation(2, S_MOTION);
  Serial.println(F("Sensor Presentation Complete"));
  //metric = gw.isMetricSystem();
  pinMode(tipSensorPin, INPUT);
  // 
  gw.sendVariable(1, V_RAIN, "0");
  gw.sendVariable(1, V_RAINRATE, "0");
  gw.sendVariable(2, V_TRIPPED, 0);
  dataMillis = millis();
  startMillis = millis();
  lastTipTime = millis() - oneHour;
  delay(1);
  // uncomment the following block the first time you include this device.
  // is necessary in order to create the five Variable buckets in the correct order.
  // once you create them, you can comment out these lines and re-upload to your arduino
  //------------------------------------------------------
  /*
  for (int j = 0; j < 5; j++)
  {
    delay(2000);
    gw.sendVariable(1, dayBuckets[j] [0], 1);
  }
  delay(2000);
  gw.sendVariable(2,V_VAR1, 1);
  delay(2000);
  gw.sendVariable(2,V_VAR2, 1);
  */
  //------------------------------------------------------  
  rainWindow = atoi(gw.getStatus(2, V_VAR1));
  delay(2000);
  rainSensorThreshold = atoi(gw.getStatus(2, V_VAR2));
  Serial.print(F("Go!!!"));
}
//
void loop()     
{ 
  unsigned long measure = 0; // Check to see if we need to show sensor tripped in this block
  for (int i = 0; i < rainWindow; i++)
  {
    measure = measure + rainBucket [i];
  }
  measure >= rainSensorThreshold ? state = 1: state = 0;
  if (state != oldState)
  {
    gw.sendVariable(2, V_TRIPPED, state);
    oldState = state;
  }
  //
  if (millis() - lastTipTime >= oneHour)// timeout for rain rate
  {
    rainRate = 0;
    lastTipTime = millis() + 1;
  }
  if (updateVera)
  {
    gw.sendVariable(1, V_RAINRATE, rainRate);
    updateVera = false;
  }
  if ( (millis() - dataMillis) >= serialInterval)//Comment out this block to eliminate Serial prints
  {
    for (int i = 24; i <= 120; i=i+24)
    {
      updateSerialData(i);
    }
    dataMillis = millis();
  }
  if ( (millis()- startMillis) < oneHour)
  {
    if (tipBuffer > 0)
    {
      Serial.println(F("Sensor Tipped"));
      rainBucket [0] ++;
      unsigned long tipDelay = millis() - lastTipTime;
      Serial.println(tipDelay);
      if (tipDelay <= oneHour) 
      {
        rainRate = ((calibrateFactor * oneHour) / tipDelay);
        gw.sendVariable(1, V_RAINRATE, rainRate);
        Serial.print(F("RainRate= "));
        Serial.println(rainRate);
      }
      lastTipTime = millis();
      updateVera = true;
      tipBuffer--;
    }
  }
  else 
  {
    Serial.println("one hour elapsed");
    for (int i = 118; i >= 0; i--)//cascade an hour of values
    {
      rainBucket [i + 1] = rainBucket [i];
    }
    rainBucket[0] = 0;
    gw.sendVariable(1, V_RAIN, tipCounter(24));// send 24hr tips
    startMillis = millis();
    for (int j = 0; j < 5; j++)
    {
      int total = 0;
      for (int i = 0; i < dayBuckets[j][1]; i++)
      {
        total = total + rainBucket [i];
      }
      gw.sendVariable( 1, dayBuckets[j] [0], total);
    }
    hourCount++;
    if (hourCount >=4)//8 times daily update the Sensor variables
    {
      rainWindow = atoi(gw.getStatus(2, V_VAR1));
      if (rainWindow < 6) 
      {
        rainWindow = 6;
        gw.sendVariable( 2, V_VAR1, rainWindow);
      }
      if (rainWindow > 120) 
      {
        rainWindow = 120;
        gw.sendVariable( 2, V_VAR2, rainWindow);
      }
      rainSensorThreshold = atoi(gw.getStatus(2, V_VAR2));
      delay(2000);
      if (rainSensorThreshold < 1) 
      {
        rainSensorThreshold = 1;
        gw.sendVariable( 2, V_VAR2, rainSensorThreshold);
      }
      if (rainSensorThreshold > 1000) 
      {
        rainSensorThreshold = 1000;
        gw.sendVariable( 2, V_VAR2, rainSensorThreshold);
      }
      hourCount = 0;
    }
  }
}
//
void sensorTipped()
{
  static unsigned long last_interrupt_time = 0;
  unsigned long interrupt_time = millis();
  if (interrupt_time - last_interrupt_time > 200)
  {
    tipBuffer++;
  }
  last_interrupt_time = interrupt_time;
}
//
unsigned long tipCounter(int x)
{
  int tipCount = 0;
  for ( int i = 0; i < x; i++)
  {
    tipCount = tipCount + rainBucket [i];
  }
  return tipCount;
}
//
void updateSerialData(int x)
{
  Serial.print(F("Tips last "));
  Serial.print(x);
  Serial.print(F("hours: "));
  int tipCount = 0;
  for (int i = 0; i < x; i++)
  {
    tipCount = tipCount + rainBucket [i];
  }
  Serial.println(tipCount);
}

RainGuage.ino (5.65 KB)

#include <EEPROM.h>
#include <SPI.h>
#include <EEPROM.h>

Some people feel it necessary to only include a file once.

const unsigned long oneHour = 30000UL;  // change this to a lower number for debugging

Not in my part of the world. Ditch the stupid comment.

word dayBuckets [5] [2] = {

Without looking, how big is a word? The Arduino has no use for words. It is not saddled with a useless Microslop operating ssytem.

  for (int i = 0; i < 120; i++)
  {
    rainBucket[i] = 0;
  }

The compiler did that already.

  Serial.print(F("Go!!!"));

Good thing you don’t have a Mega!!!

    lastTipTime = millis() + 1;

Plus 1? WTF?

could I use EEPROM to hold the rain history?

Yes.

PaulS:

#include <EEPROM.h>

#include <SPI.h>
#include <EEPROM.h>



Some people feel it necessary to only include a file once.

yeah, I deserve that one, I cannot say how it got there twice

const unsigned long oneHour = 30000UL;  // change this to a lower number for debugging

Not in my part of the world. Ditch the stupid comment.

was working with someone on this

word dayBuckets [5] [2] = {

Without looking, how big is a word? The Arduino has no use for words. It is not saddled with a useless Microslop operating system.

yup, a word is an int

  for (int i = 0; i < 120; i++)

{
    rainBucket[i] = 0;
  }



The compiler did that already.

Oh, OK!

  Serial.print(F("Go!!!"));

Good thing you don’t have a Mega!!!

making sure it got through the radio TX/RX…

    lastTipTime = millis() + 1;

Plus 1? WTF?

i didn’t want the processor to win the race, I guess that is dumb… :blush:

could I use EEPROM to hold the rain history?

Yes.

[/quote]

OK, so can you help me there?

BulldogLowell: This is talking to my home automation device and it relies on the hourly updates of accumulated rain over the past (up to) 5 days.

That suggests you need to store roughly 5x24 = 120 samples, each covering one hour. What's the biggest sample value you need to cope with?

I was thinking about storing numbers

a) if the sensor was 'tripped; (state ==1) b) only the numbers in the array responsible for the state (rainWindow)

I was thinking about storing numbers

Good. That's a lot easier than storing bananas.

So, how big are the numbers? 0 to 255? Bigger? int, long, long long, long long long? word? sentence? paragraph? chapter? novel?

How come Microslop doesn't have all of those?

the highest recorded rain in the US was abut 12in in an hour, or about 300mm

we could do it with a byte

Is there a radio as well?

You should be able to make an SD adapter out of a micro-SD and full-SD adapter sleeve that comes with it.

You'll need to solder pins directly to the contacts on sleeve, feed VCC through a voltage divider if your Arduino is 5V (some are 3.3, stand-alone AVR's can run half-speed on 3.3) and use that same 3.3V to pull up all the data inputs. Then put diodes on the inputs blocking 5V from the Arduino MOSI, SCK and SS but letting the same lines going LOW to drag the pullups down. Voltage leveled with just one divider that way! If you have a 3.3V supply or stepped down from 5V then you don't even need the divider, some 3.3V chips I know take in 5V VCC and have a 3.3V VDD pin that does the job for you.

SD does use SPI bus. Open the data file, write to it and close the file to update the FAT and not lose to power-out.

From what I read on the tipping bucket, they tend to under-report. It might be fun to make one that drives a clock mechanism in place of the pendulum. Every click, a 1-second tick. But that wouldn't say when, just at least how much.

Yes, I'm using the nrf24l01 on this.

I was hoping to not have to mess with the SD card. That's why I asked about EEPROM.

BulldogLowell:
the highest recorded rain in the US was abut 12in in an hour, or about 300mm

we could do it with a byte

In that case you’d just need 120 bytes of EEPROM, plus one additional byte to tell you what was the most recent entry so you could use the EEPROM as a circular buffer. However, unless you have some external time source, after a reset you would have no way to tell how old the ‘most recent’ entry was i.e. you don’t know whether the Arduino has been turned off for a second or a week. If you need to access the historical data after it was powered back on again, does it matter that you don’t know for sure when it was logged?

I think I'm OK with the 'missing' time.

I'll need one byte for if state and up to 120 bytes for the rainWindow (hours selected to use in trip sensor)

(if state ==0, then don't write the rest to EEPROM)

Since I don't know what EEPROM.h is doing with the other libraries, I don't know how to write to EEPROM safely.

That is my issue, I guess.

I’m taking another stab at trying to get this to work storing a few bytes to EEPROM, with no luck. It locks up in setup (uno)

So for the sensor.h library, I know I’m good above address 512 in EEPROM, so I would appreciate some guidance.

/*
Arduino Tipping Bucket Rain Gauge
 
June 12, 2014
 
Version 0.9c   Cleaned up the code
               LED status indicator
            
Still need:    Add read/write to EEPROM to save state and data for power interuption   
 
Arduino Tipping Bucket Rain Gauge
 
Utilizing a tipping bucket sensor, your Vera home automation controller and the MySensors.org 
gateway you can measure and sense if locan rain.  This sketch will create two devices on your 
Vera controller.  One willdisplay your total precipitation for the last 24, 48, 72, 96 and 120
hours.  The other, a sensor that changes state if there is recent rain (up to last 120 hours) 
above a threshold.  Both these settings are user definable. 
 
 This sketch features the following:
 
 * Allows you to set the rain threshold in mm  
 * allows you to determine the interval window up to 120 hours.
 * Displays the last 24, 48, 72, 96 and 120 hours total rain in Variable1 through Variable5
   of the Rain Sensor device
 * Configuration changes to Sensor device update every 3 hours.
 * Will run on any Arduino
 * Will retain Tripped/Not Tripped status and data in a power outage, saving small ammount
   of data to EEPROM
 * There is a unique setup requirement necessary in order to properly present the Vera device 
   variables.  The details are outlined in the sketch below.
 * LED status indicator
 
 by Jim (BulldogLowell@gmail.com) for free public use
 
 */
#include <SPI.h>
#include <EEPROM.h>  
#include <RF24.h>
#include <Sensor.h>
//
#define STATE_LOCATION 513
#define NUMBER_OF_HOURS 514
#define DATA_BUCKETS 515
//
Sensor gw;
boolean metric = false;
//
int tipSensorPin = 3;
int ledPin = 5; //PWM capable required
unsigned long dataMillis;
//unsigned long serialInterval = 10000UL;//commented out the serial print
const unsigned long oneHour = 3600000UL;
unsigned long lastTipTime;
unsigned long lastBucketInterval;
unsigned long startMillis;
word dayBuckets [5] [2] = {
  { V_VAR1, 24 },
  { V_VAR2, 48 },
  { V_VAR3, 72 },
  { V_VAR4, 96 },
  { V_VAR5, 120},
};
volatile byte rainBucket [120] ; // 5 days of data
const unsigned long calibrateFactor = 2UL; //Calibration in tenths of milimeters of rain per single tip
unsigned long rainRate;
float currentRain = 0;
boolean wasTipped = false;
boolean updateVera;
int rainCount;
volatile int tipBuffer = 0;
byte rainWindow = 72;//default rain window in hours
byte rainSensorThreshold = 15;//default rain sensor sensitivity in mm
byte hourCount = 24;
byte state;
byte oldState = -1;
unsigned long ledPulseTime = 500UL;
unsigned long pulseStart;
//
void setup()  
{ 
  Serial.begin(115200);
  gw.begin();
  //
  pinMode(tipSensorPin, OUTPUT);
  attachInterrupt (1, sensorTipped, CHANGE);
  pinMode(ledPin, OUTPUT);
  digitalWrite(ledPin, HIGH);
  gw.sendSketchInfo("Rain Guage", "0.9c"); 
  gw.sendSensorPresentation(1, S_RAIN);
  gw.sendSensorPresentation(2, S_MOTION);
  Serial.println(F("Sensor Presentation Complete"));
  //metric = gw.isMetricSystem();
  pinMode(tipSensorPin, INPUT);
  //
  state = EEPROM.read(STATE_LOCATION);
  if (state == 1)
  {
    byte buckets = EEPROM.read(NUMBER_OF_HOURS);
    for (int i = 0; i = buckets; i++)
    {
      rainBucket [i] = EEPROM.read(DATA_BUCKETS + i);
    }
  }
  // 
  gw.sendVariable(1, V_RAIN, "0");
  gw.sendVariable(1, V_RAINRATE, "0");
  gw.sendVariable(2, V_TRIPPED, 0);
  dataMillis = millis();
  startMillis = millis();
  lastTipTime = millis() - oneHour;
  // uncomment the following block the first time you include this device.
  // is necessary in order to create the five Variable buckets in the correct order.
  // once you create them, you can comment out these lines and re-upload to your arduino
  //------------------------------------------------------
  /*
  for (int j = 0; j < 5; j++)
  {
    delay(2000);
    gw.sendVariable(1, dayBuckets[j] [0], 1);
  }
  delay(2000);
  gw.sendVariable(2,V_VAR1, 1);
  delay(2000);
  gw.sendVariable(2,V_VAR2, 1);
  */
  //------------------------------------------------------  
  rainWindow = atoi(gw.getStatus(2, V_VAR1));
  delay(2000);
  rainSensorThreshold = atoi(gw.getStatus(2, V_VAR2));
  Serial.print(F("Radio Done"));  
  analogWrite(ledPin, 20);
}
//
void loop()     
{ 
  unsigned long measure = 0; // Check to see if we need to show sensor tripped in this block
  for (int i = 0; i < rainWindow; i++)
  {
    measure = measure + rainBucket [i];
  }
  measure >= rainSensorThreshold ? state = 1: state = 0;
  if (millis() - pulseStart < ledPulseTime)
  {
    analogWrite(ledPin, 255);
  }
  else 
  {
    analogWrite(ledPin, 20);
  }
  if (state != oldState)
  {
    gw.sendVariable(2, V_TRIPPED, state);
    //
    EEPROM.write(STATE_LOCATION, state);
    if (state == 1)
    {
      EEPROM.write(NUMBER_OF_HOURS, rainWindow);
      for (int i = 0; i < rainWindow; i++)
      {
        EEPROM.write(DATA_BUCKETS + i, rainBucket [i]);
      }
    }
    //
    oldState = state;
  }
  //
  if (millis() - lastTipTime >= oneHour)// timeout for rain rate
  {
    rainRate = 0;
    lastTipTime = millis();
  }
  if (updateVera)
  {
    gw.sendVariable(1, V_RAINRATE, rainRate);
    updateVera = false;
  }
  /*
  if ( (millis() - dataMillis) >= serialInterval)//Comment back in this block to enable Serial prints
  {
    for (int i = 24; i <= 120; i=i+24)
    {
      updateSerialData(i);
    }
    dataMillis = millis();
  }
  */
  if ( (millis()- startMillis) < oneHour)
  {
    if (tipBuffer > 0)
    {
      Serial.println(F("Sensor Tipped"));
      rainBucket [0] ++;
      pulseStart = millis();
      if (rainBucket [0] > 253) rainBucket[0] = 253; // odd occurance but prevent overflow
      unsigned long tipDelay = millis() - lastTipTime;
      Serial.println(tipDelay);
      if (tipDelay <= oneHour) 
      {
        rainRate = ((calibrateFactor * oneHour) / tipDelay);
        gw.sendVariable(1, V_RAINRATE, rainRate);
        Serial.print(F("RainRate= "));
        Serial.println(rainRate);
      }
      lastTipTime = millis();
      updateVera = true;
      tipBuffer--;
    }
  }
  else 
  {
    Serial.println("one hour elapsed");
    for (int i = 118; i >= 0; i--)//cascade an hour of values
    {
      rainBucket [i + 1] = rainBucket [i];
    }
    rainBucket[0] = 0;
    gw.sendVariable(1, V_RAIN, tipCounter(24));// send 24hr tips
    startMillis = millis();
    for (int j = 0; j < 5; j++)
    {
      int total = 0;
      for (int i = 0; i < dayBuckets[j][1]; i++)
      {
        total = total + rainBucket [i];
      }
      gw.sendVariable( 1, dayBuckets[j] [0], total);
    }
    hourCount++;
    if (hourCount >=4)//8 times daily update the Sensor variables
    {
      rainWindow = atoi(gw.getStatus(2, V_VAR1));
      if (rainWindow < 6) 
      {
        rainWindow = 6;
        gw.sendVariable( 2, V_VAR1, rainWindow);
      }
      if (rainWindow > 120) 
      {
        rainWindow = 120;
        gw.sendVariable( 2, V_VAR2, rainWindow);
      }
      rainSensorThreshold = atoi(gw.getStatus(2, V_VAR2));
      delay(2000);
      if (rainSensorThreshold < 1) 
      {
        rainSensorThreshold = 1;
        gw.sendVariable( 2, V_VAR2, rainSensorThreshold);
      }
      if (rainSensorThreshold > 1000) 
      {
        rainSensorThreshold = 1000;
        gw.sendVariable( 2, V_VAR2, rainSensorThreshold);
      }
      hourCount = 0;
    }
  }
}
//
void sensorTipped()
{
  static unsigned long last_interrupt_time = 0;
  unsigned long interrupt_time = millis();
  if (interrupt_time - last_interrupt_time > 200)
  {
    tipBuffer++;
  }
  last_interrupt_time = interrupt_time;
}
//
unsigned long tipCounter(int x)
{
  int tipCount = 0;
  for ( int i = 0; i < x; i++)
  {
    tipCount = tipCount + rainBucket [i];
  }
  return tipCount;
}
/*
void updateSerialData(int x)
{
  Serial.print(F("Tips last "));
  Serial.print(x);
  Serial.print(F("hours: "));
  int tipCount = 0;
  for (int i = 0; i < x; i++)
  {
    tipCount = tipCount + rainBucket [i];
  }
  Serial.println(tipCount);
}
*/

Divide and conquer - write a test sketch that just does what you want with the EEPROM, and leave everything else out of it.

I envisaged the storage being a 120 byte array of values, and one index byte telling you which value had been stored most recently. Every hour, you'd store your new value over the top of the oldest value and increment the index. I don't know why you'd need to read the data back, but if you want to then you know which is the newest value and can work out all the others from there. Does this sort of approach fit what you're trying to achieve?

PeterH: Divide and conquer - write a test sketch that just does what you want with the EEPROM, and leave everything else out of it.

I envisaged the storage being a 120 byte array of values, and one index byte telling you which value had been stored most recently. Every hour, you'd store your new value over the top of the oldest value and increment the index. I don't know why you'd need to read the data back, but if you want to then you know which is the newest value and can work out all the others from there. Does this sort of approach fit what you're trying to achieve?

I hadn't thought about your approach, because I was thinking "why store more than I need to EEPROM?" and "don't touch it unless you have to."

So you are suggesting 'rotating' the values and 'pointing' to the last, I can visualize that as

if new hour
  index = index ++ (say 15 in this example) 
  store index to EEPROM, position zero
  store a value in EEPROM's position 15.

on a power down, pick up the index from position zero, start at its value (15) and load (15, 15-1, 15-2, ... )

how do I turn the corner at 15-14 to get to 120?

mathematically?

Assuming that latest was the index of the latest value, and length is the total number of elements in the array:

void push(byte value)
{
    latest = (latest + 1) % length;
    savedValues[latest] = value;
}

You'd be doing this in EEPROM rather than RAM, but hopefully you get the idea.

PeterH: Assuming that latest was the index of the latest value, and length is the total number of elements in the array:

void push(byte value)
{
    latest = (latest + 1) % length;
    savedValues[latest] = value;
}

You'd be doing this in EEPROM rather than RAM, but hopefully you get the idea.

yes got it, thanks so much

If you read an unwritten EEPROM slot, IIRC it should be all bits high 255. If not then 0.
As long as you never use one of those values then that value can mark the end of current data.
0 might be best because you don’t normally record a zero rainfall period, do you?

But why not SD? Using EEPROM you have to code to read the data back out and houseclean.

GoForSmoke: If you read an unwritten EEPROM slot, IIRC it should be all bits high 255. If not then 0. As long as you never use one of those values then that value can mark the end of current data. 0 might be best because you don't normally record a zero rainfall period, do you?

But why not SD? Using EEPROM you have to code to read the data back out and houseclean.

I'm moving in that direction, in my thinking. Adding data logging later may be the actual reason to go in that direction.

I'll experiment with it. The good news is I have a working sensor and can upgrade it.

@PeterH,

again thanks for your assistance, you opened my eyes to using a circular buffer in EEPROM.

On further reading, (and as an exercise only in learning) storing the index in say, position ZERO of the EEPROM and 120 bytes following that for the 120 bytes array I’m using in my example.

In the interest of maximizing the life of my Arduino, It seems I be better off using an array of 120 + 1 bytes and flagging the index with (because I am using byte) 255, for example. I would then not be hammering the same location (location of the index) with read/writes and instead, dividing that effect through the entire length of the circular buffer. In this example, because of the size of the array, I am adding two orders of magnitude longer life, I believe.

In that case then, in setup(), I am reading the entire buffer (pseudo):

bufferIndex = 0  / a global variable I will use later in the code to increment the EEPROM circular buffer
for (int i = 0; i  < 121; i++)
{
  bufferIndex = EEPROM.read(215+i) // initial location of my 120 (+1) bytes
  if bufferIndex = 0xFF
  {
    retrieveArray(bufferIndex); // load the array 
    break;
  }
}

Am I looking right minded here?

Will you know when you've found the last written set of 120+1 once the buffer circles?