Mass Data Logger to Submit Data to Web Server (as Client)

Hi All,

I'm in the middle of a project where I'm collecting data that I must send to a web server.

I have arduino 1 collecting pulsed data and submitting an ID (byte), a count (integer) and a timestamp (unsigned long?) over I2C (Wire) to arduino 2. I have done this as the pulses I'm collecting could be as short as 6ms. There could be up to 100 'submissions' every minute, these need to find their way to a MySQL database.

I would like arduino 2 to collect this data and submit it over ethernet to my web server for processing. I have been thinking that this should be submitted every 15 minutes, but this could potentially be sending 1,500 values at a time... Is this too much for the little ATMega?

Could I log this data to an SD card (as I assume RAM would be well over capacity) and then submit it to a PHP page for processing? This installation needs to last at least 3 years maintenance free, would the SD card be capable of this kind of labour?

Does anyone know of any projects similar to this, or could anyone give me any pointers or ideas?

I send time and three temperature readings to SD at ten second intervals, i.e. 24 items/minute, and that gives about 260k/day. If the shape of your data is similar, this implies you would send about 1Mb a day, which makes a cheapo 2Gb SD card look a pretty good prospect.

I don't see why a Mega can't send 1500 values at a time. It's just a matter of reading off the card. I think the real problem is finding the time to do it, as there are too many "could be"s in your post, and it might be simpler to pass them on as soon as possible, rather than accumulate anything.

I believe you are obliged to use two Arduinos because you need the first as a data buffer. This is due to the nature and unpredictability of the data, but the problem is that the second is also a data buffer - just bigger and slower.

The nature of the data, if you can call it that, is unclear. If the input is just a pulse, that isn't data. The data is the measurement - when, and for how long, which is done by Arduino. This is starting to look like a good time to learn up about interrupts, of which the Mega has several. You might find you can get by with one Arduino.

Thanks for your feedback Nick. I'm making some good progress with the code and I've included it below in order to give a better idea of what I am doing. There will be 15 sensors rigged up to the Arduino, I'm using Micro's attached to a bespoke PCB. I think that logging the data to an SD card on a separate Arduino is a good approach to manage with a power or internet outage? I fear that if the internet connection drops for a day the Arduino would be well and truly clogged up, however an SD card wouldn't be.

All I need to achieve now is getting the data from that CSV up to the internet, any thoughts on this? Perhaps I could upload the file to an FTP server for cron processing and then delete the file from the Arduino after?

Arduino 1:

#include <Wire.h>

// Setup S02
int S02pin = 19;
int S02index = 0;
int S02reading[10];
int S02currentreading = 0;
unsigned long S02currenttimestamp = 0;
unsigned long S02timestamp[10];
boolean S02laststate = 0;

// Setup Delay Between Readings
unsigned long readingdelay = 1000;

unsigned long looptime = 0;
unsigned long nextsend = 0;
boolean backlog = 0;

int SendData(byte sensor, unsigned long timestamp, int reading)
{
  Wire.beginTransmission(4);
  Wire.write(sensor);
  Wire.write(timestamp >> 24);
  Wire.write(timestamp >> 16);
  Wire.write(timestamp >> 8);
  Wire.write(timestamp);
  Wire.write(reading >> 8);
  Wire.write(reading);
  return Wire.endTransmission();
}

// START Bootup Execution
void setup()
{
  Serial.begin(9600);
  Wire.begin();
}

// START Main Execution Loop
void loop()
{
  looptime = micros();

  // Check if S02 is ready to save
  if(S02currenttimestamp != 0 && S02currenttimestamp < (millis() - readingdelay))
  {
    if(SendData(2, S02currenttimestamp, S02currentreading) != 0)
    {
      S02reading[S02index] = S02currentreading;
      S02timestamp[S02index] = S02currenttimestamp;
      S02index++;
      backlog = 1;
    }
    S02currentreading = 0;
    S02currenttimestamp = 0;
  }
  // START Read Data S02
  if(digitalRead(S02pin) != S02laststate)
  {
    // Prep to pickup change next time
    S02laststate = digitalRead(S02pin);
    S02currentreading++;
    S02currenttimestamp = millis();
  }

  // Send Data

  if( S02index > 0 && millis() > nextsend )
  {
    // START Wire Transmission
    while(S02index != 0)
    {
      S02index--;
      Wire.beginTransmission(4);
      byte sensor = 2;
      Wire.write(sensor);
      Wire.write(S02timestamp[S02index] >> 24);
      Wire.write(S02timestamp[S02index] >> 16);
      Wire.write(S02timestamp[S02index] >> 8);
      Wire.write(S02timestamp[S02index]);
      Wire.write(S02reading[S02index] >> 8);
      Wire.write(S02reading[S02index]);
      if(Wire.endTransmission() != 0)
      { 
        S02index++;
        break;
      }
      else
      {
        S02timestamp[S02index] = 0;
        S02reading[S02index] = 0;
      }
    }
    //backlog = 0;
    nextsend = millis() + 5000;
    
    Serial.print(micros() - looptime);
    Serial.println("us Loop Execution Time");
  }
}
// END Loop
#include <Wire.h>
#include <SD.h>

int i = 0;
unsigned long ul = 0;
const int chipSelect = 4;

void setup()
{
  Wire.begin(4);
  Wire.onReceive(receiveEvent);
  Serial.begin(9600);
  Serial.print("Initializing SD card...");
  pinMode(10, OUTPUT);  // make sure that the default chip select pin is set to output, even if you don't use it:
  SD.begin(chipSelect);
}

void loop()
{
  delay(100);
}

void receiveEvent(int howMany)
{
  while(0 < Wire.available())
  {
    // Read Sensor
    String dataString = "";
    byte sensor = Wire.read();
    dataString += String(sensor);
    Serial.print("S: ");
    Serial.print(sensor);
    // Read Timestamp
    dataString += ",";
    Serial.print(", T: ");
    ul = 0;
    ul = Wire.read() << 24;
    ul |= Wire.read() << 16;
    ul |= Wire.read() << 8;
    ul |= Wire.read();
    dataString += String(ul);
    Serial.print(ul);
    // Read Value
    dataString += ",";
    Serial.print(", R: ");
    i = 0;
    i = Wire.read() << 8;
    i |= Wire.read();
    dataString += String(i);
    Serial.println(i);

    File dataFile = SD.open("datalog.csv", FILE_WRITE);
    if (dataFile) { // if the file is available, write to it:
      dataFile.println(dataString);
      dataFile.close();
      Serial.print("Wrote: ");
      Serial.println(dataString);
    }
  }
}

jamestech:
All I need to achieve now is getting the data from that CSV up to the internet, any thoughts on this? Perhaps I could upload the file to an FTP server for cron processing and then delete the file from the Arduino after?

I just send data direct to both the SD and Xively, no reading off SD. The data is the same and at the same interval, but Xively doesn't get the time stamping. The SD is for backup and the files stay there for eventual downloading.

We are doing all sorts with the data so will be using our VPS for this... However, I see the data is posted to Xively using HTTP PUT, this is something I haven't come across before, perhaps this could be a good way to get my data out. With the time it takes to get data across the ethernet I was thinking of sending the data out only every fifteen minutes, do you think this is a good choice? What might be excessive intervals for publishing?

I'm a small-scale operator and I don't know what a VPS is but, If you want to get your data out to the world, something like Xively is surely the simplest way to go. If you don't want it out to the world, I don't see the point of using ethernet at all. I really can't comment on your choice of frequency, it would depend entirely on your needs. Fifteen minutes would be useless to me, ten seconds is OK. I still don't think you need to save the stuff up and then feed it every fifteen minutes. There are rules about upload rates and quantity and I believe ten seconds is the maximum rate. I still use Xively but I understand Grovestreams is better and you might look at that. I haven't gotten round to defecting.

Thanks,

VPS = Virtual Private Server, in short it simply means we have our own web server to process the data and we would like to keep it this way, it gives us full control of the data and minimal dependencies on a third party.

We intend to have a few of these Arduino's spread around various locations where the internet connection would be made available to us, but we would have no control over it. For this reason sending the data out as a web client to our own web server makes sense. The data will be submitted with timestamps and will be used to create human read historical reports, so as long as the Arduino timestamps the data on collection I'd be happy submitting the data as infrequently as every hour.

Something I have noticed is that my timestamps in the previously posted code seem to be all over the place, do you know why this might be?

jamestech:
Thanks,

VPS = Virtual Private Server, in short it simply means we have our own web server to process the data and we would like to keep it this way, it gives us full control of the data and minimal dependencies on a third party.

We intend to have a few of these Arduino's spread around various locations where the internet connection would be made available to us, but we would have no control over it. For this reason sending the data out as a web client to our own web server makes sense. The data will be submitted with timestamps and will be used to create human read historical reports, so as long as the Arduino timestamps the data on collection I'd be happy submitting the data as infrequently as every hour.

Something I have noticed is that my timestamps in the previously posted code seem to be all over the place, do you know why this might be?

I have two outposts, one under a ski lodge and one in a residence, both several hours drive away, and in opposite directions. The former is inaccessible for the season - about ninety days. The latter can be be visited any time but usually annually. No local attention is expected at either, but they have local display and local bluetooth transmission. Ordinary domestic internet connection is available at both and that is all that is needed.

Timestamped data is downloadable from Xively but I have only done that once. Few things promote a reality check quite like a plethora of data, and a PrintScreen of a Xively page generally suffices. Everything is available from the SD but I don't read it remotely.

IF you are accumulating data in a regular manner, AND your receiver allows you to send at the same frequency, I see no point in sending it on in batches. If you are having a problem, you are probably just moving it rather than fixing it, and are likely to incur new problems in the process. IF NOT to both the above, Xively and those of that ilk are probably not the best way to go.

I'm afraid the code is meaningless, but it seems unnecessarily complicated. This could be partly because it is event-dependent rather than under the control of the Arduino.

The organisation of the code below might be useful. Strip out what you need.

/*
3x DS18B20 sensors at one second intervals.   Nokia 5110 LCD  

Two of the temperatures, plus the difference between them, is transmitted 
over bluetooth in a format suitable for BlueTooth Terminal/Graphics (CETIN)
in order to display three graphs in real time. 2014 VERSION

Every tenth reading is recorded on SD card in real numbers to two decimal
places. Each line is timestamped.

A new file is created at midnight using the date as the filename.

Credit to Hacktronics, Mellis & Igoe, Garage Box, Bildr.org, etc.
Kudos to lar3ry in Saskatchewan, Stanley in KL, Coding Badly in Tx.
*/

#include <OneWire.h>
#include <DallasTemperature.h>
#include <PCD8544.h>             // Nokia 5110
#include "Wire.h"                // MUST HAVE lib for LCD disp, SD card
#include <SD.h>
#include <SPI.h>                 // SD
#include "RTClib.h"              //Date As Filename
#include <string.h>              //Date As Filename 


#define DS1307_ADDRESS 0x68

RTC_DS1307 RTC;
char filename[] = "00000000.CSV";
File myFile;

static PCD8544 lcd;

// Custom symbols
static const byte DEGREES_CHAR = 1;
static const byte degrees_glyph[] = { 0x00, 0x07, 0x05, 0x07, 0x00 };
static const byte SLASH_CHAR = 2;
static const byte slash_glyph[] = {0x00,0x20,0x10,0x08};

// Yellow group BEN BULLEN (red LED)
byte InThermo[8] =  {
  0x28, 0x39, 0xFD, 0x50, 0x04, 0x00, 0x00, 0X69};
byte OutThermo[8] = {
  0x28, 0x09, 0xA9, 0xC0, 0x03, 0x00, 0x00, 0x95};
byte DrainThermo[8] = {
  0x28, 0x62, 0xA5, 0x2D, 0x04, 0x00, 0x00, 0x21}; 
  
#define ONE_WIRE_BUS 3
OneWire oneWire(ONE_WIRE_BUS);
DallasTemperature sensors(&oneWire);
 
int  second, minute, hour, weekDay, monthDay, month, year;
int k=0;

const int chipSelect = 4;

float InTemp, OutTemp, DrainTemp, diff; 

// Define the strings for our datastream IDs
char sensorId0[] = "InThermo";
char sensorId1[] = "OutThermo";
char sensorId2[] = "DrainThermo";

char strIn[8];
char strOut[8];
char strDrain[8];
char strdiff[8];

String stringOne, stringTwo, stringThree, stringFour;

void setup() {
   lcd.begin(84, 48);
     // Register the custom symbols...
  lcd.createChar(DEGREES_CHAR, degrees_glyph);
  lcd.createChar(SLASH_CHAR, slash_glyph);
  
  Wire.begin();
  Serial.begin(9600);
  //RTC.begin();

  delay(300);//Wait for newly restarted system to stabilize
  
  lcd.setCursor (0,0);
  lcd.println("Init SD CARD");
  // make sure that the default chip select pin is set to
  // output, even if you don't use it:
  //pinMode(10, OUTPUT);// Uno
  pinMode(53, OUTPUT);//MEGA
  
  // see if the card is present and can be initialized:
  if (!SD.begin(chipSelect)) 
  {
    lcd.println("Card failed");
    // don't do anything more:
    return;
  }
  lcd.println("CARD OK");
  delay(2000);
  
       getFileName();
      lcd.setCursor(0,3); 
      lcd.println(filename);
        delay(2000);
  
    lcd.clear();
  
  sensors.setResolution(InThermo, 12);
  sensors.setResolution(OutThermo, 12);
  sensors.setResolution(DrainThermo, 12);
      
  //Sequence for bluetooth
  // "E256,-321,982\n" 
  //so insert three variable between four strings, two are the same twice, 
  //to make a fourth string 
  
  stringOne = String("E");
  stringTwo = String(",");
  stringThree = String("\n");

    running();
  }

void loop() {

   GetClock();
  if (hour == 0 && minute == 0 && second <2)
  {
    getFileName();
  }
  //get the values from the DS8B20's 
  sensors.requestTemperatures();

  InTemp = (sensorValue(InThermo));
  OutTemp = (sensorValue(OutThermo));  
  DrainTemp = (sensorValue(DrainThermo)); 

  diff = OutTemp - InTemp;
  
dtostrf(InTemp,4, 2, strIn);
dtostrf(OutTemp,4, 2, strOut);
dtostrf(diff,4, 2, strdiff);

stringFour = stringOne + strIn + stringTwo + strOut;
stringFour = stringFour + stringTwo + strdiff + stringThree;

  Serial.println(stringFour);

  lcd.setCursor(49,0);
  lcd.print(InTemp);
  lcd.setCursor(49,1);
  lcd.print (OutTemp);
  lcd.setCursor(49,2);
  lcd.print(DrainTemp);
  lcd.setCursor(49,3);
  lcd.print(diff);  
  lcd.print(" ");
  lcd.setCursor(17,5);

  lcd.print(hour);
  lcd.print(":");
  lcd.print(minute);
  lcd.print(":");
  lcd.print(second);
  lcd.print("   ");

  k=k+1;  

  if (k>9 )
  {  
  myFile = SD.open(filename, FILE_WRITE);//<<<<<<<<<<<<< OPEN
  myFile.print(hour);
  myFile.print(":");
  myFile.print(minute);
  myFile.print(":");
  myFile.print(second);
  myFile.print(",");

  myFile.print(InTemp);
  myFile.print(",");
  myFile.print(OutTemp);
  myFile.print(",");
  myFile.print(DrainTemp);
  myFile.print(",");
  myFile.print(diff);
  myFile.println();
       myFile.close();//>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>CLOSE
       
      k=0;
  }
  delay(850);
}  // loop ends here

//sensorValue function
float sensorValue (byte deviceAddress[])
{
  float tempC = sensors.getTempC (deviceAddress);
  return tempC;
}

byte bcdToDec(byte val)  {
  // Convert binary coded decimal to normal decimal bers
  return ( (val/16*10) + (val%16) );
}

void running(){
  lcd.setCursor(0,0);
  lcd.print("In");
  lcd.setCursor(31,0);
  lcd.print("\001C ");
  lcd.setCursor(0,1);
  lcd.print("Out");
  lcd.setCursor(31,1);
  lcd.print("\001C ");
  lcd.setCursor(0,2);
  lcd.print("Drain");
  lcd.setCursor(31,2);
  lcd.print("\001C ");
  lcd.setCursor(0,3);
  lcd.print("diff");
  lcd.setCursor(31,3);
  lcd.print("\001C "); 
  lcd.setCursor(15,4);
  lcd.print("Bluetooth");  
  }

void getFileName(){
DateTime now = RTC.now();
 sprintf(filename, "%02d%02d%02d.csv", now.year(), now.month(), now.day());
}

void GetClock(){
  // Reset the register pointer
  Wire.beginTransmission(DS1307_ADDRESS);
  byte zero = 0x00;
  Wire.write(zero);
  Wire.endTransmission();
  Wire.requestFrom(DS1307_ADDRESS, 7);

  second = bcdToDec(Wire.read());
  minute = bcdToDec(Wire.read());
  hour = bcdToDec(Wire.read() & 0b111111); //24 hour time
  weekDay = bcdToDec(Wire.read()); //0-6 -> sunday - Saturday
  monthDay = bcdToDec(Wire.read());
  month = bcdToDec(Wire.read());
  year = bcdToDec(Wire.read());
}