Go Down

Topic: Compare logged data on SD card to current reading - £25 for working answer  (Read 956 times) previous topic - next topic

GadgetCanyon

Hello,

I am using an Arduino Nano and trying to make a data logger for a project in my shed. It currently records pressure and temperature ever 10 seconds. Similar to a barometer I guess. The code below currently logs the data however the problem is the next bit I'm trying to do. I would like to be able to compare the current result for both temp and pressure with a previous result 24 hours ago recorded on an SD card and turn an LED on if they are the same. The code will run for about 7 days at a time and i'll manually reset it.

I am happy to reorganize the way the information goes to the SD card if it makes it easier.

I currently have no idea how to do this and would really appropriate some help.


Code: [Select]


#include <Arduino.h>
#include <TM1637Display.h>  // adds 8 led library

//code for ADS1115
#include <Wire.h>                        // needed for scl and sda
#include <Adafruit_ADS1015.h>   //ADS library
Adafruit_ADS1115 ads1115;         // construct an ads1115 at address 0x49

// Define temp sensor stuff
#include <Wire.h>
#include "TSYS01.h"
TSYS01 sensor;

//Include SD card reader set up
#include <SD.h>
#include <SPI.h>
File myFile;



//Add rtc
#include "RTClib.h"
RTC_PCF8523 rtc;
char daysOfTheWeek[7][12] = {"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"};


// Module connection pins (Digital Pins) for LED
#define CLK 3   // set pins for led
#define DIO 4   // set pins for led
TM1637Display display(CLK, DIO);


#define CLK1 5   // set pins for led
#define DIO1 6   // set pins for led
TM1637Display display1(CLK1, DIO1);

// The amount of time (in milliseconds) between tests
#define TEST_DELAY   1000

const float  OffSet = 0.5 ;
float V, P;
float L1; //Pressure sensor reading
float T1;  //Temp sensor reading
int pinCS = 53; // Look this up for SD card  - Pin 10 on Arduino Uno

int Sec;

      //for loop rtc function
      //String Sec = "";    // string to hold input
      //int Sec1;

int Y; //for finding previous value


void setup()
{
  Serial.begin(9600);        // open serial port, set the baud rate to 9600 bps
  Serial.println("/** Water pressure sensor demo **/");

  //ADS1115 code
  ads1115.begin();  // Initialize ads1115 188uV/bit

  // for temp sensor
    Wire.begin();
    sensor.init();



 /* //Part of the rtc code
   while (!Serial) {
    delay(1);  // for Leonardo/Micro/Zero
  }
 // Serial.begin(57600);
  if (! rtc.begin()) {
    Serial.println("Couldn't find RTC");
    while (1);
  }
  if (! rtc.initialized()) {
    Serial.println("RTC is NOT running!");
    // following line sets the RTC to the date & time this sketch was compiled
    //rtc.adjust(DateTime(F(__DATE__), F(__TIME__)));
    // This line sets the RTC with an explicit date & time, for example to set
    // January 21, 2014 at 3am you would call:
    // rtc.adjust(DateTime(2014, 1, 21, 3, 0, 0));
  }

*/

  // SD Card Initialization
  pinMode(pinCS, OUTPUT);
  if (SD.begin())
  {
    Serial.println("SD card is ready to use.");
  } else
  {
    Serial.println("SD card initialization failed");
    return;
  }
  rtc.begin(); 

}







void loop()
{

 // rtc startup
     DateTime now = rtc.now();

Sec = ((now.second()));
//Sec1=(Sec.toInt());
       
if ((Sec)==00||Sec==10||Sec==20||Sec==30||Sec==40||Sec==50)
{

 
     
  int k;
  int16_t adc0, adc1, adc2, adc3; //create int for adc0 used later
  uint8_t data[] = { 0xff, 0xff, 0xff, 0xff };
  display.setBrightness(1); // set the level of brightness
  display1.setBrightness(1);//sets brightness of small (second) LED display
  adc0 = ads1115.readADC_SingleEnded(0); // read data from ads1115

//Temp Sensor info
  sensor.read();
  T1=sensor.temperature();
 
//Connect sensor to Analog 0
  V = (adc0 * 0.1875)/1000;         //adc0 * ((4.0 / 32768));     //Sensor output voltage  32767 in the max division for 15bits
  P = (V - OffSet);             //account for 0.5v sensor gives a 0psi
  L1 = (P*374);                    // pressure read by sensor 150psi/4v=37.4
 //  Serial.print(" Pressure:");
 // Serial.print(L1);
 // Serial.println(" psi");
 //Serial.println();



// Show decimal numbers with/without leading zeros
  bool lz = false;             //true all digits on, false only digits used on 
    //display.showNumberDec((L1),lz);
    display.showNumberDecEx((L1*10), (0x80 >> 0), lz);   // this statement adds a decimal point (0x80>>1) sets the decimal point location, change "1" to move it
    display1.showNumberDecEx((T1*10), (0x80 >> 2), lz);  // Temp sensor info on second LED






 // This section is for storing values to SD card
  myFile = SD.open("Nautilus.txt", FILE_WRITE);
  if (myFile) {

    myFile.print(now.day(), DEC);
    myFile.print('/');
    myFile.print(now.month(), DEC);
    myFile.print('/');
    myFile.print(now.year(), DEC);
    myFile.print(' ');
    myFile.print(now.hour(), DEC);
//    myFile.print(':');
    myFile.print(now.minute(), DEC);
//    myFile.print(':');
    myFile.print(now.second(), DEC); 
    myFile.print(" ");   
    myFile.print(float(T1));
    myFile.print(' ');   
    myFile.println(float(L1));
    myFile.close(); // close the file
  }
  // if the file didn't open, print an error:
  else {
    Serial.println("error opening test.txt");
  }




  //Part of rtc code   
    Serial.print(now.year(), DEC);
    Serial.print('/');
    Serial.print(now.month(), DEC);
    Serial.print('/');
    Serial.print(now.day(), DEC);
    Serial.print(" (");
    Serial.print(daysOfTheWeek[now.dayOfTheWeek()]);
    Serial.print(") ");
    Serial.print(now.hour(), DEC);
    Serial.print(':');
    Serial.print(now.minute(), DEC);
    Serial.print(':');
    Serial.print(now.second(), DEC);
    Serial.println();

    Serial.print("Temperature: ");
    Serial.print(T1);
    Serial.println(" deg C");
 
    Serial.print("Pressure: ");
    Serial.println(float(L1*10));
     
    Serial.println("---");
   
DateTime past (now-TimeSpan(0,0,30,6));


//X = myFile.seek(1);
myFile.readStringUntil(195830);
myFile.find("123");


Serial.println(    );
Serial.println("---");
   
   delay(TEST_DELAY); 
    }

}





wvmarle

When storing data on your sd card, don't use a human readable time (year, month, day, etc.). ZThat's very hard for a computer to work with - better use the unix timestamp, normally you can get that through Time.now(). That's an unsigned long, seconds since epoch. Now a 24-hour period is simply a difference of 24*60*60 = 86400 seconds. Later when you import that file in a spreadsheet application it's trivial to have those dates converted into something human readable again for you, while the seconds representation makes processing/graphing easier.

Now as you log every 10 seconds, that's 8,640 lines per day. So the record of 24 hours ago should be 8,640 lines before the current line in the file.

But now, how to get to a certain line? There's no line search function, just a position function. So you have to make sure every line is of identical length, so you can simply calculate where you have to go. Here the sprintf() function comes in play. This can guarantee you to write a single line of exactly the same length every time, making searching for an entry in the file easy.

Your file writing section becomes something like this:

Code: [Select]

 // This section is for storing values to SD card
  myFile = SD.open("Nautilus.txt", FILE_WRITE);
  if (myFile) {
    char line[26]; // 25 characters plus null terminator
    char temp[7];
    dtostrf(T1, 6, 2, temp); // Convert float to string.
    char press[8];
    dtostrf(L1, 7, 2, press);
    sprintf(line, "%10d %6s %7s", now.unixtime(), temp, press);
    myFile.println(line); // writes a 25-character string plus a \n so 26 characters to the file - that's one line.
  }


This way every line will be 26 characters long, and it becomes easy to read back the parts with temperature and pressure (use
Code: [Select]
strncpy(dest, src + beginIndex, endIndex - beginIndex);), and convert it back into float using atof(). The line of 24 hours ago starts 8640 * 26 = 224,640 characters ago.

For comparison: floats are never exactly the same, use an interval. So:

Code: [Select]

if (abs(yesterdayTemp - todayTemp) < 0.5) {
  // Less than 0.5° difference = equal.
}


Quality of answers is related to the quality of questions. Good questions will get good answers. Useless answers are a sign of a poor question.

GadgetCanyon

Hi wvmarle,

thank you for the help its really appreciated. Currently going through your answer and I have a few questions

1:  Can you explain this part of the code, i get sprintf() from your explination however the parts inside the brackets i'm not sure how each works. sprintf(line, "%10d %6s %7s", now.unixtime(), temp, press);

2: I'm really sorry however i'm not sure on the second part of your answer


This way every line will be 26 characters long, and it becomes easy to read back the parts with temperature and pressure (use strncpy(dest, src + beginIndex, endIndex - beginIndex);), and convert it back into float using atof(). The line of 24 hours ago starts 8640 * 26 = 224,640 characters ago.



wvmarle

1:  Can you explain this part of the code, i get sprintf() from your explination however the parts inside the brackets i'm not sure how each works. sprintf(line, "%10d %6s %7s", now.unixtime(), temp, press);
[/code]

sprintf() helps formatting string.
line is the destination
then the format string: 10 characters digits; 6 characters string; 7 characters string.
And the parameters: the timestamp (seconds since epoch), temperature, pressure.

More reference on sprintf here, and details on the format string here in this printf reference. Note that printf() as such is not available on Arduino, and also the f parameter is not implemented on Arduino, hence the workaround.

Quote
2: I'm really sorry however i'm not sure on the second part of your answer
I fixed the smiley face already...

After reading a line from your file in a char array, you can use that function to distil the appropriate parts.

Code: [Select]

char line[26]; // Line of data goes in this char array.


char temp[7]; // Store the temperature as string in here.
strncpy(temp, *line + 11, 6); // Read 6 characters from line starting at the 12th character and copy this into char array temp.
float T = atof(temp);
Quality of answers is related to the quality of questions. Good questions will get good answers. Useless answers are a sign of a poor question.

GadgetCanyon

Hi wvmarle,

Ok I am nearly there, so far i have this for writing and then reading the SD card.


Code: [Select]


// This section is for storing values to SD card
  myFile = SD.open("Nautilus.txt", FILE_WRITE);
  if (myFile) {
    char line[26]; // 25 characters plus null terminator
    char temp[7];
    dtostrf(T1, 6, 2, temp); // Convert float to string.
    char press[8];
    dtostrf(L1, 7, 2, press);
    sprintf(line, "%10d %6s %7s", now.unixtime(), temp, press);
    myFile.println(line); // writes a 25-character string plus a \n so 26 characters to the file - that's one line.
  }


// This section is for reading from the SD card
myFile = SD.open("Nautilus.txt", FILE_WRITE);
  if (myFile) {

char line[26]; // Line of data goes in this char array.
char temp[7]; // Store the temperature as string in here.
strncpy(temp, *line + 11, 6); // Read 6 characters from line starting at the 12th character and copy this into char array temp.
float T = atof(temp);

  }


I was reading your first post where you said:

strncpy(dest, src + beginIndex, endIndex - beginIndex);)

The line of 24 hours ago starts 8640 * 26 = 224,640 characters ago.

not sure how to implement this exactly?

Thank you again for all your help and patience



wvmarle

Basically, get the current file size, deduct that number, and do a seek() to point the cursor at that position and start reading. This should do the job:

Code: [Select]

File f; // The file with your data in it.
uint32_t fileSize = f.size(); // The total file size.
f.seek(fileSize - 224640); // Go to the start of yesterday's record.
char[27] oldRecord;
f.read(oldRecord, 26); // Read 26 bytes.
oldRecord[26] = 0; // Null termination (may not be needed).

char temp[7]; // Store the temperature as string in here.
strncpy(temp, *oldRecord + 11, 6); // Read 6 characters from line starting at the 12th character and copy this into char array temp.
float T = atof(temp);


That should do. As long as I counted all string lengths correctly :-) If not it'll be off. Just do a Serial.print() of the string to make sure it's got the correct record and positions.
Quality of answers is related to the quality of questions. Good questions will get good answers. Useless answers are a sign of a poor question.

GadgetCanyon

One error, i have marked when it is in the code

Code: [Select]


File f; // The file with your data in it.
uint32_t fileSize = f.size(); // The total file size.
f.seek(fileSize - 224640); // Go to the start of yesterday's record.
char[27] oldRecord; // Error highlights here
f.read(oldRecord, 26); // Read 26 bytes.
oldRecord[26] = 0; // Null termination (may not be needed).




Error

Code: [Select]


 Not used: C:\Users\Brigh\OneDrive\Documents\Arduino\libraries\SD-master
exit status 1
expected unqualified-id before '[' token


wvmarle

Quality of answers is related to the quality of questions. Good questions will get good answers. Useless answers are a sign of a poor question.

GadgetCanyon

Right i have nearly got it using the following from what you have said

Code: [Select]
// This section is for storing values to SD card
  myFile = SD.open("Nautilus.txt", FILE_WRITE);
  if (myFile) {

    char line[26]; // 25 characters plus null terminator
    char temp[7];
    dtostrf(T1, 6, 2, temp); // Convert float to string.
    char press[8];
    dtostrf(L1, 7, 2, press);
    sprintf(line, "%10d %6s %7s", now.unixtime(), temp, press);
    myFile.println(line); // writes a 25-character string plus a \n so 26 characters to the file - that's one line.

File f; // The file with your data in it.
uint32_t fileSize = f.size(); // The total file size.
f.seek(fileSize - 52); // Go to the start of yesterday's record. 60sec ago 6*26=156
char oldRecord [26];
f.read(oldRecord, 26); // Read 26 bytes.
//oldRecord[26] = 0; // Null termination (may not be needed).

//char temp[7]; // Store the temperature as string in here.
strncpy(temp, *oldRecord + 20, 6); // Read 6 characters from line starting at the 12th character and copy this into char array temp.
T = atof(temp);
   


Serial.print("20 sec ago: ");
    Serial.println(T);
    Serial.println("---"); 



    */
    myFile.close(); // close the file   
  }



The SD card has the following on it:


    21906          21.00
    21916          21.00
    21926          21.00
    21936          21.00
    21946          21.00
    21956          21.00
    21966          21.00
    21976          21.00
    21986          21.00
    21996          21.00



I have used the Serial.println(T) to show if its reading anything. So far i keep getting 0.00. any ideas?

wvmarle

I thought there should be three values per line: the time, temperature (T1) and pressure (L1) values. The pressure is missing from your output.

After the sprintf() do add
Code: [Select]
Serial.println(line); to see what is actually written to the SD card, make sure that's correct.

Likewise after the strncpy() add
Code: [Select]
Serial.println(temp); again to makes sure it contains the correct values. atof() returns 0 when the string is incorrectly formatted and it can't make sense of it.

One problem that I see: you're trying to read the line from character 20, which is where the pressure value should be (I don't see it in your output, so it'll be a string of spaces or so, meaning atof() will return 0). The temperature should be starting at character 11.
Quality of answers is related to the quality of questions. Good questions will get good answers. Useless answers are a sign of a poor question.

GadgetCanyon

right i have narrowed down the issue

Code: [Select]

 // This section is for storing values to SD card
  myFile = SD.open("Nautilus.txt", FILE_WRITE);
  if (myFile) {

    char line[26]; // 25 characters plus null terminator 26
    //char time1[11];
    //dtostrf(now.unixtime(),10,2,time1);
    char temp[7];
    dtostrf(T1, 6, 2, temp); // Convert float to string.
    char press1[8];
    dtostrf((L1*10), 7, 2, press1);
    sprintf(line, "%10d %10s %6s %7s", now.unixtime(), temp, press1);
    myFile.println(line); // writes a 25-character string plus a \n so 26 characters to the file - that's one line.
    Serial.println(line);

   
    Serial.println(now.unixtime());

    myFile.close(); // close the file   
  }



I edited one of your line "sprintf(line, "%10d %10s %6s %7s", now.unixtime(), temp, press1);" to see if it would make any difference (I added %10s). that is when i started getting -32516  M~1 below. Before i added %10s that first part of the code did not print out.

this is what i can see on the serial monitor

Code: [Select]

SD card is ready to use.
    -32516    M~1     23.96   36.70
---
    -32506    M~1     23.92   34.59
---
    -32496    M~1     23.88   36.00
---
    -32486    M~1     23.83   34.59
---
    -32476    M~1     23.76   33.89
---
    -32466    M~1     23.71   33.89
---
    -32456    M~1     23.67   31.79
---
    -32446    M~1     23.62   34.59
---
    -32436    M~1     23.58   35.30
---
    -32426    M~1     23.55   29.69
---
    -32416    M~1     23.51   34.59
---
    -32406    M~1     23.48   34.59
---
    -32396    M~1     23.45   33.19
---
    -32386    M~1     23.42   31.79
---
    -32376    M~1     23.41   34.59
---
    -32366    M~1     23.39   33.19
---



and the following is on the SD card


Code: [Select]


    -32516    M~1     23.96   36.70
    -32506    M~1     23.92   34.59
    -32496    M~1     23.88   36.00
    -32486    M~1     23.83   34.59
    -32476    M~1     23.76   33.89
    -32466    M~1     23.71   33.89
    -32456    M~1     23.67   31.79
    -32446    M~1     23.62   34.59
    -32436    M~1     23.58   35.30
    -32426    M~1     23.55   29.69
    -32416    M~1     23.51   34.59
    -32406    M~1     23.48   34.59
    -32396    M~1     23.45   33.19
    -32386    M~1     23.42   31.79
    -32376    M~1     23.41   34.59
    -32366    M~1     23.39   33.19
    -32356    M~1     23.37   34.59
    -32346    M~1     23.35   33.89
    -32336    M~1     23.32   31.09
    -32326    M~1     23.28   33.19
    -32316    M~1     23.24   34.59
    -32306    M~1     23.21   33.19



From what i can tell there seem to be an issue with the time being added.

wvmarle

Yes, there are serious issues. Multiple. I've been fighting more with sprintf, never been able to get it right the first time.

The line is much longer than expected (due to the extra %10s part). The number for time should be positive, not negative. Have to use u instead of d in the formatting string to make it unsigned.

Code: [Select]

 sprintf(line, "%10u %6s %7s", now.unixtime(), temp, press1);


This ought to work but there's something odd with the now.unixtime() function - as shown by the output on the Serial monitor.  Instead of a number, you get --- printed. Can you give me a link to the rtc library you use? I can probably quite quickly see what's going on there. There appears to be a type problem there. I'm expecting to get a timestamp as unsigned long, but it appears that's not the case here.
Quality of answers is related to the quality of questions. Good questions will get good answers. Useless answers are a sign of a poor question.

GadgetCanyon

Changed it to sprintf(line, "%10u %6s %7s", now.unixtime(), temp, press1); as requested, now getting:


Code: [Select]

SD card is ready to use.
     35430         21.97
---
     35440         21.99
---
     35450         22.00
---
     35460         22.00
---
     35470         22.01
---
     35480         22.02
---
     35490         22.02
---
     35500         22.02



the rtc lis this https://learn.adafruit.com/adafruit-pcf8523-real-time-clock/rtc-with-arduino

and the library is this https://github.com/adafruit/RTClib

el_supremo

unixtime presumably returns an unsigned long integer. The %d format in sprintf implies an 'int' which on UNO/NANO/etc is a 16-bit integer.
Use %ld.
P.S. There is a warning message from the compiler about this - for example:
Code: [Select]
C:\Users\Peter\AppData\Local\Temp\arduino_modified_sketch_136698\sketch_jul18a.ino:11:45: warning: format '%d' expects argument of type 'int', but argument 3 has type 'uint32_t {aka long unsigned int}' [-Wformat=]

   sprintf(tmp,"Long integer warning %d\n",ul);

                                             ^


Pete
Don't send me technical questions via Private Message.

GadgetCanyon

ok so changing the line to the below has worked

Code: [Select]
sprintf(line, "%10ld %6s %7s", now.unixtime(), temp, press1);

I have now tried to see the filesize using

Code: [Select]

File Nautilus; // The file with your data in it.
uint32_t fileSize = Nautilus.size(); // The total file size.
Nautilus.seek(fileSize - 52);
 
//  Serial.print(Nautilus.size());
  //  Serial.println("---");



I have tried a few different combinations of the above however none of them have producing anything only "0"


Code: [Select]


SD card is ready to use.
1531947710  21.57   56.33
1531947710

0
---
---

 

Go Up