Working on a datalogger

Hi guys,

A friend of mine mentioned wanting to collect some data from his lawn. The idea is to determine based on soil moisture, temperature, sub-surface temperature and humidity, time of day and duration, when is the best time of the day to water the lawn and for how long in order to get the best efficiency out of his water usage. We both realize this is something that is being studied by actual scientists, but it sounded like a fun project to both of us. So I started building and programming a device we can use to gather and log data that can be easily reconfigured depending on what is needed. I'm still working on my code and have a lot of other things I still want to add, but I thought I would share what I have so far because I'm after opinions, improvement ideas, additional features that I can add. And maybe somebody finds something useful for their own projects. So here's what I have so far. It takes up 59% of program memory currently, down from 73%. You might recognize bits of code you have seen in examples online, It's literally come from all over the place. Currently very heavy on comments to make it clear to myself referring back to it later what everything is doing.

//////////////////////////Libraries//////////////////////////
#include <SD.h>     //Library for SD card reader
#include <SPI.h>    //Serial Peripheral Interface library
#include <DS3231.h> //Library for DS3231 Real time clock (Specifically from rinkydinkelectronics)
#include <Wire.h>   //I2C Communication library
#include <LiquidCrystal_I2C.h>  //I2C based LCD display library
#include "DHT.h"    //Supports DHT digital sensors like the DHT22
//////////////////////////Libraries//////////////////////////
/////////////////////////Definitions/////////////////////////
#define DHTpin 5      //Digital pin connected to DHT sensor
#define SoilPin A1
#define DHTtype DHT22 //Tells the DHT library what sensor to use
/////////////////////////Definitions/////////////////////////
const int ChipSelect = 4; //Arduino pin used for SD card chip select
const int TMP1 = 0;  //This is the pin the digital thermometer is connected to
Time t; //Time-data structure (See example code)
/*----------------Initialize-the-RTC--------------------------*/
DS3231 RTC(SDA, SCL);; //This initializes the RTC using I2C bus
/*----------Define-the-LCD-address,-rows,-columns-------------*/
LiquidCrystal_I2C lcd(0x27, 16, 2); //(<Device address>, <Columns>, <Rows>)
/*----Initializing-the-DHT-Sensor------Comment-out-if-not-used*/
DHT dht(DHTpin,DHTtype);

void setup()
{
  lcd.init();       //Initialize the LCD diaplay with the pre-defined settings from above
  lcd.backlight();  //Initialize the LCD backlight
  Serial.begin(115200); //Start the serial interface with given baud rate
  RTC.begin();          //Start the real time clock
/*---------This-is-used-for-calibrating-your-RTC-if-needed---*/
/*
  rtc.setDow(Friday);     //Set day of the week
  rtc.setTime(07, 28, 0); //Set hours, minutes, seconds
  rtc.setDate(06, 27, 2018) //Set day, month, year
 */
/*-------------Checking-if-we-can-access-the-SD-card---------*/
 Serial.println("Initializing SD card..");
  if (!SD.begin(ChipSelect))
  {
    Serial.println("SD card not detected");
    return;
  }
  Serial.println("SD card initialized");
}

void loop()
{
  t = RTC.getTime(); //Get "Time" from the RTC and store in "t"
  int seconds = (t.sec);        //Take only the seconds from RTC and copy to "seconds"
  if (seconds % 5 == 0) //This is the interval timer.
                        //Seconds % 5 == 0 means if seconds divides by 5 with no remainder.
  {
    LCDtime(0,0);       //Calls LCDtime to print time to the LCD
                        //The two variables specify cursor position
    LCDanalogTemp(0,1); //Same as above, but for temperature
//    Serial.println(TempString());   //Print temperature string
//    Serial.println(TimeString());    //print current time
//    Serial.println(SoilMoisture());
    WriteSD(DataString());    //This calls WriteSD and passes it DataString by calling it directly.
    delay(1000);              //Wait until "seconds" is not divisible by 5
  }
}

int SoilMoisture()
{
  int Read = analogRead(SoilPin);   //Read from soil sensor
  Read = map(Read,1024,510,0,100);  //Number drops as moisture increases.
                                    //Calibrate the second number based on your local water
                                    //by submerging the probe and taking the raw value.
  if (Read > 100) //Jus in case the value goes slightly below 510, we wont get 102%
  {
    Read = 100;
  }
  return Read;  //Return soil moisture as percent.
}

float DHTsensor(int i)  //Returns data from an array
{
  float DHumidity = dht.readHumidity();
  float DTempF = dht.readTemperature(true);  //The addition of "true" means we get fahrenheit
  float DIndex = dht.computeHeatIndex(DTempF,DHumidity);  //Calculate heat index from temp and humidity
  float DHTdata[3] = {DHumidity, DTempF, DIndex}; //Put it all in the array
  //Usage examples: Serial.print(DHTsensor(2)); or X = (DHTsensor(2));
  return DHTdata[i];
}

void LCDanalogTemp (int X, int Y) //Sends LCD formatted temp to specified position on LCD
{
  String temp = ("Temp: ");
  String Degf = (" *F");
  String line1 = temp + (AnalogTemp()) + Degf;
  lcd.setCursor(X,Y);
  lcd.print(line1);
  Serial.print("Sent to LCD: ");Serial.println(line1);
  return;
}

void LCDtime (int X, int Y) //Sends LCD formatted time to specified position on LCD
{
  String Clock = ("Time: ");
  String line1 = Clock + (RTC.getTimeStr());
  lcd.setCursor(X,Y);
  lcd.print(line1);
  Serial.print("Sent to LCD: ");Serial.println(line1);
  return;
}
void WriteSD(String SDcardData)  //This function writes to the SD card
{
  File LogDat = SD.open("log.csv",FILE_WRITE); //Assumes file already exists
  if(LogDat)
  {
    Serial.println("Writing to SD");
    LogDat.println(SDcardData);
    LogDat.close();
  }
}

String DataString() //This function builds the data string that will be written to the SD card.
{
  float TF = AnalogTemp();
  String TM = RTC.getTimeStr();
  String CS = ",";  //The character delimiter used in our CSV data file
  String StringOut = (TM) + (CS) + (TF);
  return StringOut;
}

String TimeString() //Returns only clock time as a string. Format "00:00:00" (8 characters)
{
  String tm = RTC.getTimeStr(); //This will give Hour:minute:second
  return tm;
}

String TempString()   //To be used exclusively with AnalogTemp function
{
  float Temp = AnalogTemp();  //Copy temperature data from function AnalogTemp
  String TMP = "Temperature: ";
  String F = (" *F");
  String TempOut = (TMP) + (Temp) + (F); //Build the output string
  return TempOut;     //Return the built string to calling function
}

float AnalogTemp()    //Use this function to get temperature from analog temp sensors
{
  float volts, degC, degF, result;  //Some floats that will be used to hold our data
  volts = getVolts(TMP1);   //Get voltage from the temp sensor
  degC = (volts -0.5) * 100.0;  //Convert voltage to degrees Celsius 
  degF = degC * (9.0/5.0) + 32; //Convert Celcius to Fahrenheit
  result = degF;        //Copy the floating point number in "degF" to "Temp"
  return result;        //Return "Temp" to the function that called it
}

float getVolts(int pin)
{
  return (analogRead(pin)*0.004882814);   //Calculates analog voltage from specified pin and converts to 0-5.
}

I'd suggest you forget about adding any more features and get what you have working.
It won't compile for me. Which Time library are you using?
You also make a common mistake in DataString(). You declare "String StringOut" in the function and return StringOut as the value of the function. But StringOut is now out of scope and you'll get garbage. You do a similar thing in 2 or more of the other functions. I'll leave it as an exercise for the reader to find and fix them.

Pete

el_supremo:
I'd suggest you forget about adding any more features and get what you have working.
It won't compile for me. Which Time library are you using?

This was a frustration for me, having written code using a specific DS3231 library previously on an old computer that has since died. Starting over with a different computer and using code that I had saved, it didn't work. I managed to actually find the same library I used before which is notated on the line where that library is called.

//Library for DS3231 Real time clock (Specifically from rinkydinkelectronics)

Here is the link directly to the page where I downloaded the library.
DS3231 Library from Rinkydinkelectronics.com

It is pretty frustrating that there doesn't seem to be much regulation as to people using the same name for their version of libraries, and unfortunately, there are a few for the DS3231 real time clock module. Most of the ones I have downloaded and tried come with usage examples, which is great, but I liked this one because it can be utilized in fairly short lines of code compared to some of the others.

As for the DataString() function, I don't find that I'm getting garbage returned. I've added a Serial.print to the DataString function just so I can see over serial monitor what is in the string and it is returning time and temperature comma separated just like I expect it to.

Eg. 00:24:30,73.22

I copied this from serial monitor just now. If this is improper use can you please explain why? Because I'm getting exactly the output I expect from the function.

I've added some more things to the program over the weekend and everything gets tested and I make sure it's working before I publish my code. I don't want to lead others away on some wild tangent because I haven't verified my code.

So this whole circuit is built into a project box with the SD reader, RTC and the LCD display and I added male header pins to all of the IO that the base device isn't using for the internal components. This way I can "Easily" connect input and output devices to it and reconfigure it to log different data sets. Well, what I mean by easy is that all you have to do is open the box, feed jumper wires in through the side, plug them into the board and close it back up. That's become somewhat inconvenient. So what I am doing now is I have cut holes in one side for a pair of DE-9 ports. I have not yet decided on where each pin of these will connect, but the idea is that one DE-9 will be 5v, gnd and 7 digital IO pins, while the other will be 3v3, gnd and 7 analog IO pins. Something like this. So I'll then have an external board connected with wire terminals and headers that I can connect things to. The main reason for using a pair of DE-9s is simply because I had a lot of both male and female plugs in my parts bins. I also have three different colors of them so I can use blue for digital and black for analog. Might help keep things in order later on when all of it might not be so fresh in my mind. I'm also considering printing some labels for everything and maybe even writing a short manual. I tend to complete a project and put it aside for a while, so that's more looking out for future me.

If anybody is interested in seeing what it looks like I can take some pictures but don't expect much, this is a prototype and I'm definitely still honing my Dremel skills. Maybe it would be better if I used Fritzing or something and made a diagram of all of the connections. There are definitely better ways to build this thing and my code can still afford a lot of cleanup I'm sure so if anybody is interested in the project and wants more information so they can build their own, I'm happy to share that information. I realize I'm not building something that's never been done before but I'm trying to make it as budget friendly as possible.

Thanks for the great input so far.

Re: Time library
My bad. Your code wouldn't compile for me due to "Time t". To resolve that, I was expecting a <time.h> or <TimeLib.h>.

Re: returning a String
My bad. As @DeltaG points out this isn't returning a pointer, which is what I thought it was doing.
"t = RTC.getTime();" should have clued me in that "Time" was in the RTC library but, of course, I missed it.

Pete

Speaking of Fritzing.. I think I'm going to abandon it too. Couldn't get it to work on my Pi3 B+ running Stretch. I even tried uninstalling, update / upgrade and reinstalling it. The base program works, but none of the parts are there and I didn't feel like dickin' around with it for hours. My Pi is actually what I do most of my hobby stuff / programming on, and if I can't get something to work in a couple hours, I usually drop it.

So yes, I'll be Channeling Forrest Mims III and building schematics on graph paper. I dunno, I just like the way it usually turns out and hadn't thought of it until you suggested it Delta_G.

I've changed my code a little since I shared it last but only for immediate customization for what I'm currently going to use it for. I've also changed it physically a little bit. I've added a pair of DE-9 plugs to one with "Jumper wires" inside so each individual pin of the DE-9 can be connected to a specific IO internally. Hand-crimping all of those has been less than fun. Anyway, once those are in place, I'll be able to create whatever type of interface board I want external from the logger and either plug those right into the DE-9 or connect them with the appropriate cable.

Sounds like a whole lot of extra work right? Yes, but I want to make this as "Complete" of a project as I can because I'm planning to use it for all sorts of data logging things, and I just happened to have a whole bunch of DE-9 sockets of different styles. Since these aren't popular for much anymore, I figured they would be ok to use for this.

I'm also considering adding a few buttons to the front of the project box, and additional code to allow a user to select specific modules they want to use and configure where those are connected. So it would sort of be programmed at run time, and those specific settings would each refer to a numerical value, which would then be stored in EEPROM on the Arduino board. So next time it's booted up, it already has settings and would give the option to keep or change those. THAT is going to be a lot of work, so it's going on the back burner for now.

Speaking of Fritzing.. I think I'm going to abandon it too.

Outstanding idea!

Okay, I've got a question for you guys..

So I'm adding a pair of DE-9 ports to the side of the project box that the datalogger is in and I want to be able to connect it using these ports to external, custom interfaces using serial cables. Before I order anything, do you think I'll run into any problems connecting to things through, say maybe a 25 foot long serial cable? That would probably be longer than I need, I'm just trying to get an idea of what's going to work, but I'm not sure if there will be any attenuation, voltage drop, or interference issues that might cause things to not work right. I've read a few older threads discussing this, but I'm not sure if I can go off the same determination because in those cases they were talking about much longer lines, like 100 meters using old ethernet twisted pairs. I only plan to go 25 feet using a shielded serial cable.

Thoughts and opinions please, data if you happen to have it.

Thank you.

There is no reason for it not to work. It should be pretty easy but anything is possible with enough elbow grease.

DH12043:
There is no reason for it not to work. It should be pretty easy but anything is possible with enough elbow grease.

Yeah I think it will be fine at 25 feet. The only thing I might have to look into is radio interference. I'm working on this project for a friend who is a ham radio operator (We both are) and we talk a lot about resonance. I don't think he operates any equipment that would be transmitting a signal that resonates on a 25 foot long cable, but there is always the possibility that it's "Close enough" to the right length to pick something up. I'll just remind him to only operate QRP while he's got the thing running :slight_smile:

Got another email from him last night after I arrived at work requesting a bunch of changes. Specifically, he wants to be able to see the data in a graph displayed on a web page and it wants it to update "Live". With that, he's probably going to want to be able to download the CSV file from that same web page so I might be switching back to a NodeMCU board for the project sadly.

I'm going to try and work out some code that builds an SVG graph and simply updates the values as it goes along, and have this webpage refresh every 10 seconds or something. So I guess I need to figure out how to read the last N lines of a CSV file. So the program will take sensor data and append that to the CSV, then the web portion will read the last N lines into a buffer and "draw" those in an SVG graph and refresh the page. That last part about downloading the file... We'll tackle that later. Kinda mad that I basically have to start all over again.

Hi All,

Getting back to this, I've already started to work out how I'm going to generate my graph and make it scroll from right to left. The leftmost datapoint will drop off as a new datapoint becomes visible. This will essentially just be a "Live view" of the sensor data which will be visible when the user connects to an access point, which will be hosted by the NodeMCU board. I'll also add a download link to that page, so the user can download the entirety of the CSV data which is stored on a micro SD card attached to the device. I'll likely also have an RTC connected, since I'm not planning for this device to connect to the internet, but it needs to be able to keep track of time so that the graphed data is usable. I've also shared this code in another thread I created, specific to the question of how to make it happen in a simple manner, without the need for a server or any external services.

I've taken out some of the print lines, which were commented out anyway as they were only being used to show me the contents of parts of the array. This example doesn't actually create an SVG file, but it prints all of the data needed to make an SVG file to the serial monitor, so it can be copied and pasted into a blank file and viewed. I'ts still pretty simple, but my plan is to use this to dump time and sensor data into the multi-dimensional array and display it all on a graph. Right now it's just graphing random numbers, I'm hoping somebody finds a use for it.

int arr[4][24];
int pos = 0;

void setup() {
  Serial.begin(115200);
  int x = 0;
  int G = 20;
  for (int i = 0; i <= 23; i++)
  {
    arr[1][i] = x;
    arr[3][i] = G;
    G+=20;
    x++;
  }

}

void loop() {
  int RND = random(10, 90);
  
  arr[2][pos] = RND;
  pos++;

 for (int i = 0; i <= 23; i++)
 {
  arr[1][i]++;
  if (arr[1][i] == 24) arr[1][i] = 0;
//  Serial.print("[");Serial.print(arr[1][i]);Serial.print("]");
  int Z = arr[1][i];
  arr[0][i] = arr[2][Z];
 }

  delay(1000);
  if (pos == 24) pos = 0;
/* ------Generating a raw-text SVG graph for testing purposes ---*/
Serial.println("<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"510\" height=\"210\">");  
Serial.println("<rect x=\"5\" y=\"5\" rx=\"20\" ry=\"20\" width=\"500\" height=\"200\" style=\"fill:#A0FFA0;stroke:black;stroke-width:2;opacity:0.8\" />");
Serial.println("<text x=\"190\" y=\"25\" fill=\"black\">Plotting random numbers</text>");
Serial.println("<line x1=\"5\" y1=\"105\" x2=\"505\" y2=\"105\" style=\"stroke:green;stroke-width:1;opacity:0.3\"/>");
Serial.println("<line x1=\"5\" y1=\"52\" x2=\"505\" y2=\"52\" style=\"stroke:green;stroke-width:1;opacity:0.3\"/>");
Serial.println("<line x1=\"5\" y1=\"157\" x2=\"505\" y2=\"157\" style=\"stroke:green;stroke-width:1;opacity:0.3\"/>");

for (int i = 0; i <= 23; i++)
{
  int X1 = arr[3][i];
  int Y1 = (200 - (arr[0][i]));
  int X2 = X1;
  int Y2 = 195;
  Serial.print("<line x1=\"");Serial.print(X1);Serial.print("\" y1=\"");Serial.print(Y1);Serial.print("\" x2=\"");Serial.print(X2);Serial.print("\" y2=\"");Serial.print(Y2);Serial.println("\" style=\"stroke:blue;stroke-width=20;opacity:1\"/>");
}

Serial.println("</svg>");
Serial.println("");
/* -------------End of raw-text SVG graph generator ---------------*/
}

Hi,
I've been reading through the thread.
Question, does your code work, does it do what you want it to do?

Tom... :slight_smile:

TomGeorge:
Hi,
I've been reading through the thread.
Question, does your code work, does it do what you want it to do?

Tom... :slight_smile:

Yes, so far everything works the way I want it to. I'm getting usable data out the other end. I mostly wanted to have a discussion about this to share ideas and to have the opportunity to share code if anybody is interested in it. I'm not a programmer to make money which is probably obvious from my coding style, but I want to be able to share my work with others, since I put a lot of time into it, it may as well be of use to somebody.

I think the point you are trying to make is if it's not broken, don't fix it right?

My thinking is simply if there is a more effective way to handle data, perhaps using less code space, or if there's an opportunity for me to learn new methods, I don't want to pass that up.