Best way to format data for analysis

system · July 26, 2014, 5:51am

Hello All,
I am working on a humidity sensor unit using three humidity sensors and the SDFat Library. The data is going into a .csv file but its not in any usable order. The program pulling each reading every couple of seconds and recording it in one column. I want to be able to group the readings from each sensor in there own individual columns (as shown below in my example)

#include <dht11.h>
#include <SdFat.h>
#include <Time.h> 

#define TIME_HEADER  "T"   // Header tag for serial time sync message
#define TIME_REQUEST  7    // ASCII bell character requests a time sync message 

SdFat sd;
SdFile myFile;

dht11 DHT1; // define the sensor 1 type
dht11 DHT2; // define the sensor 2 type
dht11 DHT3; // define the sensor 3 type
const int chipSelect = 8; // sets CS for SD reader to be on pin 8

void setup()
{
  DHT1.attach(14); // set pin of sensor 1 to analog 0
  DHT2.attach(15); // set pin of sensor 2 to analog 1
  DHT3.attach(16); // set pin of sensor 2 to analog 2
  Serial.begin(9600); // start clock
  while (!Serial); // wait for arduino to start up
  Serial.println("Temperature and Humidity Readings ");
  Serial.print("3 Sensors: ");
  setSyncProvider( requestSync);  //set function to call when sync for clock required
  Serial.println("Waiting for sync message");
  delay(400);
  Serial.println("\n");
  int chk1 = DHT1.read(); // declare chk1 variable as status of sensor 1
  int chk2 = DHT2.read(); // declare chk2 variable as status of sensor 2
  
  Serial.print("Read sensor 1: "); // show status of sensor 1
  switch (chk1)
      {
        case 0: Serial.println("OK \n"); break;
        case -1: Serial.println("Checksum error"); break;
        case -2: Serial.println("Time out error"); break;
        default: Serial.println("Unknown error"); break;
      }
      
 Serial.print("Read sensor 2: "); // // show status of sensor 2
 switch (chk2)
      {
        case 0: Serial.println("OK \n"); break;
        case -1: Serial.println("Checksum error"); break;
        case -2: Serial.println("Time out error"); break;
        default: Serial.println("Unknown error"); break;
      }
      
}
  void digitalClockDisplay()// digital clock display of the time
{
  
  Serial.print(hour());
  printDigits(minute());
  printDigits(second());
  myFile.print(hour());
  myFileDigits(minute());
  myFileDigits(second());
}

void printDigits(int digits) // utility function for digital clock display: prints preceding colon and leading 0
{
  Serial.print(":");
  if(digits < 10)
  {
    Serial.print('0');
  }
  Serial.print(digits);
}

void myFileDigits(int myFileDigits)
{
  myFile.print(":");
  if (myFileDigits < 10)
  {
    myFile.print('0');
  }
  myFile.print(myFileDigits);
}


void processSyncMessage() 
{
  unsigned long pctime;
  const unsigned long DEFAULT_TIME = 1357041600; // Jan 1 2013
   if(Serial.find(TIME_HEADER)) 
   {
     pctime = Serial.parseInt();
     if( pctime >= DEFAULT_TIME)  // check the integer is a valid time (greater than Jan 1 2013)
     {
       setTime(pctime); // Sync Arduino clock to the time received on the serial port
     }
  }
}

time_t requestSync()
{
  Serial.write(TIME_REQUEST);  
  return 0; // the time will be sent later in response to serial mesg
}

void loop()
{     
     if (Serial.available()) 
     {
      processSyncMessage();
     }
     if (!sd.begin(chipSelect, SPI_HALF_SPEED)) sd.initErrorHalt(); // Initialize SdFat or print a detailed error message and halt, Use half speed like the native library.change to SPI_FULL_SPEED for more performance.
     if (!myFile.open("Readings.csv", O_RDWR | O_CREAT | O_AT_END)) // open file to write to
        {
          sd.errorHalt("opening Readings.csv for write failed"); // error message if failed
        }
  
      float h1 = DHT1.humidity;
      h1 = h1; // calibration for sensor 1
      digitalClockDisplay();
      myFile.print(',');  
      Serial.println();
      Serial.println("Sensor 1");
      Serial.print("Humidity (%): ");
      Serial.println(h1, DEC);
      Serial.print("Temperature (°C): ");
      Serial.println((float)DHT1.temperature, DEC);
      Serial.print("Temperature (°F): ");
      Serial.println(DHT1.fahrenheit(), DEC);
      Serial.print("Dew Point (°C): ");
      Serial.println(DHT1.dewPoint(), DEC);
      Serial.print("Dew PointFast (°C): ");
      Serial.println(DHT1.dewPointFast(), DEC);
      Serial.println();
      Serial.println();
      myFile.println(",");
      myFile.println("Sensor 1");
      myFile.print("Humidity (%), ");
      myFile.print(h1, DEC);
      myFile.println(",");
      myFile.print("Temperature (°C), ");
      myFile.print((float)DHT1.temperature, DEC);
      myFile.println(",");
      myFile.print("Temperature (°F), ");
      myFile.print(DHT1.fahrenheit(), DEC);
      myFile.println(",");
      myFile.print("Dew Point (°C), ");
      myFile.print(DHT1.dewPoint(), DEC);
      myFile.println(",");
      myFile.print("Dew PointFast (°C), ");
      myFile.print(DHT1.dewPointFast(), DEC);
      myFile.println(",");
      myFile.println();
      myFile.println();
      
      
      float h2 = DHT2.humidity;
      h2 = h2 + 0; // calibration for sensor 2
      
      digitalClockDisplay(); 
      myFile.print(','); 
      Serial.println();
      Serial.println("Sensor 2");
      Serial.print("Humidity (%): ");
      Serial.println(h2, DEC);
      Serial.print("Temperature (°C): ");
      Serial.println((float)DHT2.temperature, DEC);
      Serial.print("Temperature (°F): ");
      Serial.println(DHT2.fahrenheit(), DEC);
      Serial.print("Dew Point (°C): ");
      Serial.println(DHT2.dewPoint(), DEC);
      Serial.print("Dew PointFast (°C): ");
      Serial.println(DHT2.dewPointFast(), DEC);
      Serial.println();
      Serial.println();
      myFile.println("Sensor 2");
      myFile.print("Humidity (%),");
      myFile.print(h2, DEC);
      myFile.println(",");
      myFile.print("Temperature (°C), ");
      myFile.print((float)DHT2.temperature, DEC);
      myFile.println(",");
      myFile.print("Temperature (°F), ");
      myFile.print(DHT2.fahrenheit(), DEC);
      myFile.println(",");
      myFile.print("Dew Point (°C), ");
      myFile.print(DHT2.dewPoint(), DEC);
      myFile.println(",");
      myFile.print("Dew PointFast (°C), ");
      myFile.print(DHT2.dewPointFast(), DEC);
      myFile.println(",");
      myFile.println('\n');
      myFile.println('\n');
      
      
      float h3 = DHT3.humidity;
      h3 = h3 + 0; // calibration for sensor 3
      
      digitalClockDisplay(); 
      myFile.print(','); 
      Serial.println();
      Serial.println("Sensor 3");
      Serial.print("Humidity (%): ");
      Serial.println(h3, DEC);
      Serial.print("Temperature (°C): ");
      Serial.println((float)DHT3.temperature, DEC);
      Serial.print("Temperature (°F): ");
      Serial.println(DHT3.fahrenheit(), DEC);
      Serial.print("Dew Point (°C): ");
      Serial.println(DHT3.dewPoint(), DEC);
      Serial.print("Dew PointFast (°C): ");
      Serial.println(DHT3.dewPointFast(), DEC);
      Serial.println();
      Serial.println();
      myFile.println("Sensor 3");
      myFile.print("Humidity (%),");
      myFile.print(h3, DEC);
      myFile.println(",");
      myFile.print("Temperature (°C), ");
      myFile.print((float)DHT3.temperature, DEC);
      myFile.println(",");
      myFile.print("Temperature (°F), ");
      myFile.print(DHT3.fahrenheit(), DEC);
      myFile.println(",");
      myFile.print("Dew Point (°C), ");
      myFile.print(DHT3.dewPoint(), DEC);
      myFile.println(",");
      myFile.print("Dew PointFast (°C), ");
      myFile.print(DHT3.dewPointFast(), DEC);
      myFile.println(",");
      myFile.println('\n');
      myFile.println('\n');
      
      delay(1500);
      
      myFile.close();
    
}

The format the code is currently printing is shown here

And how the end product should look like

Any help or suggestions would be appreciated!

pito · July 26, 2014, 7:42am

Just print all the data delimited by a "," and with cr/lf at the end of a measurement. That's it.

A measurement at time X (use the data_xy as it is in your above code) ie.:

myFile.print(time);
myFile.print(",");
myFile.print(data1);
myFile.print(",");
myFile.print(data2);
myFile.print(",");
myFile.print(data3);
myFile.print(",");
myFile.print(data4);
myFile.print(",");
...
myFile.print(",");
myFile.print(data15);
myFile.println();

You will get the CSV format:

time,data1,data2,data3,data4,..,data15
time,data1,data2,data3,data4,..,data15
time,data1,data2,data3,data4,..,data15
..

Before the start of the measurement loop print the headers (column names) in the same manner, ie.:

myFile.print("Time");
myFile.print(",");
myFile.print("Humidity S1");
myFile.print(",");
myFile.print("Temperature S1");
myFile.print(",");
myFile.print("Dew point S1");
myFile.print(",");
...
myFile.print(",");
myFile.print("Dew point S3");
myFile.println();

So your CSV file will look like (there will be the actual numbers instead of time and data_xy of course):

Time,Temperature S1,Dew point S1,..,Dew point S3
time,data1,data2,data3,data4,..,data15
time,data1,data2,data3,data4,..,data15
time,data1,data2,data3,data4,..,data15
..

You may load such file directly to a spreadsheet then..

robtillaart · July 26, 2014, 8:06am

be aware that you must add the number of decimals you want for floats, by default it is 2.
e.g.
File.print(dewpoint, 6);
if all digits are meaningful (significant) is up to you

Second note that things like dewpoint can be calculated much faster by the spreadsheet.
This will reduce the size of the log file substantially .
However if the Arduino has enough time and size of the file is not an issue, it does not matter.

system · July 28, 2014, 8:20pm

Hey thanks for the response!

I have updated my code and it work how I like it to thanks to you. Here is the code I have to complete it.

 void loop()
{      if (!sd.begin(chipSelect, SPI_HALF_SPEED)) sd.initErrorHalt(); // Initialize SdFat or print a detailed error message and halt, Use half speed like the native library.change to SPI_FULL_SPEED for more performance.
       if (!myFile.open("Readings.csv", O_RDWR | O_CREAT | O_AT_END)) // open file to write to
        {
          sd.errorHalt("opening Readings.csv for write failed"); // error message if failed
        }
      
      
      float h1 = DHT1.humidity;
      h1 = h1; // calibration for sensor 1
      float h2 = DHT2.humidity;
      h2 = h2 + 0; // calibration for sensor 2
      float h3 = DHT3.humidity;
      h3 = h3 + 0; // calibration for sensor 3   
      int Header = 1;    
      
      for (int Header; Header < 2; Header++) // prints the headers for the spread sheet but will only do it once
      {
      
        myFile.print("Time"); 
        myFile.print(",");
        myFile.print("Humidity - Sensor 1");
        myFile.print(",");
        myFile.print("Temperature - Sensor 1");
        myFile.print(",");
        myFile.print("Dew Point - Sensor 2");
        myFile.print(",");
        myFile.print("Humidity - Sensor 2");
        myFile.print(",");
        myFile.print("Temperature - Sensor 2");
        myFile.print(",");
        myFile.print("Dew Point - Sensor 2");
        myFile.print(",");
        myFile.print("Humidity - Sensor 3");
        myFile.print(",");
        myFile.print("Temperature - Sensor 3");
        myFile.print(",");
        myFile.print("Dew Point - Sensor 3");
        myFile.println();
        myFile.close();    
      }  
    
      digitalClockDisplaySerial(); // puts data onto the Serial port
      Serial.println();
      Serial.print("Sensor 1 -Humidity (%): ");
      Serial.println(h1, DEC);
      Serial.print("Sensor 1 -Temperature (°C): ");
      Serial.println((float)DHT1.temperature, DEC);
      Serial.print("Sensor 1 -Dew Point (°C): ");
      Serial.println(DHT1.dewPoint(), DEC);
      Serial.print("Sensor 1 -Dew PointFast (°C): ");
      Serial.println(DHT1.dewPointFast(), DEC);
      Serial.print("Sensor 2 -Humidity (%): ");
      Serial.println(h2, DEC);
      Serial.print("Sensor 2 -Temperature (°C): ");
      Serial.println((float)DHT2.temperature, DEC);
      Serial.print("Sensor 2 -Dew Point (°C): ");
      Serial.println(DHT2.dewPoint(), DEC);
      Serial.print("Sensor 2 -Dew PointFast (°C): ");
      Serial.println(DHT2.dewPointFast(), DEC);
      Serial.print("Sensor 3 -Humidity (%): ");
      Serial.println(h3, DEC);
      Serial.print("Sensor 3 -Temperature (°C): ");
      Serial.println((float)DHT3.temperature, DEC);
      Serial.print("Sensor 3 -Dew Point (°C): ");
      Serial.println(DHT3.dewPoint(), DEC);
      Serial.print("Sensor 3 -Dew PointFast (°C): ");
      Serial.println(DHT3.dewPointFast(), DEC);
      Serial.println();
      
      digitalClockDisplaymyFile(); // prints all data to one row in excel
      myFile.print(',');
      myFile.print(h1, DEC);
      myFile.print(",");
      myFile.print((float)DHT1.temperature, DEC);
      myFile.print(",");
      myFile.print(DHT1.dewPoint(), DEC);
      myFile.print(",");
      myFile.print(h2, DEC);
      myFile.print(",");
      myFile.print((float)DHT2.temperature, DEC);
      myFile.print(",");
      myFile.print(DHT2.dewPoint(), DEC);
      myFile.print(",");
      myFile.print(h3, DEC);
      myFile.print(",");
      myFile.print((float)DHT3.temperature, DEC);
      myFile.print(",");
      myFile.print(DHT3.dewPoint(), DEC);
      myFile.println();
      myFile.close();
      delay(9200);
}

I created a loop for the headers by declaring a variable and only executing it once would this be a proper way to put in the header?

Thanks

Topic		Replies	Views
SD data formating Programming	34	2850	July 11, 2021
Layout of columns Programming	6	1155	May 5, 2021
Logging data to a .txt file in columns Storage	4	2035	May 6, 2021
Writing in CSV file on SD Card General Guidance	20	5161	July 17, 2023
writing header to .csv file SD file logging - blum tutorial 11 Programming	5	12081	May 5, 2021

Best way to format data for analysis

Related topics