Logging data from GPS and other sensors

Hi, im having trouble logging data from my setup. I have an arduino uno with a ladyada logger shield and a sparkfun gps shield with a EM-406 module. I also have a ADXL335 accelerometer and two DS18B20 temperature sensors hooked up to the arduino. My idea was to have the arduino collect Accelerometer data , then temperature and then GPS and log it all to a CSV file and align it all in one line. I got it to mostly do it aside from the GPS. All sensors report and log data just fine but the GPS doesnt always do, actually more often than not it wont log a thing.

My feeling is that im asking the gps for data at the wrong time. If i run the gps on its own it reports at about 1hz with valid information (for simplicity im using the tinygps library for this). however when i upload the code that integrates all the sensors, the gps doesnt always report. adding pauses after the sensors log data seems to sometimes help but the gps data still comes it at irregular intervals. Im using soft serial for the gps at 4800 baud and the rest of the sketch is at 9600 (not sure how to determine what all the other sensors can run at but 9600 seems to work)

Anyway i hope i made my problem clear, basically gps doesnt work when other sensors are part of the sketch but works perfectly on its own. Seems to be a timing problem but im not really sure how to approach it. Any nudges in the right direction are appreciated.

Code:

/*---Alan Sanchez 2011---*/

//LIBRARIES
#include <NewSoftSerial.h>     //GPS
#include <TinyGPS.h>            //GPS
#include <OneWire.h>           //DS18B20
#include <DallasTemperature.h> //DS18B20
#include "SdFat.h"             //talk to SD card
#include <Wire.h>              //logger
#include "RTClib.h"            //talk to clock

//GPS
#define RXPIN 2
#define TXPIN 3
#define GPSBAUD 4800 //GPS baud rate
TinyGPS gps; //TinyGPS object
NewSoftSerial uart_gps(RXPIN, TXPIN); //Initialize library to RX/TX pins
void logGPS(TinyGPS &gps);

//DS18B20
#define ONE_WIRE_BUS 4 //Data wire D4
#define TEMPERATURE_PRECISION 12
OneWire oneWire(ONE_WIRE_BUS); //Setup a oneWire instance to communicate with any OneWire devices.
DallasTemperature sensors(&oneWire); //Pass oneWire reference to Dallas Temperature. 
DeviceAddress Sensor00, Sensor01; // arrays to hold device addresses

//ADXL335
int x, y, z; 

//Logger
#define LOG_INTERVAL  1000 // mills between entries
#define ECHO_TO_SERIAL   0 // echo data to serial port
#define WAIT_TO_START    0 // Wait for serial input in setup()
#define SYNC_INTERVAL 2000 // mills between calls to sync()
uint32_t syncTime = 0;     // time of last sync()
RTC_DS1307 RTC; //define the Real Time Clock object
//The objects to talk to the SD card
Sd2Card card;
SdVolume volume;
SdFile root;
SdFile file;

void setup(){
      Serial.begin(115200);
      uart_gps.begin(GPSBAUD);
      sensors.begin(); //Start DS18B20 library
      
  
      // initialize the SD card
        if (!card.init()) error("card.init"); 
      // initialize a FAT volume
        if (!volume.init(card)) error("volume.init"); 
      // open root directory
        if (!root.openRoot(volume)) error("openRoot");
      // create a new file
      char name[] = "LOGGER00.CSV";
          for (uint8_t i = 0; i < 100; i++) {
          name[6] = i/10 + '0';
          name[7] = i%10 + '0';
          if (file.open(root, name, O_CREAT | O_EXCL | O_WRITE)) break;
          }
          if (!file.isOpen()) error ("file.create");
              Serial.print("Logging to: ");
              Serial.println(name);
  
       file.writeError = 0; // clear print error
  
       Wire.begin();  
       if (!RTC.begin()) {
           file.println("RTC failed");
       #if ECHO_TO_SERIAL
           Serial.println("RTC failed");
       #endif  //ECHO_TO_SERIAL
       }
       
       //&&&&&&&&&&&&&&&&//FILE HEADER//&&&&&&&&&&&&&&&&&&  
       file.println("REACH_ONE LOGGER"); 
       DateTime now;
       now = RTC.now();
       file.print(" ");
       file.print(now.year(), DEC);
       file.print("/");
       file.print(now.month(), DEC);
       file.print("/");
       file.print(now.day(), DEC);
       file.print(" ");
       file.print(now.hour(), DEC);
       file.print(":");
       file.print(now.minute(), DEC);
       file.print(":");
       file.print(now.second(), DEC);
       file.println(" ");
       file.println(" "); 
       //--------------------------------------------------

       #if ECHO_TO_SERIAL
       Serial.println("REACH_ONE LOGGER");
       Serial.println(" ");
       if (file.writeError || !file.sync()) {
       error("write header");
       }
       #endif
}


void loop(){
  
  //DS18B20
      logTemps();
  //ADXL335
      logAccel();
  //GPS 
      logGPS();    
             
  //EndLine
      file.println("  ");
  //SYNC          
      if (file.writeError) error("write data");
      if ((millis() - syncTime) <  SYNC_INTERVAL) return;
      syncTime = millis();
      if (!file.sync()) error("sync");
}


void logTemps(){
      sensors.requestTemperatures(); // Send the command to get temperatures
      DateTime now;
      file.writeError = 0; // clear print error
      uint32_t m = millis();
      now = RTC.now();// fetch the time
  
// Date/Time Stamp Data    
      file.print(now.year(), DEC);
      file.print("/");
      file.print(now.month(), DEC);
      file.print("/");
      file.print(now.day(), DEC);
      file.print(", ");
      file.print(now.hour(), DEC);
      file.print(":");
      file.print(now.minute(), DEC);
      file.print(":");
      file.print(now.second(), DEC);
      file.print(", "); 
//SENSOR00
      file.print("Temp Sensor00: ");
      file.print(", "); 
      float TSensor00 = sensors.getTempCByIndex(0);
      file.print(TSensor00);
      file.print(", "); 
      file.print("C"); 
      file.print(", "); 
      file.print(DallasTemperature::toFahrenheit(TSensor00));
      file.print(", "); 
      file.print("F");
      file.print(", "); 
//SENSOR01
      file.print("Temp Sensor01: "); 
      file.print(", "); 
      float TSensor01 = sensors.getTempCByIndex(1);
      file.print(TSensor01);
      file.print(", "); 
      file.print("C"); 
      file.print(", "); 
      file.print(DallasTemperature::toFahrenheit(TSensor01));
      file.print(", "); 
      file.print("F");
      file.print(", "); 
      
      Serial.println ("logged date & temp");
}
      
void logAccel(){
      x = analogRead(1);       // read analog input pin 0
      y = analogRead(2);       // read analog input pin 1
      z = analogRead(3);       // read analog input pin 2
  
      x = (x*10-3365)/0.675;
      y = (y*10-3335)/0.675;
      z = (z*10-3370)/0.67;
  
      file.print("Acceleretations x y z: ");
      file.print(", ");       
      file.print(x, DEC);    // print the acceleration in the X axis
      file.print(", ");       
      file.print(y, DEC);    // print the acceleration in the Y axis
      file.print(", ");       // prints a space between the numbers
      file.print(z, DEC);  // print the acceleration in the Z axis
      file.print(", "); 
      
      Serial.println("logged accelerations");
}

void logGPS(){
while(uart_gps.available()){  
      long c = uart_gps.read();   // load the data into a variable
            if(gps.encode(c)){   // if there is a new valid sentence
                //logGPS(gps);     // then log the data.

 
  //DATE & TIME
      int year;
      byte month, day, hour, minute, second, hundredths;
      gps.crack_datetime(&year,&month,&day,&hour,&minute,&second,&hundredths);
      Serial.print(month, DEC); 
      Serial.print("/"); 
      Serial.print(day, DEC);
      Serial.print("/");
      Serial.print(year);
          Serial.print("   "); 
      Serial.print(hour, DEC);
      Serial.print(":"); 
      Serial.print(minute, DEC); 
      Serial.print(":"); 
      Serial.print(second, DEC); 
      Serial.print(".");
      Serial.print(hundredths, DEC);
          Serial.print("    ");
  //POSITION LAT&LONG
      float latitude, longitude;
      gps.f_get_position(&latitude, &longitude);
      Serial.print("Lat/Long: "); 
          Serial.print("  "); 
      Serial.print(latitude,5); 
          Serial.print("  "); 
      Serial.print(longitude,5);
          Serial.print("    "); 
  //ALTITUDE,COURSE,SPEED   
      Serial.print("Altitude (meters): "); 
          Serial.print("  ");
      Serial.print(gps.f_altitude());
          Serial.print("    ");  
      Serial.print("Course (degrees): "); 
          Serial.print("  ");
      Serial.print(gps.f_course());
          Serial.print("    "); 
      Serial.print("Speed (kmph): ");
          Serial.print("  ");     
      Serial.print(gps.f_speed_kmph());
          Serial.print(" ; ");   
        Serial.println("  ");
      //Serial.println("logged GPS data");   
            }
}
}

void error(char *str) {  //function to print out errors throughout code
      Serial.print("error: ");
      Serial.println(str);
      while(1);
      }

Alan:

I have seen a few complaints about similar matters as I cruise the various fori...

In particular, check for posts by "fat16" who has commented on the SD card and data collection rates.

I think the bottom line is that the Arduino is "too slow" to do all of these things at one time as the hardware and (non)-OS is architected and organized. It should not be so but apparently it is so. Maybe we have to push for a Uno/Mega2560 with a 32MHz clock rate -- that might do it. I have the Mega 2560 and find generally that it is a fantastic little system for testing the sensors and "getting things to work" and doing all the trials. So I would happily buy this unit again and will buy more -- so please keep the comments civil everyone! Keep this a "how can we solve this" session. :)

I have written a UDP logger in Delphi to communicate with my Arduino Mega for collecting low frequency vibration. I need a collection rate of at least double the highest frequency I wish to collect. Most of the time I am interested in 50Hz and below -- so I need X,Y and Z readings -- a minmum of 3X100 or 300 readings a second. I could do that only by minimizing the use of serial and delays (millis instead) and so -- [u]and clean code[/u] -- [u]like yours![/u]

Here is my $0.02 worth on a (set of)? possible decisions on how to implement data collection...

Here is what is worth a look (imo)

Looking at your code it is obvious that you are giving all sensors equal priority in the collection scheme. The arduino does encourage this by making such code easy to write. I think this is wrong (I have done this too till now.)

Temperature should be collected on a much slower basis unless you have a reason (moving vehicle?)

Why the real time clock? Does not the GPS give much more accurate time? The you could eliminate the code for one sensor? Less code is usually faster.

If you do need three sensors what should have the highest priority? The GPS or the Clock? If you are moving slowly then the clock....

The DS1307 RTC clock does a Square wave generator which can provide a timer to an an external interrupt scheme so that readings can be done on a "schedule". It will need to start an internal timer as the 4096Hz clock rate is still too fast. I published an update to the time.h and DS1307 code in "Other Software Development" -- see if that can help in this idea as it "activates" the timers through the time.h library.

So if I count the pulses and divide by 40, there is my timer signal.... If I set up multiple counters maybe I can "stagger" the collection routines to provide a "train" of collection "time windows" -- maybe it would make it worse. I dunno... This is a tried and true technique for getting slow memory to work on a fast processor -- seen it in use for over 40 years....

Maybe we need a "mini-OS" or data collection framework to implement these ideas and others.

Most good schemes and AI systems make good decisions about what to ignore (Humans work like that most of the time I think.) -- so what can you not do and still get the desired results and achieve the goals you have set. Notice that I did not say the [u]same[/u] results -- that would not be the objective. The object would be to deal with the problem of collection by doing as little as possible -- but no less than necessary.

I do not believe that there is a "brute force" fix other than a faster processor. But judicious use of interrupts and "real time" techniques might work.

Because these processors are so darned inexpensive, it may be best to collect with a suite of smaller processors and consolidate readings on the UNO and write them out to the SD card or a UDP server... (In UDP terminology it is the SERVER that receives...)

This is the idea behind the XBEE idea -- but that cost builds up faster than a suite of Arduinos.... so if you don't need RF -- then what do you think my choice would be?

Interested in your thoughts on these ideas...

Looking at your code does not offer any obvious improvement ideas to me. It's pretty clean and simple considering the task. Arduino gurus can offer suggestions I'm sure -- just not us mortals

Just FYI, I have my GPS logger working with EEPROM. It's not as large as sd card but it only requires minimal load on Arduino so it's free to sense other sensors. You also get to think what's important and what's not and compact your data before saving. The lavish amount of space on an sd card corrupts you easily, thinking you're using a general purpose computer but you're using a microcontroller.

Here is my blog post. I have an interactive GPS logger working with LCD and buttons. Take a look:

http://liudr.wordpress.com/2010/12/29/fully-functional-arduino-gps-logger/

WillR: Thanks for the quick and lengthy response. I have tried staggering the data collection and it does work. However your timer idea will work tons better than what i had previously done which was just put a bunch of millis() commands before the data collection. So im going to look into implementing the timers using the RTC, might ask for some help if i get stuck.

This is the beginning of a high altitude balloon setup, hence why i was collecting time/date stamp from both the gps and the RTC. Since gps cuts off at 60k ft i believe i was intending to use the RTC to time stamp after that but while i have gps its nice to get the more accurate data. The reason i was trying to collect data the way shown in the code as opposed to staggering was just for post processing. If everything has its own column its easy to run everything through matlab as opposed to having a chunk of temp data some gps etc.

As far as why its not collecting, I have a version of the code that prints everything to a terminal window as opposed to the card (which is what i had uploaded here at first) and it has the same problems, gps doesnt record. However with delays added between each sensor's collection every once in a while i get some gps data to go in. The problem i feel is that the gps is being sampled at the wrong intervals and the arduino is not spending enough time on it to get a valid nmea sentence.

liudr: great project, loved the interface. This would be good, but for my high altitude balloon project the EEPROM is not big enough. And youre right about the SD card haha a gift and a curse.

Since gps cuts off at 60k ft

Not if you use the right GPS.

http://showcase.netins.net/web/wallio/GPSrcvrsvs60kft.htm

I like the Garmin GPS18LVC.

-j

Alan:

Here is another suggestion…

Scrap the ADXL335 – and have a look here… (He writes good code – I think — see note at end.)
http://www.schwietering.com/jayduino/bma180/

See post by S.C. Jurgen…

I have made some minor changes to the library and will likely make a few more and will post them if they work out…

The BMA180 is about the same price --at least I find so at Sparkfun and a distributor Creaton Inc. here in Canada…

He uses the Interrupt off the BMA180 breakout to trigger the collection. The chip is faster too. You do have to remember to add the wire to the Mega2560 pin 2 (digital) as I discovered though… :roll_eyes:

As an aside, though the ADXL335 is 2g as I recall, it is much less noisy than the MMA7361L which has a rating of 1.5g. I am currently using that chip.

I have yet to plug the data from the BMA180 into the analysis programs I wrote…
ADRSN (any day real soon now…) Like today I hope…

If you understand that library you are well on the way.

Then pull the Clock Library changes I put in the other forum (Other Software Development). Though I wish they would just update the V022 libraries after fixing up my additions and making them over “to standard”.

What I know about C++ could fill a thimble to the 0.01% level…

Hey Alan,

I’m using the similar list of sensors and facing the same problem. GPS is not storing the data. Rest everything is working fine. Differences are, I’m storing everything on different data file in .txt format (one file for one sensor) & I’m not using temperature sensor instead I’m using a Mic which records the audio and store it in .wav format. What could be the issue ? Could you please elaborate about how exactly did you stagger data ?

Help Appreciated.