write to SD card, read and send to pc

Hi all,

I need a help to build a gps data logger using Mega 2560 and Ethernet shield with sd card. So far i already log the gps data to SD card using SD.h and tinyGps.
Below is my code for logging data to SD card

#include <TinyGPS.h>
#include <SD.h>
#include <stdlib.h>
#include <SPI.h>

TinyGPS gps;
static char dtostrfbuffer[20];
int CS = 4;
int LED = 13;


//Define String
String SD_date_time = "invalid";
String SD_lat = "invalid";
String SD_lon = "invalid";
String SD_speed = "invalid";
String SD_heading = "invalid";


static void gpsdump(TinyGPS &gps);
static bool feedgps();
static void print_float(float val, float invalid, int len, int prec, int SD_val);
static void print_int(unsigned long val, unsigned long invalid, int len);
static void print_date(TinyGPS &gps);
static void print_str(const char *str, int len);

void setup()
{

 // stp=1;
  pinMode(CS, OUTPUT);  //Chip Select untuk SD Card
  pinMode(LED, OUTPUT);  //LED Indicator
  pinMode(53, OUTPUT); //SS pin mega 2560
  //Serial interfaces
  Serial.begin(9600);
  Serial3.begin(4800);

  while (!Serial) {
    ; // wait for serial port to connect. Needed for Leonardo only
  }



  //Connect to the SD Card
  if(!SD.begin(CS))
  {
    Serial.println("Card Failure");
    return;
  }
}


void loop()
{
  String dataString ="";
  //int id = 0;
  //switch (stp) {
  //case 1:
      bool newdata = false;
      unsigned long start = millis();

      // Ambil data setiap 2 detik
      while (millis() - start < 2000)
      {
        if (feedgps())
          newdata = true;
      }

      gpsdump(gps);

      //Tulis data ke sd card
      dataString = "CRM001, " + SD_date_time + "," + SD_lat + "," + SD_lon + "," + SD_speed + "," + SD_heading;
      if(SD_date_time != "invalid")
        digitalWrite(LED, HIGH);
      else
        digitalWrite(LED, LOW);
        
      //open file log.txt
      File dataFile = SD.open("LOG.txt", FILE_WRITE); 
      if (dataFile)
      {
        //Serial.print(id);
        //Serial.print(",");
        Serial.println(dataString);

        //pint to sd card
        //dataFile.print(id);
        //dataFile.print(",");
        dataFile.println(dataString);
      }
      
      else
      {
        Serial.println("\nCouldn't open the log file!");
      } 
       dataFile.close(); 

}
static void gpsdump(TinyGPS &gps)
{
  float flat, flon;
  unsigned long age, date, time, chars = 0;
  unsigned short sentences = 0, failed = 0;

  print_int(gps.satellites(), TinyGPS::GPS_INVALID_SATELLITES, 5);
  print_int(gps.hdop(), TinyGPS::GPS_INVALID_HDOP, 5);
  gps.f_get_position(&flat, &flon, &age); 
  print_float(flat, TinyGPS::GPS_INVALID_F_ANGLE, 9, 5, 1); //LATITUDE
  print_float(flon, TinyGPS::GPS_INVALID_F_ANGLE, 10, 5, 2); //LONGITUDE
  print_int(age, TinyGPS::GPS_INVALID_AGE, 5);
  print_date(gps); //DATE AND TIME
  print_float(gps.f_altitude(), TinyGPS::GPS_INVALID_F_ALTITUDE, 8, 2, 0);
  print_float(gps.f_course(), TinyGPS::GPS_INVALID_F_ANGLE, 7, 2, 4);
  print_float(gps.f_speed_kmph(), TinyGPS::GPS_INVALID_F_SPEED, 6, 2, 3);
  print_str(gps.f_course() == TinyGPS::GPS_INVALID_F_ANGLE ? "*** " : TinyGPS::cardinal(gps.f_course()), 6);

}

static void print_int(unsigned long val, unsigned long invalid, int len)
{
  char sz[32];
  if (val == invalid)
    strcpy(sz, "*******");
  else
    sprintf(sz, "%ld", val);
  sz[len] = 0;
  for (int i=strlen(sz); i<len; ++i)
    sz[i] = ' ';
  if (len > 0) 
    sz[len-1] = ' ';
  //Serial.print(sz);
  feedgps();
}


static void print_float(float val, float invalid, int len, int prec, int SD_val)
{
  char sz[32];
  if (val == invalid)
  {
    strcpy(sz, "*******");
    sz[len] = 0;
    if (len > 0) 
      sz[len-1] = ' ';
    for (int i=7; i<len; ++i)
      sz[i] = ' ';
    //Serial.print(sz);
    if(SD_val == 1) SD_lat = sz;
    else if(SD_val == 2) SD_lon = sz;
    else if(SD_val== 3) SD_speed = sz;
    else if(SD_val== 4) SD_heading = sz;
  }
  else
  {
    //Serial.print(val, prec);
    if (SD_val == 1) SD_lat = dtostrf(val,10,5,dtostrfbuffer);
    else if (SD_val == 2) SD_lon = dtostrf(val,10,5,dtostrfbuffer);
    else if (SD_val == 3) SD_speed = dtostrf(val,10,2,dtostrfbuffer);
    else if (SD_val == 4) SD_heading = dtostrf(val,10,2,dtostrfbuffer);
    int vi = abs((int)val);
    int flen = prec + (val < 0.0 ? 2 : 1);
    flen += vi >= 1000 ? 4 : vi >= 100 ? 3 : vi >= 10 ? 2 : 1;
    for (int i=flen; i<len; ++i)
      Serial.print(" ");
  }
  feedgps();
}

static void print_date(TinyGPS &gps)
{
  int year;
  byte month, day, hour, minute, second, hundredths;
  unsigned long age;
  gps.crack_datetime(&year, &month, &day, &hour, &minute, &second, &hundredths, &age);
  if (age == TinyGPS::GPS_INVALID_AGE)
  {
    //Serial.print("*******    *******    ");
    SD_date_time = "invalid";
  }
  else
  {
    char sz[32];
    sprintf(sz, "%02d-%02d-%02d %02d:%02d:%02d   ",
    year, month, day, hour, minute, second);
    //Serial.print(sz);
    SD_date_time = sz;
  }
  print_int(age, TinyGPS::GPS_INVALID_AGE, 5);
  feedgps();
}

static void print_str(const char *str, int len)
{
  int slen = strlen(str);
  for (int i=0; i<len; ++i)
    //Serial.print(i<slen ? str[i] : ' ');
    feedgps();
}

static bool feedgps()
{
  while (Serial3.available())
  {
    if (gps.encode(Serial3.read()))
      return true;
  }
  return false;
}

Now i want to read the historical and send to mysql database. The program flow is :

  1. Write data to SD card
  2. When ethernet connection is detected, read the sd card and send the data to mysql database.
  3. When ethetnet connection disconnect append the new data to SD card.

When ethernet connection is disconnect, remember the last position sent to database so when there's a connection it will continue send the historical data. Another thing is i need to store the data into separate text file for every day. How can i do that? is there any sample code for such a plan.

Thank you

Is the Arduino with ethernet shield to be a client or a server? If it is a client, it can make GET requests containing data to be stored by the script that it calls. That GET request should contain one record from the SD card. If the data on the SD card is arranged nicely (fixed length records), the last record sent could be stored in EEPROM.

If the Arduino with ethernet shield is a server, then it is the client's responsibility to keep track of the last record received and request only new(er) records. At some point, the server will then respond "No more records", when the latest record has been sent.

The arduino is act as a client and i have a gateway application using php in my local server that handle and store the data sent by arduino to mysql database. So far i already manage arduino and ethernet shield to store the data to SD Card and send to my gateway simultaneously, this condition is used when there's ethernet connection but no more historical data on SD card to be send.

here is the data format on SD card : CRM001,2013-05-26 13:36:48,-6.89231,107.56861,0.46,147.41
The data has different length some 56 chars and some 57 chars, this is due to speed and heading value changes.
I'm still struggling how to read csv file, send to database and mark the last record sent.