formatting serial streams

hi,

I'm working on a project where my uno is connected to a gps device outputting serial sentences at 9600 baud, 8 bits, no parity, it puts out more than a standard nmea sentence. Each sentence from the device has a carriage return at the end of the sentence, but the number of characters is different for each sentence. I can read this data easily to the serial port, but am having difficulty finding a way with SD.h to record the data. With gps projects involving standard nmea, there are many programs using tinyGPS and some heavy formatting to output specific things. I simply need to log exactly what is streaming from my gps to a text file, as far as I can tell, I don't need the buffering or formatting. Can anyone point me in a direction for doing this? Is there a solution as straightforward as the one I'm looking for? Is there some way I could make the arduino recognize the automatic carriage return and record the sentence?

any direction on this would be much appreciated

Is there some way I could make the arduino recognize the automatic carriage return and record the sentence?

Absolutely. The TinyGPS class does that (although I think it uses the * as the end-of-data marker).

Exactly how would depend on your code, which seems to have vanished.

Thanks for the help PaulS.

I have different programs of working code that all program different things
here is some of the jeremy blum and mikal hart codes I've put together.
It works, however, it doesn't give me what I need, which are the other sentences (PMTK and RMC) being passed through the gps. I don't know how to understand the internal workings of tinyGPS. I've read the page about it on arduiana, but as far as I can tell, it only really tells you how to query the gps for specific things like age fix, time, lattitude and longitude. Is there anything more that I can read to better understand how tinyGPS works, so that I can log all the sentences with different character lengths that I hope to log? Is there a specific command that looks for the carriage return and records the sentence so that I can put it in a datastring for logging?

#include <TinyGPS.h>
#include <NewSoftSerial.h>
#include <SD.h>
#include<stdlib.h>


/* This sample code demonstrates the normal use of a TinyGPS object.
   It requires the use of SoftwareSerial, and assumes that you have a
   4800-baud serial GPS device hooked up on pins 3(rx) and 4(tx).
*/

TinyGPS gps;
NewSoftSerial nss(3, 4);

static char dtostrfbuffer[40];
int CS = 8;

String SD_date_time = "";
String SD_lat = "";
String SD_lon = "";
String dataString = "";

static void gpsdump(TinyGPS &gps);
static bool feedgps();
static void print_float(float val, float invalid, int len, int prec, int SD_val);
static void print_int(unsigned long val, unsigned long invalid, int len);
static void print_date(TinyGPS &gps);
static void print_str(const char *str, int len);

void setup()
{
  pinMode(CS, OUTPUT);
  
  Serial.begin(115200);
  nss.begin(9600);
  
  if(!SD.begin(CS))
  {
    Serial.println("Card Failure");
    return;
  }
  
  
  
}

void loop()
{
  bool newdata = false;
  unsigned long start = millis();
  
  // Every second we print an update
  while (millis() - start < 1000)
  {
    if (feedgps())
      newdata = true;
  }
  
  gpsdump(gps);
  
  //write newest info to the card
  dataString =SD_date_time+","+ SD_lat + "N"+ "," + SD_lon + "W";
  //open the data csv file
  File dataFile = SD.open("LOG.csv", FILE_WRITE);
  if (dataFile)
  {dataFile.println(dataString);
  Serial.println(dataString);
  dataFile.close();
  }
  else
  {
    Serial.println("Couldnt open the log file!");
  }
}

static void gpsdump(TinyGPS &gps)
{
  float flat, flon;
  unsigned long age, date, time, chars = 0;
  unsigned short sentences = 0, failed = 0;
  static const float LONDON_LAT = 51.508131, LONDON_LON = -0.128002;
  
  print_int(gps.satellites(), TinyGPS::GPS_INVALID_SATELLITES, 5);
  print_int(gps.hdop(), TinyGPS::GPS_INVALID_HDOP, 5);
  gps.f_get_position(&flat, &flon, &age);
  print_float(flat, TinyGPS::GPS_INVALID_F_ANGLE, 9, 5, 1);
  print_float(flon, TinyGPS::GPS_INVALID_F_ANGLE, 10, 5, 2);
  print_int(age, TinyGPS::GPS_INVALID_AGE, 5);

  print_date(gps);

  print_float(gps.f_altitude(), TinyGPS::GPS_INVALID_F_ALTITUDE, 8, 2, 0);
  print_float(gps.f_course(), TinyGPS::GPS_INVALID_F_ANGLE, 7, 2, 0);
  print_float(gps.f_speed_kmph(), TinyGPS::GPS_INVALID_F_SPEED, 6, 2, 0);
  print_str(gps.f_course() == TinyGPS::GPS_INVALID_F_ANGLE ? "*** " : TinyGPS::cardinal(gps.f_course()), 6);
  print_int(flat == TinyGPS::GPS_INVALID_F_ANGLE ? 0UL : (unsigned long)TinyGPS::distance_between(flat, flon, LONDON_LAT, LONDON_LON) / 1000, 0xFFFFFFFF, 9);
  print_float(flat == TinyGPS::GPS_INVALID_F_ANGLE ? 0.0 : TinyGPS::course_to(flat, flon, 51.508131, -0.128002), TinyGPS::GPS_INVALID_F_ANGLE, 7, 2, 0);
  print_str(flat == TinyGPS::GPS_INVALID_F_ANGLE ? "*** " : TinyGPS::cardinal(TinyGPS::course_to(flat, flon, LONDON_LAT, LONDON_LON)), 6);

  gps.stats(&chars, &sentences, &failed);
  print_int(chars, 0xFFFFFFFF, 6);
  print_int(sentences, 0xFFFFFFFF, 10);
  print_int(failed, 0xFFFFFFFF, 9);
  Serial.println();
}

static void print_int(unsigned long val, unsigned long invalid, int len)
{
  char sz[32];
  if (val == invalid)
    strcpy(sz, "INVALID");
  else
    sprintf(sz, "%ld", val);
  sz[len] = 0;
  for (int i=strlen(sz); i<len; ++i)
    sz[i] = ' ';
  if (len > 0) 
    sz[len-1] = ' ';
  Serial.print(sz);
  feedgps();
}

static void print_float(float val, float invalid, int len, int prec, int SD_val)
{
  char sz[32];
  if (val == invalid)
  {
    strcpy(sz, "invalid");
    sz[len] = 0;
        if (len > 0) 
          sz[len-1] = ' ';
    for (int i=7; i<len; ++i)
        sz[i] = ' ';
    Serial.print(sz);
    if(SD_val==1) SD_lat = sz;
    else if (SD_val == 2) SD_lon =sz;
  }
  else
  {
    Serial.print(val, prec);
    if (SD_val == 1) SD_lat = dtostrf(val,10,5,dtostrfbuffer);
    else if (SD_val ==2) SD_lon =dtostrf(val,10,5,dtostrfbuffer);
    int vi = abs((int)val);
    int flen = prec + (val < 0.0 ? 2 : 1);
    flen += vi >= 1000 ? 4 : vi >= 100 ? 3 : vi >= 10 ? 2 : 1;
    for (int i=flen; i<len; ++i)
      Serial.print(" ");
  }
  feedgps();
}

static void print_date(TinyGPS &gps)
{
  int year;
  byte month, day, hour, minute, second, hundredths;
  unsigned long age;
  gps.crack_datetime(&year, &month, &day, &hour, &minute, &second, &hundredths, &age);
  if (age == TinyGPS::GPS_INVALID_AGE)
  {
    Serial.print("Invalid string    ");
    SD_date_time = "invalid";
  }
  else
  {
    char sz[32];
    sprintf(sz, "%02d/%02d/%02d %02d:%02d:%02d   ",
        month, day, year, hour, minute, second);
    Serial.print(sz);
    SD_date_time = sz;
  }
  print_int(age, TinyGPS::GPS_INVALID_AGE, 5);
  feedgps();
}

static void print_str(const char *str, int len)
{
  int slen = strlen(str);
  for (int i=0; i<len; ++i)
    Serial.print(i<slen ? str[i] : ' ');
  feedgps();
}

static bool feedgps()
{
  while (nss.available())
  {
    if (gps.encode(nss.read()))
      return true;
  }
  return false;
}

Thanks for any help anyone can provide!

If all you want to do is write every character read from the GPS to the SD card, 90% of that code is not needed.

  // Every second we print an update
  while (millis() - start < 1000)
  {
    if (feedgps())
      newdata = true;
  }

You will miss data while the code that follows this while loop does its thing. The feedgps() function is where the software serial port is read.

The feedgps() function returns true when the end of a GPS sentence is read. You'll want to develop a similar function. Call that in a while loop that is not time-dependent.

while(!yourfeedgps())
{
   // Nothing to do
}

When the while loop ends, it is because the end of a sentence arrived. So, log it. The loop() function will then end, and get called again, to begin reading the next sentence.

Look at the TinyGPS::encode() function to see how it collects the data into an array, and how it determines that the end of a sentence has arrived. Create an encode() function of your own that performs the same storing that TinyGPS::encode does, but detects end-of-sentence in your way.

I seem to have trouble with the "feedgps()" function.

With this code

#include <NewSoftSerial.h>
#include <TinyGPS.h>

TinyGPS gps;

static bool feedgps();
unsigned long start = millis();

NewSoftSerial nss(3,4);
void setup()  
{
  Serial.begin(115200);
  Serial.println("Start");
 nss.begin(9600);
}

void loop()                     // run over and over again
{
  bool newdata = false;
  while (millis() - start < 1000) 
     // if((int)Serial1.read() != 128 && (int)Serial1.read() !=0 && (int)Serial1.read() != -1)
      {
        if (feedgps())
          newdata=true;
          while(!feedgps())
        {
          Serial.println(feedgps());
        }
      }
  }

I get the error"'bool feedgps()' used but never defined". Any Ideas as to why I'm getting this error?

As far as I can tell, I have used it the exact same way in other programs with no errors. I'm not sure why it doesn't accept my definition of feedgps()

What is feedgps() and where does it come from?

The first code you posted had it as a function, I don't see it in the last version.


Rob

it is posted both before the setup as a "static bool", and in the main loop it is called.

I have seen the function used in a few different programs, I originally got it form the jeremy blum gps logger tutorial. Much of the code for that is in my original post. I have never seen the function explained, but it seems that the function feeds the incoming serial data to an object so that it can then be read or logged.

it is posted both before the setup as a "static bool", and in the main loop it is called.

Yes but where is the code for that function? In the last code the function doesn't exist so unless it's in one of the libraries you are going to get an error.


Rob

Rob,

I wrote some different code. I understand much better how this code works, because I relied less on tutorials to write it.

#include <NewSoftSerial.h>
#include <SD.h>
#include <stdlib.h>
NewSoftSerial nss(3,4);
int CS=8;
static char dtostrfbuffer[40];
String dataString = "";
void setup()  
{
  pinMode(CS, OUTPUT);
  
  Serial.begin(115200);
nss.begin(9600);
  Serial.print("Start");
if(!SD.begin(CS))
{
  Serial.println("Card Failure");
  return;
}
}

void loop()                     // run over and over again
{
  File dataFile =SD.open("LOG.txt", FILE_WRITE);
  if (dataFile)
  {dataFile.print((char*)nss.read());
   dataFile.close();
  }
  else
  {
    Serial.println("Couldn't open the log file!");
  }
for (int i=0;i<2200;i++){
  if (nss.available()) {
     // if((int)Serial1.read() != 128 && (int)Serial1.read() !=0 && (int)Serial1.read() != -1)
      {
       
        Serial.print((char*)nss.read());
        
       

      }
  }
}
delay (5000);
}

When I use the variable "char" instead of "char*" only the first character after the loop is logged. When I use "char*" the serial output and what is logged are both gibberish. Any idea how I can print the full sentences? I don't care so much where it cuts off so long as I can get short bursts (a few lines) of data every few seconds. When using "char" the serial output is exactly what I'm going for but I don't know how to log what I see in the screen. Do you have any ideas for how I can do this?

if possible I would like to keep stop in serial stream so that I can use the break in serial streaming to log some sensor data to a different file

Is this the same project, what happened to tinyGPS and feedgps()?

nss.read() returns a char, not a pointer to a char, so "char" is the correct cast although I doubt it's needed at all.

I don't know what the for loop is for and the code opens the file 1000s of times a second regardless of the existence of a character then writes the results of nss.read() (which will be garbage most of the time) into the file.

I've not used the SD library and I don't know how fast it writes to a card but I would think it can keep up with 9600bps, if not then you need to buffer the characters.

Assuming that's not required I think the following is closer to the mark.

#include <NewSoftSerial.h>
#include <SD.h>
#include <stdlib.h>
NewSoftSerial nss(3,4);
const int CS=8;
static char dtostrfbuffer[40];
String dataString = "";

void setup() {
  pinMode(CS, OUTPUT);
  
  Serial.begin(115200);
  nss.begin(9600);
  Serial.print("Start");
  if(!SD.begin(CS))  {
    Serial.println("Card Failure");
    while(1);
  }

}

void loop()  {
  if (nss.available()) {
    File dataFile =SD.open("LOG.txt", FILE_WRITE);
    if (dataFile)  {
      char c = nss.read();
      dataFile.print(c);
      dataFile.close();
      Serial.print (c);
    }  else  {
      Serial.println("Couldn't open the log file!");
      while(1);
    }
  }
}

Note that if there is any form of card failure I enter an endless loop, at this point I see no point in continuing if the card is bad. That may change if the program has other stuff to do that is not dependant on a working card.


Rob

Hi Rob,

the for loop was for timing the data of the output so that I could get readings in short bursts, then pause to take data from some sensors and record it in a different file(for simplicity that code isn't in there b/c it's already working). Yesterday when I was experimenting with some code, I tried something similar to what you have provided.

The output is something like this for yours

,Startb??)©IÉ I)éI9'KLL???,,2.41,1.09,2.150B
$GPGSV,3,1,09,17,4,21S10F,09$L0,,00.8,0C.,4,6,201,
,B02728P,*,B,100628030.$G82,K0$G0,3021,9T-8

You can see that characters are being skipped, but the data coming in is being parsed nicely. What is recorded above is at least 3 sentences, as indicated by the "$". Surely there is a solution, and this isn't an uncommon problem. Do you have any ideas?

Rob,

It is the same project. I couldn't figure out how to use that library, or that function effectively for my purposes, so I rewrote it without them

I assume the first line is crap caused by not being in sync with the GPS.

The next three sentences are good?

I simply need to log exactly what is streaming from my gps to a text file,

So aren't we done? Just write those characters to the SD. Or you you want to process the sentences?


Rob