Go Down

Topic: GPS Logger, reading and Writing to and from SD (Read 2434 times) previous topic - next topic

Tonyi

After combining two sketches: the first being a GPS SD logger, and the second being, a reader of SD, to variable, then writing back to a SD text file, I've run into an incomplete write.  Each sketch works fine separately.  I think the problem is that I'm running out of dynamic memory: "Sketch uses 24,574 bytes (79%) of program storage space. Maximum is 30,720 bytes.  Global variables use 1,579 bytes (77%) of dynamic memory".  I'm still a novice, so I need some guidance on how to get this sketch to work.
The parameters associated to the SD read and write portion are:  3 TXT files already written to the SD card that are permanently on the card, to copy from, at each startup of the program.  They are named; setting1.txt, setting2.txt, setting3.txt.
There are three files because I could only copy a certain amount of data from each file at a time without getting gibberish and then blank space.  The first file is a copy of a sketch; the address of the website is in the header.
Code: [Select]
//http://www.toptechboy.com/arduino/lesson-25

#include <SD.h> //Load SD card library
#include<SPI.h> //Load SPI Library

#include <Adafruit_GPS.h>    //Install the adafruit GPS library
#include <SoftwareSerial.h> //Load the Software Serial library
SoftwareSerial mySerial(3,2); //Initialize the Software Serial port
Adafruit_GPS GPS(&mySerial); //Create the GPS Object

String NMEA1; //Variable for first NMEA sentence
String NMEA2; //Variable for second NMEA sentence
char c; //to read characters coming from the GPS

int chipSelect = 4; //chipSelect pin for the SD card Reader
File mySensorData; //Data object you will write your sesnor data to


void setup() {
 
  Serial.begin(115200); //Turn on serial monitor
  GPS.begin(9600); //Turn on GPS at 9600 baud
  GPS.sendCommand("$PGCMD,33,0*6D");  //Turn off antenna update nuisance data
  GPS.sendCommand(PMTK_SET_NMEA_OUTPUT_RMCGGA); //Request RMC and GGA Sentences only
  GPS.sendCommand(PMTK_SET_NMEA_UPDATE_1HZ); //Set update rate to 1 hz
  delay(1000);
 
  pinMode(10, OUTPUT); //Must declare 10 an output and reserve it to keep SD card happy
  SD.begin(chipSelect); //Initialize the SD card reader
 
  if (SD.exists("NMEA.txt")) { //Delete old data files to start fresh
    SD.remove("NMEA.txt");
  }
  if (SD.exists("GPSData.txt")) { //Delete old data files to start fresh
    SD.remove("GPSData.txt");
  }

}

void loop() {
 
  readGPS();

  if(GPS.fix==1) { //Only save data if we have a fix
  mySensorData = SD.open("NMEA.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(NMEA1); //Write first NMEA to SD card
  mySensorData.println(NMEA2); //Write Second NMEA to SD card
  mySensorData.close();  //Close the file
  mySensorData = SD.open("GPSData.txt", FILE_WRITE);
  mySensorData.print(GPS.latitude,4); //Write measured latitude to file
  mySensorData.print(GPS.lat); //Which hemisphere N or S
  mySensorData.print(",");
  mySensorData.print(GPS.longitude,4); //Write measured longitude to file
  mySensorData.print(GPS.lon); //Which Hemisphere E or W
  mySensorData.print(",");
  mySensorData.println(GPS.altitude);
  mySensorData.close();
  }
 
}

void readGPS() {
 
  clearGPS();
  while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
  NMEA1=GPS.lastNMEA();
 
   while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
  NMEA2=GPS.lastNMEA();
 
  Serial.println(NMEA1);
  Serial.println(NMEA2);
  Serial.println("");
 
}

void clearGPS() {  //Clear old and corrupt data from serial port
  while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
 
  while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
   while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
 
}


The Second was stripped down by me.  The address of the original is in its header:

Code: [Select]
// http://overskill.alexshu.com/saving-loading-settings-on-sd-card-with-arduino/
/*
  SD card connections:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 ** CS - pin 4
 */
 
 #include <SPI.h>
 #include <SD.h>
 File mySensorData;

 void setup()
 {
 pinMode(10, OUTPUT);
 SD.begin(4);
 mySensorData = SD.open("GPSData.txt", FILE_WRITE);
  if (SD.exists("GPSData.txt")) { //Delete old data files to start fresh
    SD.remove("GPSData.txt");
  } 
  mySensorData.close();  //Close the file
  rewriteSD_HeaderPt1();
  rewriteSD_HeaderPt2();
  rewriteSD_HeaderPt3();
  }


 void loop()
 {

 }



void rewriteSD_HeaderPt1()
 {
   
 char character;
 String settingName1;
 String settingValue1;

 mySensorData = SD.open("setting1.txt");
 if (mySensorData) {
 while (mySensorData.available()) {
 character = mySensorData.read();
 while((mySensorData.available()) && (character != '[')){
 character = mySensorData.read();
 }
 character = mySensorData.read();
 while((mySensorData.available()) && (character != ']')){
 settingValue1 = settingValue1 + character;
 character = mySensorData.read();
 
 }
 mySensorData.close();

  mySensorData = SD.open("GPSData.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(settingValue1); //Write first NMEA to SD card
  mySensorData.close();  //Close the file
 settingValue1 = "";
 if(character == ']'){

 // Apply the value to the parameter
 applySetting(settingName1,settingValue1);
 // Reset Strings
 settingName1 = "";
 settingValue1 = "";
 }
 }
 // close the file:
 mySensorData.close();
 }
 }

 void rewriteSD_HeaderPt2()
 {
   
 char character;
 String settingName1;
 String settingValue1;

 mySensorData = SD.open("setting2.txt");
 if (mySensorData) {
 while (mySensorData.available()) {
 character = mySensorData.read();
 while((mySensorData.available()) && (character != '[')){
 character = mySensorData.read();
 }
 character = mySensorData.read();
 while((mySensorData.available()) && (character != ']')){
 settingValue1 = settingValue1 + character;
 character = mySensorData.read();
 
 }
 mySensorData.close();

  mySensorData = SD.open("GPSData.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(settingValue1); //Write first NMEA to SD card
  mySensorData.close();  //Close the file
 settingValue1 = "";
 if(character == ']'){

 // Apply the value to the parameter
 applySetting(settingName1,settingValue1);
 // Reset Strings
 settingName1 = "";
 settingValue1 = "";
 }
 }
 // close the file:
 mySensorData.close();
 }
}

 void rewriteSD_HeaderPt3()
 {
   
 char character;
 String settingName1;
 String settingValue1;

 mySensorData = SD.open("setting3.txt");
 if (mySensorData) {
 while (mySensorData.available()) {
 character = mySensorData.read();
 while((mySensorData.available()) && (character != '[')){
 character = mySensorData.read();
 }
 character = mySensorData.read();
 while((mySensorData.available()) && (character != ']')){
 settingValue1 = settingValue1 + character;
 character = mySensorData.read();
 
 }
 mySensorData.close();

  mySensorData = SD.open("GPSData.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(settingValue1); //Write first NMEA to SD card
  mySensorData.close();  //Close the file
 settingValue1 = "";
 if(character == ']'){

 // Apply the value to the parameter
 applySetting(settingName1,settingValue1);
 // Reset Strings
 settingName1 = "";
 settingValue1 = "";
 }
 }
 // close the file:
 mySensorData.close();
 }
 }


 void applySetting(String settingName1, String settingValue1)
 {
  //Leave empty brackets above and below alone
}
 
 // converting string to Float
 float toFloat(String settingValue1){
 char floatbuf[settingValue1.length()+1];
 settingValue1.toCharArray(floatbuf, sizeof(floatbuf));
 float f = atof(floatbuf);
 return f;
 }
 
 long toLong(String settingValue1){
 char longbuf[settingValue1.length()+1];
 settingValue1.toCharArray(longbuf, sizeof(longbuf));
 long l = atol(longbuf);
 return l;
 }
 
 // Converting String to integer and then to boolean
 // 1 = true
 // 0 = false
 boolean toBoolean(String settingValue1) {
 if(settingValue1.toInt()==1){
 return true;
 } else {
 return false;
 }
 }
 
 




Tonyi



and finally the "not working correctly" union of both:

Code: [Select]
/*
  SD card connections:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 ** CS - pin 4
 */
#include <SD.h> //Load SD card library
#include<SPI.h> //Load SPI Library

#include <Adafruit_GPS.h>    //Install the adafruit GPS library
#include <SoftwareSerial.h> //Load the Software Serial library
SoftwareSerial mySerial(3,2); //Initialize the Software Serial port
Adafruit_GPS GPS(&mySerial); //Create the GPS Object

String NMEA1; //Variable for first NMEA sentence
String NMEA2; //Variable for second NMEA sentence
char c; //to read characters coming from the GPS

int chipSelect = 4; //chipSelect pin for the SD card Reader
File mySensorData; //Data object you will write your sesnor data to


void setup() {
  
 // Serial.begin(115200); //Turn on serial monitor
  GPS.begin(9600); //Turn on GPS at 9600 baud
  GPS.sendCommand("$PGCMD,33,0*6D");  //Turn off antenna update nuisance data
  GPS.sendCommand(PMTK_SET_NMEA_OUTPUT_RMCGGA); //Request RMC and GGA Sentences only
  GPS.sendCommand(PMTK_SET_NMEA_UPDATE_1HZ); //Set update rate to 1 hz
  delay(1000);
  
  pinMode(10, OUTPUT); //Must declare 10 an output and reserve it to keep SD card happy
  SD.begin(chipSelect); //Initialize the SD card reader
  
  if (SD.exists("NMEA.txt")) { //Delete old data files to start fresh
    SD.remove("NMEA.txt");
  }
  if (SD.exists("GPSData.txt")) { //Delete old data files to start fresh
    SD.remove("GPSData.txt");
  }
  delay(99);
  rewriteSD_HeaderPt1();//the text in these 3 lines
  rewriteSD_HeaderPt2();//are only partially printed to
  rewriteSD_HeaderPt3();//GPSData.txt
 
  
}

void loop() {
  
  readGPS();
  if(GPS.fix==1) { //Only save data if we have a fix
  mySensorData = SD.open("NMEA.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(NMEA1); //Write first NMEA to SD card
  mySensorData.println(NMEA2); //Write Second NMEA to SD card
  mySensorData.close();  //Close the file
  mySensorData = SD.open("GPSData.txt", FILE_WRITE);
  mySensorData.print(GPS.latitude,4); //Write measured latitude to file
  mySensorData.print(GPS.lat); //Which hemisphere N or S
  mySensorData.print(",");
  mySensorData.print(GPS.longitude,4); //Write measured longitude to file
  mySensorData.print(GPS.lon); //Which Hemisphere E or W
  mySensorData.print(",");
  mySensorData.println(GPS.altitude);
  mySensorData.close();
  }
  
}

void readGPS() {
  
  clearGPS();
  while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
  NMEA1=GPS.lastNMEA();
  
   while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
  NMEA2=GPS.lastNMEA();
  
 // Serial.println(NMEA1);
 // Serial.println(NMEA2);
 // Serial.println("");
  
}

void clearGPS() {  //Clear old and corrupt data from serial port
  while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
  
  while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
   while(!GPS.newNMEAreceived()) { //Loop until you have a good NMEA sentence
    c=GPS.read();
  }
  GPS.parse(GPS.lastNMEA()); //Parse that last good NMEA sentence
  
}

void rewriteSD_HeaderPt1()
 {
    
 char character;
 String settingName1;
 String settingValue1;

 mySensorData = SD.open("setting1.txt");
 if (mySensorData) {
 while (mySensorData.available()) {
 character = mySensorData.read();
 while((mySensorData.available()) && (character != '[')){
 character = mySensorData.read();
 }
 character = mySensorData.read();
 while((mySensorData.available()) && (character != ']')){
 settingValue1 = settingValue1 + character;
 character = mySensorData.read();
 
 }
 mySensorData.close();

  mySensorData = SD.open("GPSData.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(settingValue1); //Write first NMEA to SD card
  mySensorData.close();  //Close the file
 settingValue1 = "";
 if(character == ']'){

 // Apply the value to the parameter
 applySetting(settingName1,settingValue1);
 // Reset Strings
 settingName1 = "";
 settingValue1 = "";
 }
 }
 // close the file:
 mySensorData.close();
 }
 }

 void rewriteSD_HeaderPt2()
 {
    
 char character;
 String settingName1;
 String settingValue1;

 mySensorData = SD.open("setting2.txt");
 if (mySensorData) {
 while (mySensorData.available()) {
 character = mySensorData.read();
 while((mySensorData.available()) && (character != '[')){
 character = mySensorData.read();
 }
 character = mySensorData.read();
 while((mySensorData.available()) && (character != ']')){
 settingValue1 = settingValue1 + character;
 character = mySensorData.read();
 
 }
 mySensorData.close();

  mySensorData = SD.open("GPSData.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(settingValue1); //Write first NMEA to SD card
  mySensorData.close();  //Close the file
 settingValue1 = "";
 if(character == ']'){

 // Apply the value to the parameter
 applySetting(settingName1,settingValue1);
 // Reset Strings
 settingName1 = "";
 settingValue1 = "";
 }
 }
 // close the file:
 mySensorData.close();
 }
}

 void rewriteSD_HeaderPt3()
 {
    
 char character;
 String settingName1;
 String settingValue1;

 mySensorData = SD.open("setting3.txt");
 if (mySensorData) {
 while (mySensorData.available()) {
 character = mySensorData.read();
 while((mySensorData.available()) && (character != '[')){
 character = mySensorData.read();
 }
 character = mySensorData.read();
 while((mySensorData.available()) && (character != ']')){
 settingValue1 = settingValue1 + character;
 character = mySensorData.read();
 
 }
 mySensorData.close();

  mySensorData = SD.open("GPSData.txt", FILE_WRITE); //Open file on SD card for writing
  mySensorData.println(settingValue1); //Write first NMEA to SD card
  mySensorData.close();  //Close the file
 settingValue1 = "";
 if(character == ']'){

 // Apply the value to the parameter
 applySetting(settingName1,settingValue1);
 // Reset Strings
 settingName1 = "";
 settingValue1 = "";
 }
 }
 // close the file:
 mySensorData.close();
 }
 }


 void applySetting(String settingName1, String settingValue1)
 {
  //Leave empty brackets above and below alone
}
 
 // converting string to Float
 float toFloat(String settingValue1){
 char floatbuf[settingValue1.length()+1];
 settingValue1.toCharArray(floatbuf, sizeof(floatbuf));
 float f = atof(floatbuf);
 return f;
 }
 
 long toLong(String settingValue1){
 char longbuf[settingValue1.length()+1];
 settingValue1.toCharArray(longbuf, sizeof(longbuf));
 long l = atol(longbuf);
 return l;
 }
 
 // Converting String to integer and then to boolean
 // 1 = true
 // 0 = false
 boolean toBoolean(String settingValue1) {
 if(settingValue1.toInt()==1){
 return true;
 } else {
 return false;
 }
 }


I've spent many hours stripping the originals, and re-organizing the second two sketches into one and would appreciate any help or suggestions on how to make it function correctly.

-dev

You can just attach files to your post, if they're too big.  See the "Attachments and other options" in the lower-left corner of the post editor.

Major changes you should make:

*   Don't use the String class.  It adds 1600 bytes to the program size and can cause memory corruption at random times.  Just use char arrays (aka "C strings").

*   You are using the old SD library.  SDFat is the current version.  It has many bug fixes and improvements.

*   Use a reliable GPS library.  The Adafruit_GPS library does not verify the NMEA checksum, so it can allow invalid data to be accepted.  My NeoGPS is smaller, faster and more accurate than all other libraries.

*   Don't use delay.  The GPS device transmits characters in batches, with a "quiet time" between batches.  You must coordinate time-consuming operations (e.g., SD write) with that cycle.

*   Don't use SoftwareSerial.  It disables interrupts for long periods of time.  This will interfere with your sketch or with other libraries.  In your case, it disables interrupts for 1ms for every GPS character received.  Combined with your use of delay, most of the CPU time is spent waiting for received bits or delays.  :(  See sketch below for alternatives.

*   Don't open and close the file for every GPS update.  Instead, open the file in setup, write each GPS update, flush occasionally, and close the file when something happens (e.g., a button push, a certain amount of time elapses, etc.).  You can flush after every update, but this will slow the sketch down and reduce the life of the SD card.  In your current sketch, you are not appending the new GPS data, you are re-creating the file every time.  It will have only one GPS record.

Recommendations:

*   Don't save the raw NMEA data.  This is 5 to 20 times the size of the parsed fix information.  Many NMEA sentences contain duplicate information, so this takes 5 to 20 times longer to write.

*   Indent your code.  This helps with readability, which could be the single most important thing to help us understand your sketch, or for you to debug your sketch.  Just press control-T in the IDE editor to auto-format your code.  This would have helped you with rewriteSD_Header functions.  What a mess!

*   Don't use obvious comments.  // Recommend useful comments.  -_-

*   Tell us which GPS you are using.  Most GPS devices use 3.3V, not the 5V that most Arduinos use.  To avoid damaging your GPS, you should "level-shift" the RX/TX pins with a few discrete parts or a level-shifting module.  Many vendors say "5V compatible" because the GPS VCC takes 5V, but the RX/TX pins may specify 3.3V.

Here is a sketch that incorporates all these changes:

Code: [Select]
/*
  SD card connections:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 ** CS - pin 4
 */

int chipSelect = 4;

#include <SdFat.h>
#include <SPI.h>

SdFat SD;
File  gpsDataFile; // Data object you will write your sensor data to
const char *gpsFilename = "GPSData.txt";

//---------------------
//  Pick a serial port for the GPS

// BEST choice is a HardwareSerial port:
//    Use Serial1 on a Mega, Leo or Due board
//    You could use Serial on any board (disconnect GPS
//       when uploading over USB).
//#define gpsPort Serial1

// 2nd best:
#include <AltSoftSerial.h>
AltSoftSerial gpsPort; // must be on specific pins (8 & 9 for an UNO)

// 3rd best: must be baud rate 9600, 19200 or 38400
//#include <NeoSWSerial.h>
//NeoSWSerial gpsPort(3,2);

// Worst: SoftwareSerial NOT RECOMMENDED
//---------------------

#include <NMEAGPS.h>
NMEAGPS GPS;
gps_fix fix;


void setup()
{
 // Serial.begin(115200);
  gpsPort.begin( 9600 );
 
  // Configure the GPS device
  GPS.send_P( &gpsPort, F("PGCMD,33,0") );  //Turn off antenna update nuisance data
  GPS.send_P( &gpsPort, F("PMTK314,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0") ); //Request RMC and GGA Sentences only
  GPS.send_P( &gpsPort, F("PMTK220,1000") ); //Set update rate to 1 hz

  pinMode(10, OUTPUT); // reserve it to keep SD card happy
  if (!SD.begin(chipSelect)) {
    // Serial.println( F("No SD card/reader found!") );
    // Do something?  blink the LED? Beep?
    // for (;;); // hang here!
  }

  if (SD.exists(gpsFilename)) { //Delete old data files to start fresh
    SD.remove(gpsFilename);
  }

  gpsDataFile = SD.open(gpsFilename, FILE_WRITE);

  //  These routines were identical, except for the filename.  Just
  //     use one routine and pass in the filename argument.
  rewriteSD_Header( "setting1.txt" );
  rewriteSD_Header( "setting2.txt" );
  rewriteSD_Header( "setting3.txt" );

}


void loop()
{
  // Read characters and parse into a fix structure
  if (GPS.available( gpsPort )) {

    // Get the new fix data (GPS quiet time just started)
    fix = GPS.read();

    // Only save data if we have a fix
    if (fix.valid.status &&
        (fix.status >= gps_fix::STATUS_STD)) {

      gpsDataFile.print( fix.latitude(), 5 );
      gpsDataFile.print( ',' );
      gpsDataFile.print( fix.longitude(), 5);
      gpsDataFile.print( ',' );
      gpsDataFile.println( fix.altitude() );
      gpsDataFile.flush(); // this works, but it slows things down
    }
  }
 
  // if (button pressed) {
  //   gpsDataFile.close();
  //   while (true) { // forever
  //     blink an LED?  Beep?  LED off?
  //   }
  // }

} // loop


bool skipTo( char c, File &file )
{
  while (gpsDataFile.available()) {
    if (gpsDataFile.read() == c)
      return true;
  }
  return false;
}

void rewriteSD_Header( const char *filename )
{
  File settingsFile = SD.open( filename );
  if (settingsFile.isOpen()) {

    if (skipTo( '[', settingsFile )) {

      // Get the name
      const  size_t MAX_NAME_LEN = 32;
      char   name[ MAX_NAME_LEN ];
      size_t nameLen = settingsFile.readBytesUntil( '=', name, sizeof(name)-1 );

      if (nameLen > 0) {
        name[ nameLen ] = '\0'; // NUL-terminate the C string (NOT String!)

        //  Now get the value
        const  size_t MAX_VALUE_LEN = 32;
        char   value[ MAX_VALUE_LEN ];
        size_t valueLen = settingsFile.readBytesUntil( ']', value, sizeof(value)-1 );

        value[ valueLen ] = '\0';

        applySetting( name, value ); // Do I really have to say "Apply the setting"?!?
       
        // If you really want to copy the settings into the gpsDataFile,
        //    you could write them here:
        gpsDataFile.print( '[' );
        gpsDataFile.print( name );
        gpsDataFile.print( '=' );
        gpsDataFile.print( value );
        gpsDataFile.println( ']' );
      }
    }

    settingsFile.close();
  }
}

float toFloat( char *value )
{
  return atof( value );
}

long toLong( char *value )
{
 return atol( value );
}

// 1 = true
// 0 = false
boolean toBoolean( char *value )
{
  if (atoi( value ) == 1) {
    return true;
  } else {
    return false;
  }
}

// Some variables that can be configured by the SD settings files
float foo;
long  bar;
bool  snafu;


void applySetting( char *name, char *value )
{
  // If the name is one of the variables that can be configured,
  //    convert the value C string to the type of that variable.

  if (strcmp( name, "foo" ) == 0) {
    foo = toFloat( value );

  } else if (strcmp( name, "bar" ) == 0) {
    bar = toLong( value );

  } else if (strcmp_P( name, (const char *) F("snafu") ) == 0) {
    // The strcmp_P variant uses less RAM, because the second
    //   argument is in FLASH memory, not RAM.  Use the F macro
    //   plus a type cast, as required by the definition of strcmp_P.
    //   You can replace the above strcmp calls in the same way,
    //   saving the RAM for the C strings "foo" and "bar".
    snafu = toBoolean( value );
  }
}

Your original program uses 24466 byes of program space and 1579 bytes of RAM.

The NeoGPS program uses 18884 bytes of program space and 1027 bytes of RAM.  This saves 5600 bytes of program space and 550 bytes of RAM, a huge difference.

If you want to try it, NeoGPS is available from the Arduino IDE Library Manager, under the menu Sketch -> Include Library -> Manage Libraries.

Cheers,
/dev

Tonyi

Thanks for the highly informative reply /dev!  Now you have givin' me a full plate of data to study up on, and try out.  To answer one of your questions, the unit is a Chinese VK2828U7G5LF GPS Module and runs from 3.3~5v.  I'm glad I ordered 2 of them, because I was running the code, in my first example, and the unit would not start spitting out GPS data. So, I messed with the wires on the breadboard and it finally started (green LED blinking) which means its connected and working.  
When I read the data off the SD card I noticed the parsed coordinates where way off, and confirmed that by creating a *.kml file and loading it into google maps.  It was at a single location, in the Artic circle. LOL  I only mention this because I will head your advice and use try using 3.3v to power it.
The plan is to use this module to record GPS and barometric information, using a pressure sensor, which I have received yet using a dirigible.  The reason I had the 3 txt files load in "void setup" is to print out a header, used in google maps, onto the SD card, as not to have to do it manually each time I use the GPS unit.  I was unable copy and paste the entire text, perhaps due to my limited knowledge, so I was forced to divide the header into 3 *.txt files.  I was also planning to use a button press, to load a footer at the end of the GPS read, to write the footer, of the google maps code.
I will upload the three txt files I use for the header code, if I can find the upload button!

Tonyi

I was using the "quick reply", in my previous post, and that editor doesn't include the "Attachments and other options" option.  As you can probably tell, I have no formal training in coding, or know anybody that codes, so being said, I have a question and most like more later.  Is the reason that I could only copy so little amounts of a text file due to the use of "String" instead of char arrays?  I ask because it would be nicer to use 1 txt file for the header instead of three.   I've attached the three txt files ...

-dev

Quote from: Tonyi
the unit would not start spitting out GPS data. So, I messed with the wires on the breadboard and it finally started (green LED blinking) which means its connected and working.  
When I read the data off the SD card I noticed the parsed coordinates where way off, and confirmed that by creating a *.kml file and loading it into google maps.  It was at a single location, in the Arctic circle.
When the GPS device first powers up, it may not know anything at all, not even the correct time.  This makes it take longer to get the "first fix".  It can easily take 10 minutes or longer, depending on the quality of the satellite reception.  You will almost certainly need to be near a window, or maybe outside.

If you are talking about data saved by your old sketch, then it may have saved an invalid location.  NeoGPS is very careful to remember which pieces of GPS information are valid.  It has a flag for each piece.  In your sketch, you had this test:

    if (GPS.fix == 1)

With NeoGPS, you can test whether the fix status is valid:

    // Only save data if we have a fix
    if (fix.valid.status &&
        (fix.status >= gps_fix::STATUS_STD)) {

If the status is valid, then its value can be used.  In this case it is compared to the enumerated values for a status:

      enum status_t {
        STATUS_NONE,
        STATUS_EST,
        STATUS_TIME_ONLY,
        STATUS_STD,
        STATUS_DGPS
      };

Now you can see that the fix status is not just a 0 or a 1.  The best fix quality is "differential", when the GPS device is receiving many satellites with good signal quality.  In the sketch I provided, it will not write a record when the status is less than STD, which implies that a 3D fix has been computed (lat, lon and alt).

The same is true of each piece in the NeoGPS fix structure.  Generally, you should test the valid flag before using a value.

Quote
The reason I had the 3 txt files load in "void setup" is to print out a header, used in google maps, onto the SD card, as not to have to do it manually each time I use the GPS unit.
They can all go in one file, and I wouldn't bother with the bracket delimiters:

Code: (header.txt) [Select]
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
  <Style id="yellowPoly">
    <LineStyle>
      <color>7f00ffff</color>
      <width>4</width>
    </LineStyle>
    <PolyStyle>
      <color>7f00ff00</color>
    </PolyStyle>
  </Style>
  <Placemark><styleUrl>#yellowPoly</styleUrl>
    <LineString>
      <extrude>1</extrude>
      <tesselate>1</tesselate>
      <altitudeMode>absolute</altitudeMode>
      <coordinates>

With indentation, you can see the structure of the XML header.

BTW, you can click the "select" above to highlight all the XML in that "code" block.  Then press ctrl-C to copy it to your clipboard.  Open a text editor window and press ctrl-V to paste it into the editor.  Then select the menu File -> Save As to save it as your own "header.txt" file.  Copy that file to the SD card, and you're in business.

And now that we know more about what you're trying to do, the sketch can be even smaller:

Code: [Select]
/*
  SD card connections:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 ** CS - pin 4
 */

int chipSelect = 4;

#include <SdFat.h>
#include <SPI.h>

SdFat SD;
File  gpsDataFile; // Data object you will write your sensor data to
const char *gpsFilename = "GPSData.txt";

//---------------------
//  Pick a serial port for the GPS

// BEST choice is a HardwareSerial port:
//    Use Serial1 on a Mega, Leo or Due board
//    You could use Serial on any board (disconnect GPS
//       when uploading over USB).
//#define gpsPort Serial1

// 2nd best:
#include <AltSoftSerial.h>
AltSoftSerial gpsPort; // must be on specific pins (8 & 9 for an UNO)

// 3rd best: must be baud rate 9600, 19200 or 38400
//#include <NeoSWSerial.h>
//NeoSWSerial gpsPort(3,2);

// Worst: SoftwareSerial NOT RECOMMENDED
//---------------------

#include <NMEAGPS.h>
NMEAGPS GPS;
gps_fix fix;


void setup()
{
 // Serial.begin(115200);
  gpsPort.begin( 9600 );
 
  // Configure the GPS device
  GPS.send_P( &gpsPort, F("PGCMD,33,0") );  //Turn off antenna update nuisance data
  GPS.send_P( &gpsPort, F("PMTK314,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0") ); //Request RMC and GGA Sentences only
  GPS.send_P( &gpsPort, F("PMTK220,1000") ); //Set update rate to 1 hz

  pinMode(10, OUTPUT); // reserve it to keep SD card happy
  if (!SD.begin(chipSelect)) {
    Serial.println( F("No SD card/reader found!") );
    for (;;); // hang here!
  }

  if (SD.exists(gpsFilename)) { //Delete old data files to start fresh
    SD.remove(gpsFilename);
  }

  gpsDataFile = SD.open(gpsFilename, FILE_WRITE);

  copyFile( "header.txt" );
}


void loop()
{
  // Read characters and parse into a fix structure
  if (GPS.available( gpsPort )) {

    // Get the new fix data (GPS quiet time just started)
    fix = GPS.read();

    // Only save data if we have a fix
    if (fix.valid.status &&
        (fix.status >= gps_fix::STATUS_STD)) {

      gpsDataFile.print( fix.latitude(), 5 );
      gpsDataFile.print( ',' );
      gpsDataFile.print( fix.longitude(), 5);
      gpsDataFile.print( ',' );
      gpsDataFile.println( fix.altitude() );
      gpsDataFile.flush(); // this works, but it slows things down
    }
  }
 
  // if (button pressed) {
  //   copyFile( "trailer.txt" );  // copy the rest of the XML stuff
  //   gpsDataFile.close();
  //   while (true) { // forever
  //     blink an LED?  Beep?  LED off?
  //   }
  // }

} // loop


void copyFile( const char *filename )
{
  File file = SD.open( filename );
  if (file.isOpen()) {

    uint8_t buffer[ 256 ];
    for (;;) {
      int numRead = file.read( &buffer[0], sizeof(buffer) );
      if (numRead == 0)
        break;
      gpsDataFile.write( buffer, numRead );
    }

    file.close();
  }
}

Keep talking!  Maybe we can get this down to 3 lines of code...  ;)

Tonyi

Just a quick note to thank you for continued help.  Also, after tinkering with my GPS unit, I decided to try uploading a different sketch and the GPS is now again working fine; I accidently had uploaded one of my failed sketches....  I also tried using the 3.3v output on the Nano to power the GPS and it would not light up/ turn on.  BTW, this unit does pick up the satellites inside, and rather quickly, good deal for a $8 unit.
I was copy and pasting the header, for the google maps, after the fact, as you suggested, when I came up with the idea of using the SD disk, and have a finished document, that was ready for loading after each GPS use.  And those ideas, always seem to cause problems.  This leads me to another question.   Can files be saved, to the SD card in any other extension other than *.TXT?   I ask because I tried saving directly as a *.kml file, on the SD card and that SD card became unreadable, not detected, in Windows after that; or perhaps was it just a coincidence that it malfunctioned?

-dev

Quote
Most GPS devices use 3.3V, not the 5V that most Arduinos use.
So, you're using a 3.3V Nano?  No level-shifting of RX/TX is needed, but the GPS module VCC pin probably needs 5V, unless it has a separate 3.3V supply pin.  As you discovered.

Quote
Can files be saved, to the SD card in any other extension other than *.TXT?   I ask because I tried saving directly
Yes, just use a different filename in SD.open.

Quote
...and that SD card became unreadable, not detected, in Windows after that
You need to close the open files before yanking the card out.  That's why many people use a button to stop recording.  It closes all files to prevent corrupting the card.  You will have to reformat the card on Windows before it will work in the Arduino again.

Be sure to post your sketch when you have changed something, because we won't know what code you are trying.

Cheers,
/dev

Tonyi

Quote
❝Most GPS devices use 3.3V, not the 5V that most Arduinos use."

So, you're using a 3.3V Nano?  No level-shifting of RX/TX is needed, but the GPS module VCC pin probably needs 5V, unless it has a separate 3.3V supply pin.  As you discovered..
Not to be a smart ass, but you quoted yourself in this statement.

My Nano is a 5v version, but the pin next to D13 is 3.3v's, and it also has a 5v output and a 6v~20's Vin.
Quote
❝Can files be saved, to the SD card in any other extension other than *.TXT?   I ask because I tried saving directly"

Yes, just use a different filename in SD.open.
If I remember correctly I SD.opened it as a *.txt and then tried saving it as a *.klm, oops, now I know, thanks.  I always powered down, (disconnected the power) before extracting the SD card.  I've tried to use windows, SDFormater and mintool partition Wizard, to repair that broken SD card, but none of them even detect the card as being connected.  Do you have any SD formatting tools, you can suggest to repair undetected SD cards?

After I figured out there was nothing wrong with my GPS module, I tried using the code you supplied.  You will see the options I chose in the attached code below.  The reading and writing from the GPS is working fine but I am having problems getting the header to write to the *.txt file.  As I understood, most likely incorrectly, this is how I attempted to execute your instructions.
Quote
// in void setup()....

 gpsDataFile = SD.open(gpsFilename, FILE_WRITE);

  //  These routines were identical, except for the filename.  Just
  //     use one routine and pass in the filename argument.
  rewriteSD_Header( filename );
  //rewriteSD_Header( "setting2.txt" );
  //rewriteSD_Header( "setting3.txt" );

//*************AND FURTHER DOWN IN THE SKETCH*************

// in void rewriteSD_Header

void rewriteSD_Header( const char *filename = "setting1.txt" )
{
  File settingsFile = SD.open(filename);
  if (settingsFile.isOpen()) {  //........................
And I am pretty sure that is where my errors take place.

-dev

Quote from: Tonyi
Not to be a smart ass, but you quoted yourself in this statement.
LOL, I was being smart.  I was pointing out an assumption I made because you didn't tell us which Arduino you are using.  When you said "3.3v output on the Nano", I thought you meant it was a 3.3V Nano that didn't have a 5V pin.  Which would have meant it was ok to hook the 3.3V GPS RX/TX pins to the 3.3V Nano AltSoftSerial/NeoSWSerial pins.  But No.

Ok, a 5V Nano can usually receive from a 3.3V GPS device.  But the transmit pin from a 5V Nano should not be connected to a 3.3V GPS RX pin.  It can damage the GPS device, so you really should use a resistor divider to shift the transmit voltage down by 1/3.

I also don't know what kind of card reader you're using.  Is it a 5V or 3.3V reader?  Does it have transistors for level-shifting from the 5V Nano levels to the typical 3.3V SD card SPI pins?  If not, you may have damaged the card.  That may be why you can't format it now.

If it does not have on-board level shifting, you will need a level-shifting module, one that uses FET's to quickly and safely shift the voltage levels down (to the card) or up (to the Arduino).

The "How to Use this Forum" page lists all the things you should describe, so that we don't ASSUME anything.  ;)

Quote from: Tonyi
I tried using the code you supplied.
You didn't try the second sketch I supplied in Reply #5, along with the "headers.txt" file contents.  You can call it "headers.xml" if you want and use that in setup:

    copyFile( "header.xml" );

You can also create the GPS data file with an XML extension:

    const char *gpsFilename = "GPSData.xml";

Cheers,
/dev

Tonyi

I've been making a lot of simple mistakes lately and was sure that I was using the INO from post #5, until I noticed there wasn't a "copyFile( "header.xml" );" to be found on the sketch.   Things are working beautifully now!  I added a space between the Header and GPS files, so it is more easily read, in the final  GPSData.txt file. 
All I have left is to implement the footer.txt with a push button then finally try writing the file extension as a *kml file.  I'm not sure but you keep mentioning *.xml, did you misread the extension as I had posted it, or is there a specific reason to use the *xml extension. 
The only addition I made was:

Code: [Select]
      int numRead = file.read( &buffer[0], sizeof(buffer) );
      if (numRead == 0)
        break;
      gpsDataFile.write( buffer, numRead );
     
    }
  gpsDataFile.println(" ");  // added two lines of code, to make line of space,
  gpsDataFile.println(" "); // between the header and GPS data, for readability
    file.close();
  }

-dev

Quote
you keep mentioning *.xml, did you misread the extension as I had posted it, or is there a specific reason to use the *xml extension.
Ah, yes...  It is XML (first line), but more specifically, it is KML (second line).  The KML extension is correct.

Glad it's working!

Tonyi

I was trying to finish the close portion of the header and got stumped with an error while compiling. 
Code: [Select]
exit status 1
'copyFile' was not declared in this scope
 
I know there would be more code to follow but since it failed to compile I  figured, it would be a good time, to stop and let the teacher, show me the proper way, to get this to function.  Here is what I did. 
Code: [Select]
/*
 
/*
  SD card connections:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 ** CS - pin 4
 */

int chipSelect = 4;
const int buttonPin = 5;
int buttonState = 0;   
#include <SdFat.h>
#include <SPI.h>

SdFat SD;
File  gpsDataFile; // Data object you will write your sensor data to
const char *gpsFilename = "GPSData.txt";

//---------------------
//  Pick a serial port for the GPS

// BEST choice is a HardwareSerial port:
//    Use Serial1 on a Mega, Leo or Due board
//    You could use Serial on any board (disconnect GPS
//       when uploading over USB).
//#define gpsPort Serial1

// 2nd best:
//#include <AltSoftSerial.h>
//AltSoftSerial gpsPort; // must be on specific pins (8 & 9 for an UNO)

// 3rd best: must be baud rate 9600, 19200 or 38400
#include <NeoSWSerial.h>
NeoSWSerial gpsPort(3,2);

// Worst: SoftwareSerial NOT RECOMMENDED
//---------------------

#include <NMEAGPS.h>
NMEAGPS GPS;
gps_fix fix;


void setup()
{
 // Serial.begin(115200);
  gpsPort.begin( 9600 );
 
  // Configure the GPS device
  GPS.send_P( &gpsPort, F("PGCMD,33,0") );  //Turn off antenna update nuisance data
  GPS.send_P( &gpsPort, F("PMTK314,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0") ); //Request RMC and GGA Sentences only
  GPS.send_P( &gpsPort, F("PMTK220,1000") ); //Set update rate to 1 hz
  pinMode(buttonPin, INPUT);
  pinMode(10, OUTPUT); // reserve it to keep SD card happy
  if (!SD.begin(chipSelect)) {
    Serial.println( F("No SD card/reader found!") );
    for (;;); // hang here!
  }

  if (SD.exists(gpsFilename)) { //Delete old data files to start fresh
    SD.remove(gpsFilename);
  }

  gpsDataFile = SD.open(gpsFilename, FILE_WRITE);

  copyFile( "header.txt" );
 
}


void loop()
{
  // Read characters and parse into a fix structure
  if (GPS.available( gpsPort )) {

    // Get the new fix data (GPS quiet time just started)
    fix = GPS.read();

    // Only save data if we have a fix
    if (fix.valid.status &&
        (fix.status >= gps_fix::STATUS_STD)) {


      gpsDataFile.print( fix.longitude(), 5);
      gpsDataFile.print( ',' );
      gpsDataFile.print( fix.latitude(), 5 );
      gpsDataFile.print( ',' );
      gpsDataFile.println( fix.altitude() );
      gpsDataFile.flush(); // this works, but it slows things down
    }
  }
 
    // read the state of the pushbutton value:
  buttonState = digitalRead(buttonPin);

  // check if the pushbutton is pressed.
  // if it is, the buttonState is HIGH:
  if (buttonState == HIGH) {
     copyFile( "trailer.txt" );  // copy the rest of the XML stuff
     gpsDataFile.close();
     while (true) { // forever
  //     blink an LED?  Beep?  LED off?
     }
  // }

} // loop


void copyFile( const char *filename )
{
  File file = SD.open( filename );
  if (file.isOpen()) {

    uint8_t buffer[ 256 ];
    for (;;) {
      int numRead = file.read( &buffer[0], sizeof(buffer) );
      if (numRead == 0)
        break;
      gpsDataFile.write( buffer, numRead );
     
    }
  gpsDataFile.println(" ");
  gpsDataFile.println(" ");
    file.close();
  }
}



-dev

Indentation is your friend.  I spotted the problem immediately because of indentation:

Code: [Select]
   if (buttonState == HIGH) {
     copyFile( "trailer.txt" );  // copy the rest of the XML stuff
     gpsDataFile.close();
     while (true) { // forever
  //     blink an LED?  Beep?  LED off?
     }
  // }

The problem is that the closing brace for the if statement is commented out.  The "symptom" that caught my eye was two closing braces at the same indentation level.

You should also fix the two println's you added in copyFile, for the same reason.

If you use the Arduino IDE to edit your source code, just press control-T to auto-format everything.  If not, you could try one of these Source Code Editor applications.  I think most developers use Emacs, Notepad++ and vi/Vim, if they aren't locked into a Micro$oft suite (Visual Studio, too bloated IMO  :P).

Tonyi

I saw that you had just posted, last night, and was trying to get my last attempt in before, I presumed, you retired for the day; it was getting late.  I thought about checking for simple errors again, and ctr t to correct the format, but decided to get it to you asap, and that caused my usual type of simple error; I've been plagued with...  The program is working fine, even the closing trailer, and LED I added to visually detect a successful close.
I'm guessing you mean, I should fix the formatting of the closing brackets, code for the gpsDataFile.println(" ")?
Code: [Select]
You should also fix the two println's you added in copyFile, for the same reason.
I thought you might show me a better, more efficient way, to create the spacing, after the header.txt was added.  Also, a cleaner method, to create a space, before the closing "trailer.txt" is added.
I know its probably not recommended, but, since pin 10 was already, being used as an output, I used it for the LED; ON while the program is running and OFF when I press the trailer.txt is being added, at the termination, of the running of the program.   What, if any are your thoughts on the use of pin 10, which is "reserved".
Also, as I'm sure you know, the GPS altimeter is very inaccurate, especial at startup, so I ordered a pressure sensor (barometer) and would appreciate a bit more of you guidance implementing into the code.  So, please stick around; its on the slow boat from China (or its lost in the mail; a month waiting so far). Code submitted  and remembered to ctr t it!
Code: [Select]

/*
  SD card connections:
 ** MOSI - pin 11
 ** MISO - pin 12
 ** CLK - pin 13
 ** CS - pin 4
*/

int chipSelect = 4;
const int buttonPin = 5;
int buttonState = 0;
#include <SdFat.h>
#include <SPI.h>

SdFat SD;
File  gpsDataFile; // Data object you will write your sensor data to
const char *gpsFilename = "GPSData.txt";

//---------------------
//  Pick a serial port for the GPS

// BEST choice is a HardwareSerial port:
//    Use Serial1 on a Mega, Leo or Due board
//    You could use Serial on any board (disconnect GPS
//       when uploading over USB).
//#define gpsPort Serial1

// 2nd best:
//#include <AltSoftSerial.h>
//AltSoftSerial gpsPort; // must be on specific pins (8 & 9 for an UNO)

// 3rd best: must be baud rate 9600, 19200 or 38400
#include <NeoSWSerial.h>
NeoSWSerial gpsPort(3, 2);

// Worst: SoftwareSerial NOT RECOMMENDED
//---------------------

#include <NMEAGPS.h>
NMEAGPS GPS;
gps_fix fix;


void setup()
{
  // Serial.begin(115200);
  gpsPort.begin( 9600 );

  // Configure the GPS device
  GPS.send_P( &gpsPort, F("PGCMD,33,0") );  //Turn off antenna update nuisance data
  GPS.send_P( &gpsPort, F("PMTK314,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0") ); //Request RMC and GGA Sentences only
  GPS.send_P( &gpsPort, F("PMTK220,1000") ); //Set update rate to 1 hz
  pinMode(buttonPin, INPUT);
  pinMode(10, OUTPUT); // reserve it to keep SD card happy
  if (!SD.begin(chipSelect)) {
    Serial.println( F("No SD card/reader found!") );
    for (;;); // hang here!
  }

  if (SD.exists(gpsFilename)) { //Delete old data files to start fresh
    SD.remove(gpsFilename);
  }

  gpsDataFile = SD.open(gpsFilename, FILE_WRITE);

  copyFile( "header.txt" );

}


void loop()
{
  // Read characters and parse into a fix structure
  if (GPS.available( gpsPort )) {

    // Get the new fix data (GPS quiet time just started)
    fix = GPS.read();

    // Only save data if we have a fix
    if (fix.valid.status &&
        (fix.status >= gps_fix::STATUS_STD)) {


      gpsDataFile.print( fix.longitude(), 5);
      gpsDataFile.print( ',' );
      gpsDataFile.print( fix.latitude(), 5 );
      gpsDataFile.print( ',' );
      gpsDataFile.println( fix.altitude() );
      gpsDataFile.flush(); // this works, but it slows things down
    }
  }

  // read the state of the pushbutton value:
  buttonState = digitalRead(buttonPin);

  // check if the pushbutton is pressed.
  // if it is, the buttonState is HIGH:
  if (buttonState == HIGH) {
    gpsDataFile.println(" ");
    copyFile( "trailer.txt" );  // copy the rest of the XML stuff
    gpsDataFile.close();
    while (true) { // forever
      digitalWrite(10, LOW);
    }
  }

} // loop


void copyFile( const char *filename )
{
  File file = SD.open( filename );
  if (file.isOpen()) {

    uint8_t buffer[ 256 ];
    for (;;) {
      int numRead = file.read( &buffer[0], sizeof(buffer) );
      if (numRead == 0)
        break;
      gpsDataFile.write( buffer, numRead );

    }
    gpsDataFile.println(" ");
    gpsDataFile.println(" ");
    file.close();
  }
}




Go Up