SOLVED SoftSerial and SD lib collision

Projects Programming
1

Hi all,

it seems like a bug , I don't do any heavy lifting in the code, all simple stuff , and it turns out that bug is consistent all the way.

So using Arduino UNO and IDE 1.0.6 I hooked up GPS on SoftSerial lib, SD card on SPI and Serial to USB to watch traffic.

BUG:

If I include output of data to SD card, output to USB serial goes strange (different lines of data mix together).

If I turn off that part of code, all works fine and lines of data from GPS come in normal.

So here is the code:

#include <SPI.h>
#include <SoftwareSerial.h>


#include <SD.h>

//SPI SD Card Pins
//MOSI = Pin 11
//MISO = Pin 12
//SCLK = PIN 13
int CS_pin = 10;

char buffer[100];
char c;
int c_cnt=0;
int c_avail = 0;

SoftwareSerial mySerial(2, 3); // RX, TX

void setup()
{
   
  Serial.begin(9600);
  Serial.println("Initializing Card");
  //CS Pin is an output
  pinMode(CS_pin, OUTPUT);
  digitalWrite(CS_pin, HIGH);  
  
  //Initialize Card
  if (!SD.begin(CS_pin))
  {
      Serial.println("Card Failure");
      return;
  }
  Serial.println("Card Ready");
   
  File logFile = SD.open("LOG.csv", FILE_WRITE);
  if (logFile)
  {
   logFile.close();
  Serial.println("Printing to log");
  }
  else
  {
   Serial.println("Couldn't open log file");
  }
 
 
  mySerial.begin(9600);
  mySerial.println("Hello, world?");
 
  
}


void loop()
{
 
  c_avail = mySerial.available();
  for(int i = 0 ; i < c_avail; i++){
    c = mySerial.read();
    buffer[c_cnt] = c;
    
    if(c == '\r'){
      //Serial.print("[R]");
      continue;
    }

    if(c == '\n'){
      //Serial.print("[N]");      
      buffer[c_cnt]='\0';
      //Serial.println(buffer);
 
      //> ==== BUG START ====
      //> File logFile = SD.open("LOG.csv", FILE_WRITE);
      //> if (logFile)
      //> {
      //>  logFile.println(buffer);
      //>  logFile.close();
      //> }  
      //> ==== BUG END ====

      c_cnt=0;
      Serial.println(buffer);
      continue;
    }
    c_cnt++;
    //Serial.write(c);
  }  
}

If the BUG code is turned off output is:

Initializing Card
Card Ready
Printing to log
$GPRMC,130841.00,V,,,,,,,170215,,,N72
$GPVTG,,,,,,,,,N30
$GPGGA,130841.00,,,,,0,00,99.99,,,,,,69
$GPGSA,A,1,,,,,,,,,,,,,99.99,99.99,99.9930
$GPGSV,4,1,13,02,16,241,,05,52,294,,06,00,203,,07,57,054,*72
$GPGSV,4,2,13,09,30,096,,10,51,175,,13,25,286,23,20,29,164,7C
$GPGSV,4,3,13,23,02,100,,26,38,289,28,27,02,045,,28,25,161,75
$GPGSV,4,4,13,30,87,158,4B
$GPGLL,,,,,130841.00,V,N45
$GPRMC,130842.00,V,,,,,,,170215,,,N71
$GPVTG,,,,,,,,,N30
$GPGGA,130842.00,,,,,0,00,99.99,,,,,,6A
$GPGSA,A,1,,,,,,,,,,,,,99.99,99.99,99.9930
$GPGSV,4,1,13,02,16,241,,05,52,294,,06,00,203,,07,57,054,*72
$GPGSV,4,2,13,09,30,096,,10,51,175,,13,25,286,23,20,29,164,7C
$GPGSV,4,3,13,23,02,100,,26,38,289,28,27,02,045,,28,25,161,75
$GPGSV,4,4,13,30,87,158,4B
$GPGLL,,,,,130842.00,V,N46
$GPRMC,130843.00,V,,,,,,,170215,,,N70
$GPVTG,,,,,,,,,N30
$GPGGA,130843.00,,,,,0,00,99.99,,,,,,6B
$GPGSA,A,1,,,,,,,,,,,,,99.99,99.99,99.9930
$GPGSV,4,1,13,02,16,241,,05,52,294,,06,00,203,,07,57,054,*72
$GPGSV,4,2,13,09,30,096,,10,51,175,,13,25,286,23,20,29,164,7C
$GPGSV,4,3,13,23,02,100,,26,38,289,28,27,02,045,,28,25,161,75
$GPGSV,4,4,13,30,87,158,4B
$GPGLL,,,,,130843.00,V,N47
$GPRMC,130844.00,V,,,,,,,170215,,,N77
$GPVTG,,,,,,,,,N30
$GPGGA,130844.00,,,,,0,00,99.99,,,,,,6C
$GPGSA,A,1,,,,,,,,,,,,,99.99,99.99,99.9930
$GPGSV,4,1,13,02,16,241,,05,52,294,,06,00,203,,07,57,054,*72
$GPGSV,4,2,13,09,30,096,,10,51,175,,13,25,286,23,20,28,164,7D
$GPGSV,4,3,13,23,02,100,,26,38,289,28,27,02,045,,28,25,161,75
$GPGSV,4,4,13,30,87,158,4B
$GPGLL,,,,,130844.00,V,N40
$GPRMC,130845.00,V,,,,,,,170215,,,N76
$GPVTG,,,,,,,,,N30

If BUG code is turned on, output is:

0,,26,38,288,25,27Initializing Card
Card Ready
Printing to log
$GPRMC,130741.00,V,,,,,,,170215,,,N7D
$GPVTG,,,,,,,,,N30
$GPGGA,130741.00,,,,,0,00,99.99,,,,,,66
$GPRMC,130742.00,V,,,,,,,170215,,,N7E
$GPVTG,,,,,,,,,N30
$GPGGA,130742.00,,,,,0,00,99.99,,,,,,65
,,30,87,164,78
$GPG$GPRMC,130743.00,V,,,,,,,170215,,,N7F
$GPVTG,,,,,,,,,N30
$GPGGA,130743.00,,,,,0,00,99.99,,,,,,64
,,30,87,164,78
$GPG$GPRMC,130744.00,V,,,,,,,170215,,,N78
$GPVTG,,,,,,,,,N30
$GPGGA,130744.00,,,,,0,00,99.99,,,,,,63
$GPRMC,130745.00,V,,,,,,,170215,,,N79
$GPVTG,,,,,,,,,N30
$GPGGA,130745.00,,,,,0,00,99.99,,,,,,62
1,,30,87,164,78
$GP$GPRMC,130746.00,V,,,,,,,170215,,,N7A
$GPVTG,,,,,,,,,N30
$GPGGA,130746.00,,,,,0,00,99.99,,,,,,61
,,30,87,164,78
$GPG$GPRMC,130747.00,V,,,,,,,170215,,,N7B
$GPVTG,,,,,,,,,N30
$GPGGA,130747.00,,,,,0,00,99.99,,,,,,60
1,30,87,164,78
$GPG$GPRMC,130748.00,V,,,,,,,170215,,,N74
$GPVTG,,,,,,,,,N30
$GPGGA,130748.00,,,,,0,00,99.99,,,,,,6F
,30,87,164,78
$GPGL$GPRMC,130749.00,V,,,,,,,170215,,,N75
$GPVTG,,,,,,,,,N30
$GPGGA,130749.00,,,,,0,00,99.99,,,,,,*6E
1,30,87,164,*78

Any hint on this?

SOLUTION:

So just to share with anyone who needs it, in my current setup solution was to change line in SoftwareSerial.h

#define _SS_MAX_RX_BUFF 256 // RX buffer size was 64

This works as is should 8)

Thanks to everyone!

2

Opening and closing the file for each write to the SD card is probably taking too long. Try making the file variable global and opening it just once in setup(). Then in your loop just write to it, no open or close. As a safeguard you could call flush() after each write to the card, or every N writes, whatever you're comfortable with.

Add a test for input of a key over Serial (or add a button to your board and look for it being pressed) to your loop and call close when that happens.

Also, increase the Serial to something faster, like 115200 baud. You may not need this here but sometimes a little extra grease ends up being a good idea at some point.

3

Have a look at the 2nd and 3rd examples in serial input basics. I think your system should work if you put your code for saving to the SD Card where the function showNewData() is. That function is called when all of the data has been received.

Breaking your code into small single-purpose functions makes it much easier to see the overall logic.

...R

4

Opening a file requires a 512 byte buffer in SRAM. I'm wondering if you have that much memory available. If commenting out the file open bit makes the rest of the code work, I'd have to guess that you don't.

http://playground.arduino.cc/Code/AvailableMemory

5

SD and SoftwareSerial don't use up enough combined memory to overlow an Uno's 2K RAM.

I think it's just timing. The GPS spitting out close to 400 bytes each second in the form of half a dozen sentences and the code is grabbing one sentence at a time and writing it to the SD card, along with opening and closing the file each time. If the latency exceeds 64ms, and it will sometimes, the receive buffer will overflow.

Leaving the file open will help a lot but it might not solve the problem 100%. Robin is suggesting capturing all of the data before writing; that should work since then there would be hundreds of milliseconds to write to the card. But there may not be enough memory to create the nearly 400 byte buffer needed. Or maybe there is, I don't know. I wonder if all of those NMEA sentences are really necessary for the application, whatever it is.

6

SD and SoftwareSerial don't use up enough combined memory to overlow an Uno's 2K RAM.

By themselves, no. But in combination with lots of string literals, variables, and the stack, possibly. It's certainly worth checking, rather than dismissing out of hand.

7

PaulS:
By themselves, no. But in combination with lots of string literals, variables, and the stack, possibly. It's certainly worth checking, rather than dismissing out of hand.

You're right.

I'll test it....

8

I added checks in a number of places, including before and after the open calls.
The smallest number returned for free memory was 757 bytes.

I also added a test before writing to the buffer since without it the sketch kept crashing and restarting:

void loop()
{
 
  c_avail = mySerial.available();
  for(int i = 0 ; i < c_avail; i++){
    c = mySerial.read();
  if (c_cnt < 99)              // < ------------------- added check (leave room for terminator)
    buffer[c_cnt] = c;
    
    if(c == '\r'){
      //Serial.print("[R]");
      continue;
    }
9

First thanks everyone for helping out!

Just to answer about sentences, it is all comming from GPS module itself on the serial interface.
Module is burned to 9600/N/1 so I cannot change that.

As proposed I've put File to global and flusing after printing to SD, but all doesn't work the same.
It is so explicit BUG that It hurts :cry:

Code is really small and does nothing more then takes from serial and dumps to sd, is it possible that such a small thing could cause problems (it seems so).

I don't think It will help, but I will try to switch hardware and software Serial, maybe it helps.

Somehow SD is interfering with Serial or other way around.
IMHO I don't think RAM memory is the issue (but that just opinion of person in trouble).

10

Are you sure you can't send the GPS a command to send fewer NMEA sentences? Even if you can't, you could write code to ignore the ones you don't care about and only buffer those that you do.

This serial GPS stuff screws up a lot of people. It's a shame they aren't typically available as SPI devices where you could simply request the latest fix data on demand. It would make life a lot easier.

11

Unfortunately I can't, would be nice.
I don't understand what you mean when you say (write) buffering, because I use static buffer of 100 chars and overwrite that specific buffer each time i get new sentence.

Maybe just dumping all availabe data from Serial to temp buffer and then process it would offload somehow Serial itself, any thoughts on that?

12

I have encountered this problem before. It is not a "bug" with some kind of interference between the serial and the SD card. It is a matter of speed. Your GPS module speed is too low for the amount of different gps sentences you are trying to collect.

Get a gps module which will work with a higher serial baud rate.

Your serial communication to your computer is also running too slowly. Why 9600 ?

Try getting rid of the Serial.println( ) to the computer, which is right after you attempt to write to the SD card file. Try running the app again, and see if the correct data is on the card.

You probably have enough time to write the data to the SD card, or by serial to the computer, but not both.

13

I'd also suggest checking that you haven't got more than 99 characters.

14

nikola1010:
I don't understand what you mean when you say (write) buffering, because I use static buffer of 100 chars and overwrite that specific buffer each time i get new sentence.

Your buffer of 100 characters is big enough for one sentence but not large enough for the rest of the data. The problem is that while you're writing the first sentence to the SD card more GPS data is arriving and is not getting buffered.

With a large enough buffer you could save all of the sentences that come in each second. You'd have to write the code to recognize when the last of it has been received. Then write it to the SD card.

If you don't have that much RAM or you don't want to save all of those sentences you can instead write code to recognize the individual sentences and simply not buffer the ones you don't care about. It's a little more complicated than just looking for a '\n' character but it's not that bad.

Another way to do it would be to increase the RX buffer in SoftwareSerial, assuming you can spare the RAM.

A GPS running at a higher baud rate wouldn't really help you with this problem.

15

Ok just got back with new GPS module that supports 115200.
Tested with putty (COM14, 115200/N/1) works excellent.
Again makes problems in testing code, for some reason it displays gibberish.

#include <SPI.h>
#include <SoftwareSerial.h>


#include <SD.h>

//SPI SD Card Pins
//MOSI = Pin 11
//MISO = Pin 12
//SCLK = PIN 13
int CS_pin = 10;

char buffer[100];
char c;
int c_cnt=0;
int c_avail = 0;

SoftwareSerial mySerial(2, 3); // RX, TX
File logFile;

void setup()
{
   
  Serial.begin(115200);
  Serial.println("Initializing Card");
  //CS Pin is an output
  pinMode(CS_pin, OUTPUT);
  digitalWrite(CS_pin, HIGH);  


  //Initialize Card
  if (!SD.begin(CS_pin))
  {
      Serial.println("Card Failure");
      return;
  }
  Serial.println("Card Ready");
   
 logFile = SD.open("LOG.csv", FILE_WRITE);
  if (logFile)
  {
   // logFile.println(", , , ,"); //Just a leading blank line, incase there was previous data
 //   String header = "ID, Light, Temp, IR1, IR2";
 //   logFile.println(header);
  // logFile.println(data);
  // logFile.println("\n");
   //logFile.close();
      Serial.println("Printing to log");
  }
  else
  {
      Serial.println("Couldn't open log file");
  }
 
 
 mySerial.begin(115200);
 mySerial.println("Hello, world?");
 
  
}

void loop(){
  if(mySerial.available()){
    Serial.write(mySerial.read());
  }

}

Output of that is:

Initializing Card
Card Ready
Printing to log
$£¨US(‰dÅb±±X,,––KJTš5)IGP«ªÄbb±±X,,§•S¤D:AŽA,––KK & ‚±åu.9œ–Ê$£¨US‰dÅb±±X,,––KJTš5)IGP«ªÄbb±±X,,§•&SHD:A£TŠ‰Åbb±ÁX00–N”ʱ±

:o :o

Now it becomes more and more :confused:

It says that softserial supports 115200 and I've tested it (on the same computer with 115200 in putty).

16

jboyton:
With a large enough buffer you could save all of the sentences that come in each second. You'd have to write the code to recognize when the last of it has been received. Then write it to the SD card.

Do you mean that I should for example create buffer[200] and then copy all available() bytes to buffer and process them inside buffer?