How to make read() faster than serial.available()

Projects Programming
1

Dear All,

I am facing a boring issue.
I am using OpenLog and I am trying to read a file.

To read a file, you have to enter the following code

( My issue is on the following code

          _openlog.print(F("read "));
          _openlog.print(fileName);
          
          //_openlog.print(F(" 0"));
          //_openlog.flush();
          _openlog.print(F("\r"));
//          _openlog.print(13,BYTE);
          /*
          _openlog.print(start);
          
          if(end > 0)
          {
            _openlog.print(F(" "));
            _openlog.print(end);
          }
          */
      

          //sprintf(buffer2, "read %s\r", fileName);
          //Serial.print(buffer2);

          //Wait for OpenLog to respond with 'r'
          while(1) {
            if(_openlog.available())
            {
              if(_openlog.read() == '\r')
              {
                isCommandMode = false;
                break;
              }
            }
          }
// Code to code 2

Just after the above code, it start reading the file while _openlog is available.

(The output is ASCII)

//code 2
int charactersRead = 0;

  for(int timeOut = 0 ; timeOut < 1000 ; timeOut++)
  {
    while(_openlog.available() > 0)
    {
      char r = (_openlog.read());
      Serial.write(r); //Take the character from OpenLog and push it to software serial TX
      charactersRead++;//= charsToRead;
      
      timeOut = 0;
    }

    delay(1);
    
  }

answer = 1;

Serial.write() should print this

{0,228,03,1771,2d6f,12,55&ti=150811231018}{0,228,03,1771,2d6f,12,55&ti=150811231409}{0,228,03,1771,2d6f,12,60&ti=150811233950}{0,228,03,1771,2d6f,12,57&ti=150811235133}{0,228,03,1771,2d6f,12,58&ti=150811235318}{0,228,03,1771,2d6f,12,61&ti=150811235646}{0,228,03,1771,2d6f,12,61&ti=150812000855}{0,228,03,1771,2d6f,12,62&ti=150812001456}{0,228,03,1771,2d6f,12,59&ti=150812001727}{0,228,03,1771,2d6f,12,59&ti=150812002009}{0,228,03,1771,2d6f,12,63&ti=150812021312}{0,228,03,1771,2d6f,12,63&ti=150812021614}{0,228,03,1771,2d6f,12,54&ti=150812022653}{0,228,03,1771,2d6f,12,56&ti=150812022817}{0,228,03,1771,2d6f,12,59&ti=150812023815}{0,228,03,1771,2d6f,12,62&ti=150812024332}{0,228,03,1771,2d6f,12,63&ti=150812025805}{0,228,03,1771,2d6f,12,62&ti=150812030627}

but it print this

{0,228,03,1771,2d6f,12,55&ti=150811231018}{0,228,03,1771,2d6f,12,55&ti=1508140,76,=252,22i05037f853{21d,i142,2,&000,7f2526215122&0,11,2&13,8031f6=2}2,22i11001f252{,7d,=002,22i00.

>

The '>' is normal. It means that file is read.

If you look at the the seond display, the print should fine until the 75th caracter.

This why I am tring to understand.

I think it's because Serial.available() hold 64 byte, and then it crash. My code run at 9600 baud.

How an I make the _openlog.read() "remove" the red caracter from the Serial, when it's read???

Do you see my problem?

//code 2
int charactersRead = 0;

  for(int timeOut = 0 ; timeOut < 1000 ; timeOut++)
  {
    while(_openlog.available() > 0)
    {
      char r = (_openlog.read());
      Serial.write(r); //Take the character from OpenLog and push it to software serial TX
      charactersRead++;//= charsToRead;
      
      timeOut = 0;
    }

    delay(1);
    
  }

answer = 1;

Thank a lot

2

What baud rate are you using to talk to OpenLog?
What baud rate are you using to talk to Serial?

4

pierrot10:
How an I make the _openlog.read() "remove" the red caracter from the Serial, when it's read???

Post your complete program so we can see all the details and everything in its context.

To answer the question in your title

How to make read() faster than serial.available()

Serial.read() is far faster than the speed at which data arrives even at a high baud rate.

My guess is that your code is written in such a way that it is blocking progress.

Maybe you will get some ideas from the examples in serial input basics which are simple, reliable and non-blocking.

...R

5

How an I make the _openlog.read() "remove" the red caracter from the Serial, when it's read???

That's REALLY a stupid question. The read() method DOES remove the character from the buffer. That is what it does.

6

What is it with the leading underscores in names?

Leading underscores in global variables are reserved. Don't use them.

http://snippets-r-us.com/

7

Dear All,
Thank for your replies and sorry to answer so late.

@Johnwasser and Delta_G
My baud rate are 9600 for serial (UART) and my two serial _openlog and _cell
OpenLog does not recommand to go hiher than 9600 and it's default rate for OpenLog

@Pauls,
I am agree it's a stupid question or may be how I formed it. I know that read() remove caracters after reading
But why it can not read correctly all data and why it "crash" after the 75th display (11 byte after the buffer limit).
How can I make sure, that read remove the caracters. I am confused

@Nick Gammon
Ok this I did not know. I red the private variable should be predeced by an _. As I declared it as private, I added an _
I can change it to openlog instead of _openlog ...

class WI968C{
 private:
 SoftwareSerial _cell;
 SoftwareSerial _openlog;

 const char * _apn;
 const char * _apn_user;
....

@Robin2
Here is my code of my function read()

// the two last parameters are not used yet, but it will
uint8_t WI968C::sd_read(char * fileName, int start, int end)
{ 
  uint8_t answer = 0;
  // If SD_ACTIVE is defined
  #ifdef SD_ACTIVE
    // IF SD card is well initialized and work and inserted
    if(isSDok)
    {
      #ifdef DEBUG_SD
        sprint(F("\nSD READ "));
        sprintln(fileName);
      #endif
    
      // Listen the Serial port 2,3
      _openlog.listen();
      delay(100);
      if (_openlog.isListening())
      {
        // OpenLog has two mode : command mode '>' and writing mode mode '<'
        // Before entering the read command, I make sure openlog is in command mode
        if(sd_gotoCommandMode())
        {
          // Send read command and the file name
          _openlog.print(F("read "));
          _openlog.print(fileName);
          _openlog.print(F("\r"));

          //Wait for OpenLog to respond with 'r'
          while(1) {
            if(_openlog.available())
            {
              if(_openlog.read() == '\r')
              {
                isCommandMode = false;
                break;
              }
            }
          }


          int charactersRead = 0;
          for(int timeOut = 0 ; timeOut < 1000 ; timeOut++)
          {
            while(_openlog.available() > 0)
            {
              char r = _openlog.read();
              // Here, it should print all caraceters but it run wrong at 75th caracter
              Serial.write(r);
              //charactersRead++;//= charsToRead;
              timeOut = 0;
            }

            delay(1);
          }


          answer = 1; // Temporarely

        }
        else
        {
          #ifdef DEBUG_SD
            //sprintln(F("Fail cmd mode"));
          #endif
          answer = -51;
        } // end if gottoCommandMod

       // Listen _cell
        do
        {
         // return to listing Serial port (10,11)
          _cell.listen();
          #ifdef DEBUG_SD
            sprintln(SD_LISTING_CELL);
          #endif
          delay(100);
        }while(!_cell.isListening());


      } // end listing
      else
      {
        #ifdef DEBUG_SD
          sprintln(SD_ERR_LISTING_OPENLOG);
        #endif
        //isCommandMode = false;
        answer = -52;
      } 

    }
    else // If isSDok
    {
      #ifdef DEBUG_SD
        sprintln(SD_KO);
      #endif
      answer = -53;
    }
  #else
    #ifdef DEBUG_SD
      sprintln(SD_UNACTIVE);
    #endif
    answer = -54;
  #endif

  return answer;
}

I hope you can help me to understand why I can not read 756 caracters of a txt file from my SD card (OpenLog) or why I can not read, at least 128 caracters...

Thank for your help

8 
 SoftwareSerial _cell;
 SoftwareSerial _openlog;

Using two instances of software serial rarely works for anybody. Why do you think you will be the exception?

9

pierrot10:
@Robin2
Here is my code of my function read()

I can't make sense of your code without spending too much time studying it.

The first thing that jumped out at me is the fact that you have delay() in your code. You should have no delay()s.

You also seem to have stuff about an SD card (?) - what has that got to do with receiving data ?

Did you look at the examples in serial input basics? Is there any reason not to use one of them ?

...R

10 
  uint8_t answer = 0;

...

        answer = -52;

Trouble brewing here. uint8_t is an unsigned integer. Why are you assigning a negative number to it?

11

Dear All

Thank for your answers

@Nick
Here feel me a bit stupid.
I assign negativ number for error and positive for good answer. Usualy there are only '1' or '2'.
So in that case, I can not declar answer as uini8_t because answer can return negativ number. I should better use 'int' because it range is from -32,768 to 32,767 while uin8_t goes from 0 to 255. (correct me if I am wrong)

@Pauls
May be I did it wrong, but I am doing same as this

The only différence, I am using a class.
How should I do better?
How can I do the same as

#include <SoftwareSerial.h>
// software serial #1: TX = digital pin 10, RX = digital pin 11
SoftwareSerial portOne(10,11);

// software serial #2: TX = digital pin 8, RX = digital pin 9
// on the Mega, use other pins instead, since 8 and 9 don't work on the Mega
SoftwareSerial portTwo(8,9);

My constructor is :

WI968C::WI968C():_cell(10,11),_openlog(2,3)// RX / TX
{


   /*FTP*/
  //_apn = "internet";
  _apn = "click";
  _apn_user = "";
  _apn_password = "";
  
}
12

How should I do better?

Use a Mega, with 4 hardware serial ports.

While you are logging, you can not listen for phone calls/text messages. While you are listening for phone calls/text messages, you can not be logging data.

13

PaulS:
While you are logging, you can not listen for phone calls/text messages. While you are listening for phone calls/text messages, you can not be logging data.

It's not hard to interleave the two. Serial takes time between chars.

14

PaulS:
While you are logging, you can not listen for phone calls/text messages. While you are listening for phone calls/text messages, you can not be logging data.

Dear Pauls, thank for your answer.
Yes I am aware about this. The logging while happen only when GSM network is not available. Each 30sec, my module look for GPS position and send it to a remote server. If it can not be sent for any reason (particularely when GSM netork is not available), it log the GPS fix to the SD card. Secondly, reading or writing take a short time, so the SMS can wait. But it's a good observation, thank.

I also want to use a mini pro, because it small. I also hesitated with a micro or a nano but input voltage it 7-12V and I am using a 3.7 Lihium battery. So the mini pro 3.3V is the best.

In any case I have to understand why, it's not reading, at least, up to 128 caracters :slight_smile:

15

It's not hard to interleave the two. Serial takes time between chars.

Dear GoForSmoke,
May be my english is not the best but I do not understand what you excately want to say.

16

Serial characters take time to send and receive There is long enough between characters arriving on one port to print one out on a different port even at 115200 baud.

The processor is greatly faster than serial. It is possible to send and read together without slowing either down and .... do other things too.

Learn to write the Blink Without Delay sketch or one like it from a tutorial. It is a bit like juggling balloons with one hand, each requires a small tap now and then and never held while the rest fall.

I have code that does use SIM900 GSM. I found the library itself stops everything else up 30 to 60 seconds during the begin(). Nothing else runs then.
The person I did the code for doesn't need data every minute so it's no big deal to him. But I would have the GSM on a dedicated controller unable to stop data collection.

17

:o

18

viphuchit:
:o

Remember that baud rate is bits/sec. Also that serial has a start bit and a stop bit. 115200 baud is 11520 chars/sec. And Arduino UNO is 16 million cycles/sec.

At 115200 baud you should be able to read 1 character every 1388 cycles. Yes, there is time to read the available char, send a char out through another port, blink a status led and a few other things before the next read is available. The thing to do is not waste over 1000 cycles just waiting for anything.

I have a sketch that watches button bounce. While doing that it counts how many times loop() runs a second and blinks a status led and shoots out serial reports every so often. It runs over 57000x/sec. I can run a lot of buttons, 1 per loop() and still be extremely responsive.

When I add a delay(1) into that same loop() is runs 970 to 980x/sec. A delay(1) steals 16000 cycles.

19

Good day!
I juts come bak from my travel and I could make a new test. I took a new breakout board and I only connect a Arduino Mini pro with a Openlog with a couple of wire to connect the Tx and Rx pin and the GND/VCC.

I first write a simple code to use the OpenLog with UART and it's work!!!! I can read more than 742 caracters

Then I created a serial port

SoftwareSerial portOne(10,11);

and I connect the Rx and Tx of the OpenLog to 10 and 11.
I do not use the listen() function because I started with only one serial port.
.. and it did not work. I have the same problem as before.

Then now I asked my self if OpenLog board can only be used with UART (pin 0 and 1 of Arduino) and why it does not work in Serial Port???

What do you think.

Now I have a second problem, because my Rx and Tx of the GPS must be connected to the pins 0 and 1 of Arduino Mini pro (UART).

How can I use two devose in UART? Can use a logic switch?
If the pin 5 of Mini pro is HIGH, the logical switch connect Rx/tx of OpenLog to the Mini pro (0,1)
If the pin 5 of Mini pro is LOW, the logical switch connect Rx/Tx of the GPS to the Mini Pro (0,1)

Thank for your help

20

pierrot10:
I juts come bak from my travel and I could make a new test. I took a new breakout board and I only connect a Arduino Moni pro with a Openlog with a coupke of wire to connect the Tx and Rx pin and the GND/VCC.

You know perfectly well that if you were that careless writing your code it would not work. Why must we put up with careless typing ?

I do not use the listen() function because I started with only one serial port.
.. and it did not work. I have the same problem as before.

Then now I asked my self if OpenLog board can only be use with UART (pin 0 and 1 of Arduino) and why it does not work in Serial Port???

What do you think.

You have been here long enough now to know that you should post the latest version of your code without me having to waste my time asking for it.

Now I have a second problem, because my Rx and Tx of the GPS must be connected to the pins 0 and 1 of Arduino Mini pro (UART).

Give us all the information about all of your project so we can see the whole picture and give you sensible advice.

...R

21

Dear Robin2

Sorry for my mistakes.

Here is my latest code. I created an new ino file with only the code I need to test OpenLog.
Note, I defined PORTONE. In that way, I can easly switch between UART and Serial port.

#include <SoftwareSerial.h>

#define SDCELLIDLOGFILE "CELLLOG.TXT"
#define BUFFERSIZE 200
#define DEBUG

//#define PORTONE

char buff[BUFFERSIZE];

bool readSerial = false;
int page = 1;

SoftwareSerial portOne(10,11);

void setup() {
  // put your setup code here, to run once:

  #ifdef PORTONE
   portOne.begin(9600);
   Serial.begin(9600);
  #else
    Serial.begin(9600);
  #endif
  
  Serial.println(F("Begin"));
  
  pinMode(4,OUTPUT);       // Reset OpenLog
  


// RESET OPENLOG  
Serial.println(F("Rest"));
   // Rest OpenLog
  digitalWrite(4, LOW);
  delay(100);
  digitalWrite(4, HIGH);

  // Wait for OpenLog to indicate file is open and ready for writing
  while(1) {
    
    #ifdef PORTONE
      if(portOne.available())
        if(portOne.read() == '<') break;
    #else
      if(Serial.available())
        if(Serial.read() == '<') break;
   #endif
  }
   
// CHANGE TO COMMAND MODE
   #ifdef PORTONE
     portOne.write(26);
     portOne.write(26);
     portOne.write(26);
   #else
     Serial.write(26);
     Serial.write(26);
     Serial.write(26);
   #endif
   
   // WAIT FOR > WHICH INDICATE THE COMMAND MODE
   while(1)
   {
     #ifdef PORTONE
       if(portOne.available())
       {
         if(portOne.read() == '>')
         {
           break;
         }
       }
     #else
       if(Serial.available())
       {
         if(Serial.read() == '>')
         {
           break;
         }
       }
     #endif
   }

  
 // READ FILE
   sd_read(SDCELLIDLOGFILE);
          
}

void loop() {
  // put your main code here, to run repeatedly:
  
}

void sd_read(char * filename)
{
   // READ
  
// SEND THE COMMAND READ AND THE FILE TO BE RED
  #ifdef PORTONE     
     portOne.print(F("read "));
     portOne.print(filename);
     portOne.print(F("\r"));
  #else
     Serial.print(F("read "));
     Serial.print(filename);
     Serial.print(F("\r"));
  #endif  

     //Wait for OpenLog to respond with 'r'
     while(1) {
       #ifdef PORTONE
         if(portOne.available())
         {
           if(portOne.read() == '\r')
           {
             break;
           }
         }
       #else
         if(Serial.available())
         {
           if(Serial.read() == '\r')
           {
             break;
           }
         }
       #endif
     }

     Serial.println(F(""));

     int charactersRead = 0;
     for(int timeOut = 0 ; timeOut < 1000 ; timeOut++)
     {
  #ifdef PORTONE
         while(portOne.available() > 0){
            char r = portOne.read();
  #else         
         while(Serial.available() > 0){
           char r = Serial.read();  
  #endif     
           // > INDICATE THAT OPENLOG FINISHED TO READ THE FILE. WHEN FINISH (> = COMMAND MODE)
           if(r == '>')
           {
            // buff[charactersRead++] = '\0';
             Serial.print(r);
             Serial.println(F("Fin"));
             
             int o = page-1;
             int t = (BUFFERSIZE-1) * o;
             Serial.println(charactersRead + t);
             Serial.println(page);
             timeOut = 1000; // leave for
             break; // Leave while
           }
           else
           {
// DO NOT READ THE dot WHEN readSerial IS FALSE
             if(r=='.' && readSerial == false) 
             {
               //Serial.println(F("dot"));
             }
             else
             {
// READ ONLY THE CARACTERS BETWEEN { and }
               // Stop reading and display the next caracters after a new line
               if(r=='}')
               {
                 readSerial = false;
                 
                 //buff[charactersRead] = '\n';
                 charactersRead++; 
                
                 Serial.print('\n');
               }
     
               if(readSerial)
               {
                 // We can start reading the needed caracters
                 Serial.print(r);
                 
                 //buff[charactersRead] = r;
                 charactersRead++;
               }
               
               
               if(r=='{')
               {  
                 readSerial = true;
               }
               
              }// end else
            } // end if(r == '>')
             
            
            timeOut = 0;
            
          
            
            
            
            
           } // end while
            //delay(1);
         } // end for
}

I hope it's help.
Thank