Solved: Advice to move away from serial.readBytes()

Hi there, I need some help to stop using serial.readBytes() in my sketch, I understand it is a blocking function and therefore undesirable, and while it works for my project, I believe reading one byte at a time would work better.

I have created a circuit to interface with my car's ECU (early Nissan Consult protocol, OBDII does not apply). As far as communicating with the ECU, the circuit works.

I send a few bytes to the ecu (via Serial1 on a mega), it should reply (and does) and begins a stream of bytes at 9600 baud as below:

0xFF 0x04 data1 data2 data3 data4

This repeats continuously. The first byte is a generic response, the second byte is the number of bytes to follow. The data bytes are some engine parameters that keep the same position within the response and I know what they are.

I have read through Robin2's Serial Input Basics, however the problem using an example from there is that the data could possibly match the start/end markers.

Here is the code that allows me to display the values currently:

 while (Serial1.available() > 0) {

    Serial1.readBytes(consultReply,9);
    
    for (int i = 0; i < 9; i++) {
      Serial.print(consultReply[i]);   //print to serial monitor for debugging
      Serial.print(" ");
    }
    
    if (consultReply[0] == 0xFF && consultReply[1] == 0x04 && consultReply[6] == 0xFF && consultReply[7] == 0x04) {    //check for start & end markers
      waterTemp = consultReply[2] - 50;
      tps = consultReply[3];
      rpm = (((consultReply[4] * 256.0) + consultReply[5]) * 12.5);
      Serial.print("    ***   ");
      Serial.print(millis() - timenow);          //make a timestamp to check response time
      timenow = millis();
    }

    Serial.println();
}

So this is what works, and I can output the variables to an LCD, however I think it could be better, and my output on the serial monitor with the millis timing the response times does end up with some gaps of ~200ms.

I check for 0xFF 0x04 at each end of the array to stop the data acting as start/end markers.

The consultReply array is deliberately longer than the data so it can 'line up' and match the start/end markers.

I'm using serial.Print for debugging and this is a snippet of my output:

4 83 24 0 0 255 4 83 24 
4 83 24 0 0 255 4 83 24 
4 83 24 0 0 255 4 83 24 
255 4 83 24 0 0 255 4 83     ***   232
0 0 255 4 83 24 0 0 255 
4 83 24 0 0 255 4 83 24 
255 4 83 24 0 0 255 4 83     ***   92
0 0 255 4 83 24 0 0 255 
4 83 24 0 0 255 4 83 24 
255 4 83 24 0 0 255 4 83     ***   92
0 255 4 83 24 0 0 255 4 
255 4 83 24 0 0 255 4 83     ***   66
83 24 0 0 255 4 83 24 0 
255 4 83 24 0 0 255 4 83     ***   65
255 4 83 24 0 0 255 4 83     ***   38
255 4 83 24 0 0 255 4 83     ***   39
83 24 0 0 255 4 83 24 0 
24 0 0 255 4 83 24 0 0 
255 4 83 24 0 0 255 4 83     ***   91
255 4 83 24 0 0 255 4 83     ***   39
255 4 83 24 0 0 255 4 83     ***   38
255 4 83 24 0 0 255 4 83     ***   39
255 4 83 24 0 0 255 4 83     ***   39
255 4 83 24 0 0 255 4 83     ***   37

This recognises when a readable string arrives in consultReply and prints a timestamp since the last one arrived.
I do find it strange how the output doesn't "wrap around" i.e seems to skip numbers quite a lot.

I have attempted to use Serial1.read(), put the bytes into a buffer, check the buffer for the correct arrangement of the start/end markers, however when using Serial1.read() in any way the data is ok for a couple of seconds, then I seem to end up with many numbers repeating and I cannot extract any useable data - I will end up with lines like:

255 255 255 255 0 0 0 0 
4 4 4 0 0 0 0 255 255 255 
255 255 4 83 255 255 255
0 0 0 0 0 0 0 0

etc.

I don't understand why Serial1.readBytes() will consistently get the serial data in the correct order, but Serial1.read() into a buffer will read the same byte several times.

Any advice for me on this?
Happy to provide more info if required.
Cheers

Maybe try something like this:

char
    rxBuff[20];

byte
    waterTemp,
    tps;
int
    rpm;
    
void setup( void ) 
{
    Serial.begin(9600);
    Serial1.begin(9600);
    
}//setup


//RX state names
#define RX_HEADER       0
#define RX_NUMBYTES     1
#define RX_DATA         2
//
void loop( void ) 
{
    char
        ch;
    byte
        idx;
    static unsigned long
        timeStart;
    static byte
        stateRX = RX_HEADER,
        numBytes;

    if( Serial1.available() > 0 )
    {
        do
        {
            char ch = Serial1.read();
            switch( stateRX )
            {
                case    RX_HEADER:
                    if( ch == 0xff )
                    {
                        timeStart = millis();
                        stateRX = RX_NUMBYTES;
                        
                    }//if
                        
                break;

                case    RX_NUMBYTES:
                    numBytes = ch;
                    if( numBytes == 0x04 )
                    {
                        idx = 0;                    
                        stateRX = RX_DATA;
                        
                    }//if
                    else
                        stateRX = RX_HEADER;
                                            
                break;

                case    RX_DATA:
                    rxBuff[idx++] = ch;
                    numBytes--;
                    if( numBytes == 0 )
                    {
                        waterTemp = rxBuff[0] - 50;
                        tps = rxBuff[1];
                        rpm = (int)(((rxBuff[2] << 8) + rxBuff[3]) * 12.5);
                        
                        Serial.print("\n    ***   ");
                        Serial.print( "Water oC: " ); Serial.println( waterTemp );
                        Serial.print( "TPS %   : " ); Serial.println( tps );
                        Serial.print( "RPM     : " ); Serial.println( rpm );
                        
                        Serial.print( "Time    : " ); Serial.println( millis() - timeStart );
                                            
                        stateRX = RX_HEADER;
                        
                    }//if    
                    
                break;

            }//switch
            
        }while( Serial1.available() );
        
    }//if

}//loop

Thanks Blackfin, I'll give that a shot.

So I inserted that code into my sketch, and initially I had no luck, so I tried changing ch into an unsigned char and this yielded some results, however as per my previous tries with Serial1.read() it would work for a second and then start yielding strange results (as attached see "Formatted output")

I then tried to just print variable ch as it came in to analyse, and it also had strange results (see "ch as read" attached)

After that I tried reading the size of the Arduino's buffer with Serial.println(Serial1.available()) and it quickly shot up to 62. I assume this means the buffer is overflowing, making it impossible to get an entire string? I didn't think 9600 baud was that fast, and my code doesn't have any delay()s in it. Unfortunately I can't change the way the ECU transmits.

I have attached my code, please excuse the imperfections :slight_smile:

Cheers

ArduinoConsult_v3.ino (4.47 KB)

ch as read.txt (10 KB)

Formatted output.txt (60.1 KB)

Please, post a complete structure of the transmission frame (I mean frame length = number of data bytes; any end marker; you have already given beginning marker; any checksum) that is coming from the engine. You have shown: 0xFF 0x04, data1 data2 data3, data4; after that what? How often the engine sends data. Are data coming as binary or ASCII?

Here is a link to the ECU's protocol:
Consult Protocol & Commands

Paragraph 1 & 2 explain protocol and format. The command I am sending to recieve the stream is para 3.5.

Thanks

quick edit: Would I be better to send a stop command to the ecu, read & analyse data, then send the command again?

Thoughts:

  • I noticed in my post that I had declared ch as a char and then again in the loop as a char. You've left the original char decl but have changed loop decl to unsigned char. You should remove the "char ch;"

  • "...and my code doesn't have any delay()s in it..."; Take the delay(20) out of loop(). It's not required. At 9600 baud a character takes just over 1.04mS to clock in. You could, in theory, receive 19 characters into the receive buffer while you do nothing in delay().

  • Can you move the code within "if(needtosend == true)" into your consultConnect() function?

  • Might want to change waterTemp to an int since "-50" could lead to a negative result.

Oops - I was looking so closely at my recieve function I didnt notice the delay in loop().
I have removed that and changed the other things you suggested however still having the same problem.

Any diff?

#include <SoftwareSerial.h>

void DisplayValues( unsigned long tS );

boolean consultConnected = false;
byte consultInitByte = 0x00;

unsigned char 
    rxBuff[20];
int 
    waterTemp, 
    lastwaterTemp;
unsigned char 
    tps, 
    lasttps;
int 
    rpm, 
    lastrpm;

#define RX_HEADER       0
#define RX_NUMBYTES     1
#define RX_DATA         2

SoftwareSerial screenSerial(7, 8);

void setup() 
{
    Serial.begin(9600);
    Serial1.begin(9600);
    screenSerial.begin(9600);
    pinMode(LED_BUILTIN, OUTPUT);
    delay(1000);

    screenSerial.print("t0.txt=\"Starting\"");
    lcdSend();

    delay(1000);
    consultConnect();
    delay(500);
}

void loop() 
{
    if (consultConnected == false) 
        consultConnect();
    else
        readValues();

}//loop

void lcdSend() 
{        
    //shorten code to send stuff to lcd
    screenSerial.write(0xff);
    screenSerial.write(0xff);
    screenSerial.write(0xff);
}


void consultConnect() 
{
    unsigned long         
        consultConnectTimeout = millis();
        
    // Attempt to initialize consult to ecu conversation
    // Keep retrying until it initializes
    
    while( consultConnected == false && ((millis() - consultConnectTimeout) < 3000ul) ) 
    {
        Serial1.write(0xFF);
        Serial1.write(0xFF);
        Serial1.write(0xEF);
        delay(1);
        consultInitByte = Serial1.read();
        
        if (consultInitByte == 0x10) 
        {
            delay(1);
            screenSerial.print("t0.txt=\"Connected\"");
            lcdSend();
            consultConnected = true;

            Serial1.write(0x5A);
            Serial1.write(0x08);
            Serial1.write(0x5A);
            Serial1.write(0x0D);
            Serial1.write(0x5A);
            Serial1.write(0x00);
            Serial1.write(0x5A);
            Serial1.write(0x01);
            Serial1.write(0xF0);
            
        }//if
        
    }//while
    
    if (consultConnected == false) 
    {
        screenSerial.print("t0.txt=\"Timed out\"");
        lcdSend();
        //consultConnected = true;    //changed temporarily, correct this later

    }//if
    
}//consultConnect

void readValues() 
{
    unsigned char
        ch;
    byte
        idx;
    static unsigned long
        timeStart;
    static unsigned char
        stateRX = RX_HEADER,
        numBytes;

    if( Serial1.available() > 0 )
    {
        do
        {
            ch = Serial1.read();
            
            switch( stateRX )
            {
                case    RX_HEADER:
                    if( ch == 0xff )
                    {
                        timeStart = millis();
                        stateRX = RX_NUMBYTES;
                       
                    }//if
                       
                break;

                case    RX_NUMBYTES:
                    numBytes = ch;
                    if( numBytes == 0x04 )
                    {
                        idx = 0;                   
                        stateRX = RX_DATA;
                       
                    }//if
                    else
                        stateRX = RX_HEADER;
                                           
                break;

                case    RX_DATA:
                    rxBuff[idx++] = ch;
                    numBytes--;
                    if( numBytes == 0 )
                    {
                        waterTemp = rxBuff[0] - 50;
                        tps = rxBuff[1];
                        rpm = (int)(((rxBuff[2] << 8) + rxBuff[3]) * 12.5);
                        
                        DisplayValues( timeStart );
                              
                        stateRX = RX_HEADER;
                       
                    }//if   
                   
                break;

            }//switch
           
        }while( Serial1.available() > 0 );
       
    }//if
    
}//readValues

void DisplayValues( unsigned long tS )
{
    Serial.print("\n    ***   \n");
    Serial.print( "Water oC: " ); Serial.println( waterTemp );
    Serial.print( "TPS %   : " ); Serial.println( tps );
    Serial.print( "RPM     : " ); Serial.println( rpm );   
    Serial.print( "Time    : " ); Serial.println( millis() - tS );
                                          
    if (waterTemp != lastwaterTemp) 
    {
        screenSerial.print("n0.val=");
        screenSerial.print(waterTemp);
        lcdSend();
        lastwaterTemp = waterTemp;
    }//if

    if (tps != lasttps) 
    {
        screenSerial.print("n1.val=");
        screenSerial.print(tps);
        lcdSend();
        lasttps = tps;
    }//if

    if (rpm != lastrpm) 
    {
        screenSerial.print("n2.val=");
        screenSerial.print(rpm);
        lcdSend();
        lastrpm = rpm;
    }//if
    
}//DisplayValues

Hi Blackfin, I had an idea to change the baudrate of the screen and the serial port to 115200, and along with your great code the parameters now display correctly and instantaneously.

Thanks very much for your help :slight_smile: