Increasing the Rx Buffer of a Teensy3.2 [ Solved #14]

This is further to the thread User editable look up table - Programming Questions - Arduino Forum

The project is almost on the verge of completion … the only hiccup is that I am unable to send via Serial large amounts of waveform data. In fact not more than about 200 bytes. Upto this count the data is received , parsed and I also get a confirmation on the number of bytes got. Beyond this the program does not confirm back. I know I must ask this on Teensy forum but the response is far better here :slight_smile:

Possibly I should increase the Rx buffer ??

The full Teensy 3.2 code is here :

// Hardware used in Teensy3.2

#include <EEPROM.h>

# define StartPin  5
# define SerComPin 6
# define CycleOnLed 9
# define AliveLed 13
# define DACpin A14


int voltLUT[100] ; /* = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                     6, 26, 128, 256, 384, 512, 614, 665, 704, 742, 781,
                     806, 819, 845, 857, 870, 883, 883, 889, 902, 909,
                     915, 921, 921, 921, 921, 921, 921, 928, 941, 953,
                     998, 1024, 1088, 1152, 1408, 1856, 2175, 2559,
                     2847, 3084, 3321, 3558, 3794, 3839, 3903, 3967,
                     4031, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095
                   }; */  // Uncomment for first time load of profile..
int eeAddress = 0;
int lutIndex  = 0;

unsigned long updateMillisec = millis();
unsigned long updateInterval = 10;
char startChar ;
bool sendLutViaSerial = false;
bool getLutViaSerial = false;
char incomingByte ;
int charIndex ;
int const charCount = 1000;
bool started, ended;
char LUTFromPC[charCount];
int userArray[100];

//@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

void setup() {

  Serial.begin(9600);

  pinMode(StartPin, INPUT);
  digitalWrite( StartPin, HIGH);
  pinMode(SerComPin, INPUT);
  pinMode(CycleOnLed, OUTPUT);
  analogWriteResolution(12);

  digitalWrite(CycleOnLed, HIGH);
  delay(1000);
  digitalWrite(CycleOnLed, LOW);
}

//@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

void loop() {

  bool startState = digitalRead(StartPin);

  if (!startState) {                                  // Check Cycle Start input.
    digitalWrite(CycleOnLed, HIGH);
    if (millis() - updateMillisec > updateInterval) {
      updateMillisec = millis();
      analogWrite(DACpin, voltLUT[lutIndex]);
      Serial.println(voltLUT[lutIndex]);
      lutIndex++ ;
      if (lutIndex > 99 ) lutIndex = 99;
    }
  }
  else {                                            // Reset and wait for next cycle..
    lutIndex = 0;
    digitalWrite(CycleOnLed, LOW);
    analogWrite(DACpin, 0);
  }
  //+++++++++++++++++++++++++++++++++

  checkSerialCommand();                             // Check if Serial session is required
  //+++++++++++++++++++++++++++++++++

  if (sendLutViaSerial) {                           // SEND the LUT via Serial to the LabVIEW application
    lutIndex = 0;
    do {
      Serial.print(voltLUT[lutIndex]);
      Serial.print(',');
      lutIndex++;
    } while (lutIndex < 100);
    Serial.println('E');                           // Mark the EOF
  }
  //+++++++++++++++++++++++++++++++++

  if (getLutViaSerial) {                           // GET the LUT via Serialfrom the LabVIEW application
    while (Serial.available() > 0)                 // Is there any data in Serial buffer ??
    {
      incomingByte = Serial.read();                // Read the incoming byte .. this removes it from Serial buffer

      if (incomingByte == '<')                     // Wait for the start marker..
      {
        started = true;                            // Got the start marker. Set receive boolean to true..
        charIndex = 0;
        LUTFromPC[charIndex] = '\0';               // Throw away any incomplete characters
      }

      else if (incomingByte == '>')                // Wait for the end marker
      {
        ended = true;                              // Got the end marker ...
        break;                                     // Stop reading - exit from while loop!
      }

      else                                         // Read the message from the Serial buffer !
      {
        if (charIndex < charCount)                 // Make sure there is room
        {
          LUTFromPC[charIndex] = incomingByte;     // Add char to array
          charIndex++;
          LUTFromPC[charIndex] = '\0';             // Add NULL to end.. keep on adding
        }
      }
    }

    if (started && ended)                          // All data read from the Serial buffer... process it.
    {
      parseData();                                 // Read and store the Voltage info in variables.
      started = false;
      ended = false;
    }
  }

} //loop

//@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

void checkSerialCommand() {
  if (Serial.available() > 0) {
    startChar = Serial.read();
    if (startChar == 's') sendLutViaSerial = true;
    else if ( startChar == 'g' ) getLutViaSerial = true;
    else {
      sendLutViaSerial = false;
      getLutViaSerial = false;
    }
  }
}

//==========================

void parseData()
{
  char * strtokIndx;                                // This is used by strtok() as an index

  strtokIndx = strtok(LUTFromPC, ",");              // Get the first entry. Comma replaced by NULL after this..
  userArray[0] = atoi(strtokIndx);
  for (int count = 1; count < 100; count++ ) {
    strtokIndx = strtok(NULL, ",");
    userArray[count] = atoi(strtokIndx);
  }
  writeEEPROM();                                    // Write the look up table to the EEPROM
  readEEPROM();                                     // Update the LUT with latest pattern.
  
  Serial.print( "Got ");
  Serial.print(charIndex);
  Serial.println(" char fom PC");                   // Just to know if all went well !!
  charIndex = 0;
}
//==========================

void writeEEPROM() {
  eeAddress = 0;
  EEPROM.put(eeAddress, userArray);
}
//==========================

void readEEPROM() {
  eeAddress = 0;
  EEPROM.get(eeAddress, voltLUT);
}

The code for sending the waveform array has been done in LabVIEW and that has been tested on other serial devices for sending somewhat large amounts of data ( > 600 bytes )

Yes you can increase the RX buffer. There's lots of memory in a Teensy so you can make it stupidly big, in the thousands.

The Teensy files for this are stored (in Windows) in you user profile AppData. Due to a very early design decision, they are actually inside a folder called "avr" even though the Teensy 3.2 is not an avr.

Isn’t the “serial” port you are using actually the native USB port to the pc, on a teensy3?

I don’t think the problem is lack of buffering...

westfw:
Isn’t the “serial” port you are using actually the native USB port to the pc, on a teensy3?

I don’t think the problem is lack of buffering...

Its not the port issue, as smaller waveforms store even now and any size waveform data can be read out via Serial with no issues. Its only the large waveform data that has the issue.

And now that you mention it, I think Tx buffer size may not be the issue. My serial read is fast enough to empty the buffer much faster than it can ever fill up even with a blazing baud rate.

OK the issue had nothing to do with buffers. This function code had a bug in it :

void checkSerialCommand() {
  if (Serial.available() > 0) {
    startChar = Serial.read();
    if (startChar == 's') sendLutViaSerial = true;
    else if ( startChar == 'g' ) getLutViaSerial = true;
    else {
      sendLutViaSerial = false;
      getLutViaSerial = false;
    }

It appears that the else case was firing resetting both the entry traps for either Sending or Getting Serial data. Technically once either of these become true and the relevant function is executing there should be no possibility of this function firing. But it seems it does and resetting the Boolean getLutViaSerial.

So I moved this into the Serial read function after all parsing is all finish and the EEPROM is updated.

Problem solved.

( But of course a new problem has come in … I write an array of 100 integers received from the Serial stream into the EEPROM. Then when I read it back, I find some random , rouge entries and one of them is a float !! Next will be to kill THAT bug :frowning:

I write an array of 100 integers received from the Serial stream into the EEPROM. Then when I read it back, I find some random , rouge entries and one of them is a float !!

this is not written in the best possible way:

void parseData()
{
  char * strtokIndx;                                // This is used by strtok() as an index

  strtokIndx = strtok(LUTFromPC, ",");              // Get the first entry. Comma replaced by NULL after this..
  userArray[0] = atoi(strtokIndx);
  for (int count = 1; count < 100; count++ ) {
    strtokIndx = strtok(NULL, ",");
    userArray[count] = atoi(strtokIndx);
  }
  ...

as you force reading 100 entries in userArray without caring to look if strtokIndx is NULL... A possible proper way to write an strtok iteration would be like

void parseData()
{
  uint8_t index = 0;
  char* token = strtok(LUTFromPC, ","); // Get the first entry. Comma replaced by NULL after this.
  while (token) {
    if (index  < 100) userArray[index++] = atoi(token); // don't overflow
    token = strtok(NULL, ",");
  }

  if (index < 100) Serial.println(F("Buffer is not full. LUT did not have 100 entries"));
  else if (index >= 100) Serial.println(F("Buffer Overflow"));
  ...

if you don't have 100 data and write still the full 100 int buffer, you might have random garbage in EEPROM but as you are reading them in an int array, I don't see how you get a float... that's just not technically possible, the compiler is managing the data types...

Note as well that your initial LUT data - if you were to try to uncomment - would still not initialize the array because of the ; before the = {...})

int voltLUT[100] ; /* = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                     6, 26, 128, 256, 384, 512, 614, 665, 704, 742, 781,
                     806, 819, 845, 857, 870, 883, 883, 889, 902, 909,
                     915, 921, 921, 921, 921, 921, 921, 928, 941, 953,
                     998, 1024, 1088, 1152, 1408, 1856, 2175, 2559,
                     2847, 3084, 3321, 3558, 3794, 3839, 3903, 3967,
                     4031, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
                     4095, 4095
                   }; */  // Uncomment for first time load of profile..

Last but not least, 100 should not be all around your code as a magic number, define at the top of your program a constant of type uint8_t (max 256 entries) or size_t based on max capacity you envision and use that constant throughout the code for array size, iteration and overflow limit etc... this way if you want your change the size of the LUT you don't have to go through all the code hoping you don't forget to change one of these... that's a good practice

@J-M-L

Good points. I have never used the parsing the way you have shown. New to me. Will try out. And yes the point of using a literal constant at many places is bad coding. Will correct it.

Thanks for your time!

@J-M-L

Carried out the changes as suggested by you. But looks like the issue is elsewhere. See the two images shown below … First ..result of what I am writing to the EEPROM and Second ...what I am reading from the EEPROM.

The value at 420ms has changed from 921 to 91 ; the value at 690ms has changed from 4095 to 4.09541E+7 whatever it means ! And I have tried the W/R process atleast 10 times with same result. I tried to write to a different EEPROM address. No use.

why don't you print the userArray when you parse it and the voltLUT array when you ingest it from EEPROM? that will show you if the issue is in the array or a bug with the DAC and the       analogWrite(DACpin, voltLUT[lutIndex]);

when you mention 4095 on a 12 bit DAC this is the max value the DAC is able to receive. are you sure you did not send 4096 by any chance?

@J-M-L

As suggested by you I now have a recording of the Data As Sent by my LV App / Data Received and Parsed by the MCU / Data Sent back by the MCU after reading from Look Up Table.

Of course with no method of error correction, one would expect to see dropped or added numbers. But my situation is a little odd... if you look at the second and third plots you will notice one 91 which is circled. This needs to be 921. And another 40954095 circled. This needs to be 4095 ,4095. What is interesting is both these errors are there in all the instances I have tried apart from other random errors. Now how can a error happen at the same place, always, assuming its a transmission error.

And on a more common note how would one handle this Serial transfer to make it error free ? CRC ??
Waveform Data at different points.JPG

can you share your updated receiving code? would be good also to print the data straight as you get it, before parsing and after parsing

@J-M-L

Sure .. I have posted the code based on your suggested changes and which was used to get the data that I posted in my earlier post. Yes that is a good idea to print out before parsing... let me do that also.

I really appreciate your relentless follow up. Thanks.

#include <EEPROM.h>

# define StartPin  5
# define SerComPin 6
# define CycleOnLed 9
# define AliveLed 13
# define DACpin A14

const int lutSize = 100; 
int voltLUT[lutSize] ; 
int eeAddress = 10;
int lutIndex  = 0;

unsigned long updateMillisec = millis();
unsigned long updateInterval = 10;
char startChar ;
bool sendLutViaSerial = false;
bool getLutViaSerial = false;
char incomingByte ;
int charIndex ;
int const charCount = 500;
bool started, ended;
char LUTFromPC[charCount];
int userArray[lutSize];

//@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

void setup() {

  Serial.begin(9600);

  pinMode(StartPin, INPUT);
  digitalWrite( StartPin, HIGH);
  pinMode(SerComPin, INPUT);
  pinMode(CycleOnLed, OUTPUT);
  analogWriteResolution(12);

  digitalWrite(CycleOnLed, HIGH);
  delay(1000);
  digitalWrite(CycleOnLed, LOW);

  readEEPROM();                      // Read the EEPROM and load the Look up table
}

//@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

void loop() {
  
  bool startState = digitalRead(StartPin);

  if (!startState) {                                  // Check Cycle Start input.
    digitalWrite(CycleOnLed, HIGH);
    if (millis() - updateMillisec > updateInterval) {
      updateMillisec = millis();
      analogWrite(DACpin, voltLUT[lutIndex]);
      Serial.println(voltLUT[lutIndex]);
      lutIndex++ ;
      if (lutIndex > lutSize ) lutIndex = lutSize;
    }
  }
  else {                                            // Reset and wait for next cycle..
    lutIndex = 0;
    digitalWrite(CycleOnLed, LOW);
    analogWrite(DACpin, 0);
  }
  //+++++++++++++++++++++++++++++++++

  checkSerialCommand();                             // Check if Serial session is required
  //+++++++++++++++++++++++++++++++++

  if (sendLutViaSerial) {                           // SEND the LUT via Serial to the LabVIEW application
    lutIndex = 0;
    do {
      Serial.print(voltLUT[lutIndex]);
      Serial.print(',');
      lutIndex++;
    } while (lutIndex < lutSize);
    sendLutViaSerial = false;
  }
  //+++++++++++++++++++++++++++++++++

  if (getLutViaSerial) {                           // GET the LUT via Serialfrom the LabVIEW application
    while (Serial.available() > 0)                 // Is there any data in Serial buffer ??
    {
      incomingByte = Serial.read();                // Read the incoming byte .. this removes it from Serial buffer

      if (incomingByte == '<')                     // Wait for the start marker..
      {
        started = true;                            // Got the start marker. Set receive boolean to true..
        charIndex = 0;
        LUTFromPC[charIndex] = '\0';               // Throw away any incomplete characters
      }

      else if (incomingByte == '>')                // Wait for the end marker
      {
        ended = true;                              // Got the end marker ...
        break;                                     // Stop reading - exit from while loop!
      }

      else                                         // Read the message from the Serial buffer !
      {
        if (charIndex < charCount)                 // Make sure there is room
        {
          LUTFromPC[charIndex] = incomingByte;     // Add char to array
          charIndex++;
          LUTFromPC[charIndex] = '\0';             // Add NULL to end.. keep on adding
        }
      }
    }

    if (started && ended)                          // All data read from the Serial buffer... process it.
    {
      parseData();                                 // Read and store the Voltage info in variables.
      started = false;
      ended = false;
      getLutViaSerial = false;
    }
  }

} //loop

//@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

void checkSerialCommand() {
  if (Serial.available() > 0) {
    startChar = Serial.read();
    if (startChar == 's') sendLutViaSerial = true;
    else if ( startChar == 'g' ) getLutViaSerial = true;
  }
}

//==========================

void parseData()
{
  char * strtokIndx;                                // This is used by strtok() as an index
  int index = 0; 
  strtokIndx = strtok(LUTFromPC, ",");              // Get the first entry. Comma replaced by NULL after this..
  userArray[0] = atoi(strtokIndx);

  while (strtokIndx) {
    if (index  < lutSize-1) userArray[index++] = atoi(strtokIndx); // Do not overflow
    strtokIndx = strtok(NULL, ",");
    Serial.print(userArray[index]);                 // Use for debugging...
    Serial.print(','); 
  }
  
  writeEEPROM();                                    // Write the look up table to the EEPROM
  readEEPROM();                                     // Update the LUT with latest pattern.
  
  Serial.print( "Got ");
  Serial.print(charIndex);
  Serial.print(" char fom PC....");
  
  if (index < lutSize) Serial.println(F("Buffer is not full. LUT did not have 100 entries"));
  else if (index >= lutSize) Serial.println(F("Buffer Overflow"));
  charIndex = 0;
}
//==========================

void writeEEPROM() {
  eeAddress = 0;
  EEPROM.put(eeAddress, userArray);
}
//==========================

void readEEPROM() {
  eeAddress = 0;
  EEPROM.get(eeAddress, voltLUT);
  //  for ( int count = 0; count < lutSize; count++) {    // Only to debug..
  //    Serial.println(voltLUT[count] );
}

//============E N D===========

@X-M-L

And here is the data as received by the MCU before parsing. For sure the two persistent errors seems to happen at the stage of the data coming in. If only I can crack this then I guess the code can be wrapped up. And just to rule out the LabVIEW side I generated a different profile and sent. It had errors at various other points and not at the 420ms and 690ms places.

DataAsRecedByMCU.JPG

any chance you could update Serial to 115200 instead of 9600 ?

note that you are overflowing at the end of your array

      if (lutIndex > lutSize ) lutIndex = lutSize;

also lutIndex is used in many places which could create issues. use local variables when possible

also your getting ascii from the serial input is not great. You really should really iterate until you get the proper end char, not just Serial is empty. This is a recipe for failure and I think this is where your bug is (likely) - you've emptied the buffer but you did not receive everything you needed.

I refactored your code a bit (done here, so did not test at all)

#include <EEPROM.h>

const byte StartPin =  5;
const byte SerComPin = 6;
const byte CycleOnLed = 9;
const byte AliveLed = 13;
const byte DACpin = A14;

const size_t lutSize = 100;
uint16_t voltLUT[lutSize] ;
uint16_t userLUT[lutSize];

const uint16_t eeAddress = 10;

bool sendLutViaSerial = false;
bool getLutViaSerial = false;

const size_t maxIncomingCount = 1000;
char ASCII_LUTFromPC[maxIncomingCount + 1];


//==========================
void printVoltLUT()
{
  for (size_t count = 0; count < lutSize; count++) {
    Serial.print(voltLUT[count]);
    if (count != lutSize - 1) Serial.write(',');
  }
  Serial.println();
}

void writeEEPROM() {
  EEPROM.put(eeAddress, userLUT);
}

void readEEPROM() {
  EEPROM.get(eeAddress, voltLUT);
  printVoltLUT();   // --- DEBUG ---
}

void parseData()
{
  size_t index = 0;
  char* strtokPtr = strtok(ASCII_LUTFromPC, ","); // Get the first entry. Comma replaced by NULL after this..
  while (strtokPtr) {
    if (index  < lutSize) userLUT[index++] = atoi(strtokPtr); // Do not overflow
    strtokPtr = strtok(NULL, ","); // get the next one
    // --- DEBUG ---
    Serial.print(userLUT[index]);
    if (index != lutSize - 1) Serial.write(',');
    // -------------
  }
  Serial.println(); // --- DEBUG ---

  writeEEPROM(); // Write the look up table to the EEPROM
  readEEPROM(); // Update the LUT with latest pattern.

  // --- DEBUG ---
  Serial.print(F("Got "));
  Serial.print(index);
  Serial.println(F(" values fom PC...."));

  if (index < lutSize) Serial.println(F("ASCII LUT did not have enough entries"));
  else if (index >= lutSize) Serial.println(F("Buffer Overflow"));
  // -------------
}

void checkSerialCommand()
{
  static bool started = false;
  static size_t incomingLUTIndex = 0;

  if (getLutViaSerial) { // we are receiving a LUT, GET the LUT via Serialfrom the LabVIEW application
    while (Serial.available() > 0) {  // Is there any data in Serial buffer ??
      char incomingByte = Serial.read(); // Read the incoming byte .. this removes it from Serial buffer

      if (!started) {
        if (incomingByte == '<') { // Wait for the start marker..
          started = true; // Got the start marker, ready to ingest
          incomingLUTIndex = 0;
          ASCII_LUTFromPC[0] = '\0'; // initial cString empty
        }
      } else // we are started
        if (incomingByte == '>') { // is it the end marker
          ASCII_LUTFromPC[incomingLUTIndex] = '\0'; // Add NULL to end to form proper cString
          started = false;
          getLutViaSerial = false;
          parseData();
          break; // Stop reading - exit from while loop!
        } else { // Read the message from the Serial buffer !
          if (incomingLUTIndex < maxIncomingCount) {  // Make sure there is room
            ASCII_LUTFromPC[incomingLUTIndex++] = incomingByte;  // Add char to array
          } else Serial.println(F("LUT ASCII Buffer overflow!"));
        }
    }
  } else { // check commands
    if (Serial.available() > 0) {
      char startChar = Serial.read();
      if (startChar == 's') printVoltLUT();
      else if ( startChar == 'g' ) getLutViaSerial = true;
    }
  }
}

void checkStartPin()
{
  static uint32_t updateMillisec = 0;
  const uint32_t updateInterval = 10ul;
  static size_t lutPlayBackIndex  = 0;

  if (digitalRead(StartPin) == LOW) {  // Check Cycle Start input.
    digitalWrite(CycleOnLed, HIGH);
    if (millis() - updateMillisec > updateInterval) {
      analogWrite(DACpin, voltLUT[lutPlayBackIndex]);
      updateMillisec = millis();
      Serial.println(voltLUT[lutPlayBackIndex]); // debug
      if (++lutPlayBackIndex >= lutSize ) lutPlayBackIndex = lutSize - 1; // don't read past the end of the LUT
    }
  } else {  // Reset and wait for next cycle..
    lutPlayBackIndex = 0;
    digitalWrite(CycleOnLed, LOW);
    analogWrite(DACpin, 0);
    updateMillisec = 0;
  }
}

//==========================

void setup()
{
  Serial.begin(115200);
  pinMode(StartPin, INPUT_PULLUP);

  pinMode(SerComPin, INPUT);
  pinMode(CycleOnLed, OUTPUT);

  analogWriteResolution(12);

  digitalWrite(CycleOnLed, HIGH);
  delay(1000);
  digitalWrite(CycleOnLed, LOW);

  readEEPROM();// Read the EEPROM and load the voltLUT Look up table
}


void loop() {
  checkSerialCommand(); // monitor Serial often to not loose anything
  checkStartPin(); // do we need to drive the output?
}

could you check if that works any better? (if you can't get 115200 bauds, set Serial back at 9600)

if the code compiles and run, share the FULL console output - including comments on how many data have been sent etc...

@J-M-L

Your code did the trick. Yes it runs great @ 115200 baud and there is no aberration as earlier. One of the ways I check the integrity of the Write / read to the MCU is to repeatedly click the Write and Read buttons on my LabVIEW application. Earlier within about three iterations the profile will be completely jumpled up due to the accumulated errors.

But now I can keep doing the whole day and the profile remains as is. Good lesson learnt … came this far but missed out on the array count and Serial read method. Thanks for the guided help.

Attaching the resulting console output below :
PerfectSerialRead_Write.JPG

great !

...If you can't get 115200 bauds, set Serial back at 9600

Just want to mention that Teensyduino completely ignores the Baud rate setting on Serial. It always transmits with the full USB speed.