Reduce Read,Write,Send over SoftwareSerial

[pathogenex]

Hello humans,
I've been using arduino + HM10 BLE to talk with an MIT AI2 app. I have the basic setup as follows:

• Arduino sends the app the initial variables (Mode,speed,brightness etc)
• App tells arduino when to change a value and to send updated variables after change

Now im running into some serious lag and delay issues and eventual hanging of the arduino when the App sends multiple requests.

Here is a copy of my entire code with a millis readout added to the main loop to allow me to attempt to diagnose the problem.

#include "FastLED.h"
#include <SoftwareSerial.h>

#define NUM_LEDS 144 ///144
#define PIN       3     // LED signal Pin

SoftwareSerial HM10(9, 8);///9 (RX) ///8 (TX)
CRGB pixels[NUM_LEDS];

int     leds[294]; ///For Aligning Vertical/Horizontal Motion /// NUMBER OF LEDS IN ONE CIRCLE~?leds[NUM_LEDS/4];
int     max_overall_brightness = 255;
int     overall_brightness = 255;
float   red_max = 1;
float   green_max = 1;
float   blue_max = 1;

int red;
int green;
int blue;

int     animation_rate = 5;
int     MAXanimation_rate = 10;
int     mode = 0; // Animation mode index
int     MAXmode = 14; // max mode index

int incoming_command = 0;
int animation_speed = 2;

// These variables are used within the animation methods.
// They are scoped by name, but global in the sketch,
// so that animations will not restart when commands are received.

///////////////////////////////Serial BT Receiver
int Power;
const byte numChars = 32; //array size limiter
char receivedChars[numChars]; // Array of received data
boolean RecNewData = false;
boolean SendNewData = false;
static byte ndx = 0; // index size of array

bool modechanged = true;
bool animchanged = true;
bool Brightchanged = true;
bool Powerchanged = true;

char CSV[2] = {',','\0'};
//char AAA[4] = {'M', '1', '3','\0'};
//char BBB[3] = {'S', '4','\0'};
//char CCC[4] = {'V', '8','0','\0'};
//char DDD[3] = {'P', '1','\0'};
///////////////////////////////

    
//BeaconEDIT Vars
int VerticalLEDs = 6; 
int HorizontalLEDs = 24; //LEDS in one circle

// Beacon Vars
double beacon_angle = 0;
int    beacon_led_angle = 0;
double beacon_left_distance = 0;
double beacon_right_distance = 0;
double beacon_absolute_distance = 0.0;
int    beacon_beam_width = 10; // in degrees
int    beacon_beam_width_Half = (HorizontalLEDs / VerticalLEDs);
int    beacon_falloff = 35; // in degrees
int    beacon_pixel_brightness = 0; // in degrees


volatile boolean animate = true;
volatile long animation_change_timeout;



void setup() {
  LEDS.setBrightness(255);
  FastLED.addLeds<WS2812B, PIN, GRB>(pixels, NUM_LEDS).setCorrection(TypicalLEDStrip);
  FastLED.setMaxPowerInVoltsAndMilliamps(5, 700);    // limit power of LED strip to 5V, 500mA
  FastLED.clear();

  Serial.begin(9600);
  HM10.begin(9600);

}



void loop() {
  unsigned long start = micros();
  recvWithStartEndMarkers(); // Receive BT serial data
  unsigned long end = micros();
  unsigned long delta = end - start;
  Serial.print("SerialBufferScan = ");
  Serial.println(delta);
  
  unsigned long start2 = micros();
  RecNewDataFunc(); // Use BT serial Data
  unsigned long end2 = micros();
  unsigned long echo = end2 - start2;
  Serial.print("SerialDataStore = ");
  Serial.println(echo);
  
  unsigned long start3 = micros();
  SendNewDataFunc();
  unsigned long end3 = micros();
  unsigned long fox = end3 - start3;
  Serial.print("SerialDataSend = ");
  Serial.println(fox);
  

  switch (incoming_command) {      
    
    // Color Buttons
    case 0:
      red_max = 1;
      blue_max = 0;
      green_max = 0;
      break;
    case 99:

      break;
    default:
      
      break;         
  }   
  
  switch (mode) {
    /// beacon(int beams,int vertical);
    case 0:
      lamp ();      
      break;
    case 1:
      beacon(1);
      break;
    case 2:
      beacon(2);
      break;    
    case 3:
      beacon(3);
      break;  
    case 4:
      beacon(4);
      break;  
    case 5:
      beacon(5);
      break;        
    case 6:
      beacon(6);
      break;                
    case 7:
      beacon(7);
      break;                
    case 8:
      beacon(8);
      break;
    case 9:
      beacon(9);
      break;
    case 10:
      beacon(10);
      break;
    case 11:
      beacon(11);
      break; 
    case 12:
      beacon(12);
      break;   
    case 13:
      lamp (); 
      break; 
    case 14:
      lamp (); 
      break;       
    case 99:
      rest ();                 
      break;
    default:

      break;    
  }

  animation_change_timeout = 0;
  animate = true;
}

void recvWithStartEndMarkers() {
    static boolean recvInProgress = false;
    ndx = 0;
    char startMarker = '<';
    char endMarker = '>';
    char IncomingChar;
 
    while (HM10.available() > 0 && RecNewData == false) {
        IncomingChar = HM10.read();

        if (recvInProgress == true) { // If receiving
            if (IncomingChar != endMarker)  { // if incoming is not line end marker
                receivedChars[ndx] = IncomingChar; // index chars in array 
                ndx++; // increase index number
                if (ndx >= numChars) { // if index exceeds byte limit
                    ndx = numChars - 1; // set index to re-write over last char in array
                }
            }
            else {
                receivedChars[ndx] = '\0'; // terminate the string 
                recvInProgress = false; 
                //ndx = 0; //reset index value
                RecNewData = true; // enable data release
            }
        }

        else if (IncomingChar == startMarker) { /// if string start marker found
            recvInProgress = true; // enable data copy proccess
//            Serial.println(recvInProgress);
        }
    }
}

void RecNewDataFunc() { /// Find values in received Data
    if (RecNewData == true) {
//        Serial.print("This just in ... ");
//        Serial.println(receivedChars);
          char ControlValue;
        
        if (receivedChars[0] == 77){ // if mode('M') char found ASCII    
//          mode = atoi(&receivedChars[1]); /// convert chars from 1 upwards       
//          Serial.print("mode:"); // print mode result
//          Serial.println(mode); // print mode result  
          ControlValue = receivedChars[1]; /// convert chars from 1 upwards       
          modechanged = true;
          if (ControlValue == '+'){
            if ((mode < MAXmode)&&(mode != 99)){
              mode += 1;
            }
            else if (mode == 99){
            mode = 0;
            }            
          }
          else if (ControlValue == '-'){
            if ((mode > 0)&&(mode != 99)){
              mode -= 1;  
            }
            else if (mode == 99){
            mode = 0;
            }
          }
//          Serial.print("mode:"); // print mode result
//          Serial.println(mode); // print mode result            
//          Serial.print("ControlValue:"); // print mode result
//          Serial.println(ControlValue); // print mode result  
//          Serial.print("modechanged:"); // print mode result
//          Serial.println(modechanged); // print mode result            
        }
        
        else if (receivedChars[0] == 83){ // if speed('S') char found ASCII
//          animation_rate = atoi(&receivedChars[1]); /// convert chars from 1 upwards  
          ///Serial.print("animation_rate:"); // print mode result
          ///Serial.println(animation_rate); // print mode result   
          animchanged = true;
          ControlValue = receivedChars[1]; /// convert chars from 1 upwards       
          if (ControlValue == '+'){
            if (animation_rate < MAXmode){
              animation_rate += 1;
            }
          }
          else if (ControlValue == '-'){
            if (MAXanimation_rate > 0){
              animation_rate -= 1;  
            }
          }
//          Serial.print("animation_rate:"); // print mode result
//          Serial.println(animation_rate); // print mode result 
//          Serial.print("ControlValue:"); // print mode result
//          Serial.println(ControlValue); // print mode result  
//          Serial.print("animchanged:"); // print mode result       
//          Serial.println(animchanged); // print mode result             
        }
       
        
        else if (receivedChars[0] == 86){ // if brightness('V') char found ASCII
//          int overall_brightness_Rec = atoi(&receivedChars[1]); /// convert chars from 1 upwards          
//          overall_brightness = (overall_brightness_Rec * 2.55);
//          Serial.print("overall_brightness:"); // print mode result
//          Serial.println(overall_brightness); // print mode result
            Brightchanged = true;
            ControlValue = receivedChars[1]; /// convert chars from 1 upwards       
          if (ControlValue == '+'){
            if (overall_brightness < max_overall_brightness){
              overall_brightness += 5;
            }
          }
          else if (ControlValue == '-'){
            if (overall_brightness > 5){
              overall_brightness -= 5;  
            }
          }
//          Serial.print("overall_brightness:"); // print mode result
//          Serial.println(overall_brightness); // print mode result    
//          Serial.print("ControlValue:"); // print mode result
//          Serial.println(ControlValue); // print mode result  
//          Serial.print("Brightchanged:"); // print mode result       
//          Serial.println(Brightchanged); // print mode result                       
          }
        
        else if (receivedChars[0] == 82){ // if red('R') char found ASCII
          int red_Rec = atoi(&receivedChars[1]); /// convert chars from 1 upwards          
          green_max = ((float)red_Rec / 255); /// convert 255 into decimal for animations
//          Serial.print("red_max:"); // print mode result
//          Serial.println(red_max); // print mode result
          incoming_command = 99;

          }        
        
        else if (receivedChars[0] == 71){ // if green('G') char found ASCII
          int green_Rec = atoi(&receivedChars[1]); /// convert chars from 1 upwards          
          red_max = ((float)green_Rec / 255); /// convert 255 into decimal for animations
//          Serial.print("green_max:"); // print mode result
//          Serial.println(green_max); // print mode result
          incoming_command = 99;
          }
        
        else if (receivedChars[0] == 66){ // if blue('B') char found ASCII
          int blue_Rec = atoi(&receivedChars[1]); /// convert chars from 1 upwards          
          blue_max = ((float)blue_Rec / 255); /// convert 255 into decimal for animations
//          Serial.print("blue_max:"); // print mode result
//          Serial.println(blue_max); // print mode result
          incoming_command = 99;
          }
          
        else if (receivedChars[0] == 80){ // if Power('P') char found ASCII
          Powerchanged = true;
          ControlValue = receivedChars[1]; /// convert chars from 1 upwards       
          if (ControlValue == '+'){
              mode = 0;  
              Power = 1;
          }
          else if (ControlValue == '-'){
              mode = 99; 
              Power = 0;
          }
//          Serial.print("Power:"); // print mode result
//          Serial.println(Power); // print mode result          
//            Serial.print("ControlValue:"); // print mode result
//            Serial.println(ControlValue); // print mode result  
//            Serial.print("Powerchanged:"); // print mode result       
//            Serial.println(Powerchanged); // print mode result       
        }
//        else if (receivedChars[0] == 88){ // if SendValuesRequest('X') char found ASCII
//        SendNewData = true;
//        }        
        RecNewData = false;
        SendNewData = true;
    }
}

void SendNewDataFunc() { /// Find values in received Data
    if (SendNewData == true) {
          int BrightnessConversion = (overall_brightness / 2.55);
          char BUFFER[6];
          
          if (modechanged == true){
          HM10.write('M');
          HM10.write(itoa((mode),BUFFER,10));
          HM10.write(CSV);
//          HM10.write(itoa((mode += 'M') += ','),BUFFER,10);
          }
          
          if (animchanged == true){
          HM10.write('S');
          HM10.write(itoa(animation_rate,BUFFER,10));
          HM10.write(CSV);          
//          HM10.write(itoa((animation_rate += 'S') += ','),BUFFER,10);
          }
          
          if (Brightchanged == true ){
          HM10.write('V');
          HM10.write(itoa(BrightnessConversion,BUFFER,10));
          HM10.write(CSV);    
            
//          HM10.write(itoa((BrightnessConversion += 'V') += ','),BUFFER,10);
          }
          
          if (Powerchanged == true){
          HM10.write('P');
          HM10.write(itoa(Power,BUFFER,10));
          HM10.write(CSV);     
//          HM10.write(itoa((Power += 'P') += ','),BUFFER,10);     
          }
        
        modechanged = false;
        animchanged = false;
        Brightchanged = false;
        Powerchanged = false;
        SendNewData = false;
    }
}

void break_for_input() {
 if (animation_change_timeout > 100) {
      animate = false;
 }
}

void change_animation_rate(bool dir) {

  if (dir == 0){
    if (animation_rate < MAXanimation_rate) {
     animation_rate++;
    }
  }
  else if (dir == 1){
    if (animation_rate < 0) {
     animation_rate--;
    }
  }
}

void rest () {
  long loop_start = millis();
  while(animate) {
    for(int i=0; i<NUM_LEDS; i++) {
      pixels[i] = CRGB(0, 0, 0);
    }
    FastLED.show();
    animation_change_timeout = millis() - loop_start;
    
    EVERY_N_MILLIS(100){
    break_for_input();
    }

  }
}
  
void lamp () {
  long loop_start = millis();
  while(animate) {
    //Some quick and dirty grascale whitebalance and min brigtness corrections
    red = max_overall_brightness;
    green = max_overall_brightness - 3/max_overall_brightness;
    if (green < 0) green = 0;
    if (max_overall_brightness > 1 && green < 2) green = 2;
    blue = max_overall_brightness * .45;
    if (max_overall_brightness > 1 && blue < 2) blue = 2;
      
    // Aapply color maximums
    red *= red_max * (overall_brightness / 255.0);
    green *= green_max * (overall_brightness / 255.0);
    blue *= blue_max * (overall_brightness / 255.0);
    
    for(int i=0; i<NUM_LEDS; i++) {
      pixels[i] = CRGB(green, red, blue);
    }
    FastLED.show();
    animation_change_timeout = millis() - loop_start;
    EVERY_N_MILLIS(100){
    break_for_input();
    }
  }
}


void beacon(int beams) {
  // Set start time for minimum time to chenge interval
  long loop_start = millis();

  while (animate) {
    for(int i=0; i<NUM_LEDS; i++) {
      
      // Beacon angle is normalized to -180 to 180 degrees to make angular distance calc easier
      //Find the angle on the tube of the current LED
//      beacon_led_angle = fmod((i * (360.0 / 11.41)), 360.0); ///OG VALUES FMOD =  https://en.cppreference.com/w/c/numeric/math/fmod
        
        ////////Beacon angle used to change animation types//////////
        
        if (beams == 1){ //SingleBeam
          beacon_led_angle = fmod(i * (360 / HorizontalLEDs), 360);
        }
        else if (beams == 2){ //DoubleBeam
          beacon_led_angle = fmod((i * (360 / (HorizontalLEDs /2) )), 360); 
        }
        else if (beams == 3){ // Meteors
          beacon_led_angle = fmod((i * (360 / (HorizontalLEDs - VerticalLEDs) ) ), 360); 
        }
        else if (beams == 4){ //Falling Spiral
          beacon_led_angle = fmod((i * (360 / ((HorizontalLEDs - VerticalLEDs)/2) )), 360); 
        }
        else if (beams == 5){ //Climbing Spiral
          beacon_led_angle = fmod((i * (360 / ((HorizontalLEDs - VerticalLEDs)/3) )), 360); 
        }
        else if (beams == 6){ // ODD/EVEN Flash
          beacon_led_angle = fmod((i * (360 / ((HorizontalLEDs - VerticalLEDs)/VerticalLEDs) )), 360); 
        }
        else if (beams == 7){ // Climbing Ring
          beacon_led_angle = fmod(i * (360 / (HorizontalLEDs * VerticalLEDs)), 360); 
        }  
        else if (beams == 8){ // Beacon Flash Fade
          beacon_led_angle = fmod(i * ((HorizontalLEDs * VerticalLEDs)/360), 360); 
        }                                       
        else if (beams == 9){ // Falling Ring
          beacon_led_angle = fmod(i * (360 * HorizontalLEDs), 360);
        }                                       
        else if (beams == 10){ // Thick Falling Ring
         beacon_led_angle = fmod((i * (360 * (HorizontalLEDs / VerticalLEDs) )), 360); 
        }     
        else if (beams == 11){ // Rain
         beacon_led_angle = fmod((i * (360 * (HorizontalLEDs*(HorizontalLEDs*2)) )), 360); 
        }
        else if (beams == 12){ // TwistFill
         beacon_led_angle = fmod(i * (i/HorizontalLEDs), 45);
        }        
        
                                                  
      //Normalize heading to -180 to 180 for angular distance to beacon calc
      if (beacon_led_angle < beacon_angle) beacon_led_angle += 360;
      beacon_left_distance = beacon_led_angle - beacon_angle;
      
      if (beacon_left_distance < 180) {
        beacon_absolute_distance = abs(beacon_left_distance);
      } else {
        beacon_absolute_distance = abs(360.0 - beacon_left_distance);
      }
      
      //Find pixel brightness given beam characteristics
      if (beacon_absolute_distance <= beacon_beam_width_Half) {
        beacon_pixel_brightness = overall_brightness;
      } else if (beacon_absolute_distance <= (beacon_beam_width_Half + beacon_falloff)) {
        beacon_pixel_brightness = overall_brightness - ((overall_brightness/(float)beacon_falloff) * (beacon_absolute_distance - beacon_beam_width_Half));
      } else {
        beacon_pixel_brightness = 0;
      }

      

      //Some quick and dirty grascale whitebalance and min brightness corrections
      red = beacon_pixel_brightness;
      green = beacon_pixel_brightness - 3/beacon_pixel_brightness;
      if (green < 0) green = 0;
      if (beacon_pixel_brightness > 1 && green < 2) green = 2;
      blue = beacon_pixel_brightness * .45;
      if (beacon_pixel_brightness > 1 && blue < 2) blue = 2;
      
      //Finally, apply color maximums
      red *= red_max;
      green *= green_max;
      blue *= blue_max;
      
      //Assign color
      pixels[i] = CRGB(green, red, blue);
    }
    
 
      beacon_angle += (0.05 * (animation_rate * animation_rate * 3));
      FastLED.show();
        if (beacon_angle >= 360) {
          beacon_angle -= 360;
        }
      
    animation_change_timeout = millis() - loop_start;
    break_for_input();
   }
}

From the app the longest string sent is "<R255>" (But will be in groups of 3 - RGB values)
From the arduino the longest string sent is "M20," (4 chars? )

If someone can give me advise on the best course of action that would be great.
Any options need to be non-blocking and I am currently considering:

• Create a storage variable inside the App with a clock delay to only send one snippet at a time.
  (I would prefer to have limited delay between button presses and arduino action so not prefferable)
• Honestly thats all i've got. SerialOptimisation isn't something i've ever considered...

Here is an example of the times taken once inputs are received and sent. (20,5,5 is idle jumps to 100s to start with then spikes into the 10000's)

[qubits-us]

Sounds like a job for AckMe..
Maybe it gives you some ideas..

good luck.. ~q

[Railroader]

For the debugging texts, and other texts, crank up serial.begin. 115200 usually works well and outputs data faster.

[groundFungus]

Can you change the baud rate of the HM10? I have used SoftwareSerial up to 38400 baud with success. That is the fastest for reliable communication that I have found.

The FastLED show() function disables interrupts while it executes. Anything coming into the SoftwareSerial port while interrupts are disabled will be ignored or corrupted.

[pathogenex]

@qubits-us

Sounds like a job for AckMe..
Maybe it gives you some ideas..

good luck.. ~q

What is ackme? From the single picture i'd guess its a baudrate speed tester?

[pathogenex]

@groundFungus

Can you change the baud rate of the HM10? I have used SoftwareSerial up to 38400 baud with success. That is the fastest for reliable communication that I have found.

So all of you believe its down to the baudrate? I was reluctant to increase it as i found hundreds of nightmare posts of people who tried but it seems simple enough.

Alright i did it. Yup higher baud = beauty.

Thanks for adept quick replies humans, your help is much appreciated!

On the same topic. Anyone know the max safe baudrate of a nano? Probably a clone.

To test for it do i just crank up Serial.begin and see where it starts pooping itself?

[pathogenex]

@groundFungus

The FastLED show() function disables interrupts while it executes. Anything coming into the SoftwareSerial port while interrupts are disabled will be ignored or corrupted.

Would it be advised to disable serial before using that function then re-enabling it afterwards? or would that still cause problems?

stupid question as with serial disabled it cant copy into its buffer... Good to know that so i can find a solution Tyvm

[qubits-us]

It can take thousands of commands preloaded into a data file and sends one chunk at a time waiting for an ACK before sending next chunk..
~q

[sterretje]

Your best bet is to send one byte / character from the app, receive it at the Arduino and echo it back to the sender. Only when the sender has received the echo, it will send the next byte / character.

Updating 144 NeoPixels takes 8 times 1.25 microseconds for one colour in a pixel; times 3 for 3 colours and times 144 for the full strip. That is 4.3 milliseconds. During that time you will not be able to much anything as interrupts are disabled as mentioned by @groundFungus.

One character at 9600 baud takes approximately 1 milliseconds, you're sending four or more so if they arrive in those 4.3 milliseconds you have a problem.

[pathogenex]

This is some damned usefull information. Thank you very much. Ive managed to punch the baudrate up to 57600 and it seems stable. there is still input lag but what you're saying should help me mitigate that even further. Do you have any examples of "echoing bytes" to hand?
If not i'm sure i can find some.

[sterretje]

Below is part from recvWithStartEndMarkers()

  while (HM10.available() > 0 && RecNewData == false)
  {
    IncomingChar = HM10.read();

    if (recvInProgress == true)
...
...

To echo, just send the receide char back

  while (HM10.available() > 0 && RecNewData == false)
  {
    IncomingChar = HM10.read();
    HM10.write(IncomingChar);    // <<================

    if (recvInProgress == true)
...
...

[pathogenex]

@sterretje

To echo, just send the receide char back

And then i would limit the number of bytes sent, from within the app to the full buffer size of the arduino serial?

E.g:
If (setbytes <= buffersize) {
BLE.write(bytes)

Stupid response again.

I want to respond to each byte, as this will also enable me to setup a resend incase the byte gets lost or corrupted.

[sterretje]

No, you send one character and wait for the echo.

In Arduino code it would look something like below; not optimised so it's blocking code.

char message[] =  "abcd";
// send message byte by byte
for(uint cnt = 0; cnt<strlen(message);cnt++)
{
  // send byte
  Serial.write(message[cnt]);
  // wait for echo
  while(Serial.available() == 0) ;
  char ch = Serial.read();
  
  // validation
  if(ch != message[cnt])
  {
    // something is wrong.
  }
}

For you to translate to MIT and fiddle a bit more.

[pathogenex]

@sterretje

For you to translate to MIT

ahaha, i hate MIT block setup. Its really nice for setting up UI and interaction but i wish it had a scripting function for things like this.
If you're bored check out this post and see the monstrosity i created at last post to handle incomming data in AI2

Would you advise doing the same in arduino to handle incoming serial data as well?

[groundFungus]

I looked at the MIT block language and had a hard time understanding how to use it. I admit that I did not spend a lot of time at it. After using Labview for years, I though it would be easier.

[pathogenex]

It just involves a huge amount of repetition for tasks that in scripting languages are trivial.

It also doesnt help that lists(arrays) start at index 1

[pathogenex]

@sterretje

No, you send one character and wait for the echo.

A question i have. Does this work for two way communcation? Or what is the "official" term for byte echoing so i can search for myself

[groundFungus]

Maybe a kind of "handshaking".

[pathogenex]

plenty of names i can think of but they dont show to be much of use for AI2 and BLE. It's not got the best information and alot the info is very outdated.

"Callback, Handshake, ByteBack etc etc"

[sterretje]

@pathogenex

Does this work for two way communcation?

I do not quite understand. But you only have to do this when you send from your MIT AI2 app to the Arduino. You do not have to do this when you send from the Arduino to the MIT AI2 app unless that app also takes ages to process the incoming data.

The data that you send from the Arduino is stored in a buffer that is emptied using interrupts (at least that is for Serial, I've never studied SoftwareSerial in detail but I think it works the same). If you update the strip, the interrupts are disabled by FastLED.show() which stops the transfer; when the update of the strip is finished, interrupts are enabled again by FastLED.show() and transfer resumes.

There however is possibly one problem and that is that the bit timing in SoftwareSerial might also be interrupt driven; if so, that will bugger up the byte tat is being transferred. The way around that is to use a HM10.flush() after you have written a message. e,g,

HM10.write('S');
HM10.write(itoa(animation_rate,BUFFER,10));
HM10.write(CSV);
HM10.flush();
// not sure if this is needed
// add a one character transfer time (1ms @ 9600 baud)
delay(1);

The flush will prevent the code to continue till the data is transferred; the additional 1 millisecond delay makes sure that the last byte (if bit timing is interrupt driven) is also transferred without interruption.