Object Tracking With Pyserial & Arduino Keeps Freezing

Hey everyone,

I'm using Opencv in Python to track objects through a webcam and send there location to an Arduino. Currently I'm getting it to send data but after about a minute everything freezes, if I move the objects out of the webcam after it freezes it will start running again. I think it's because the buffer is getting overflowed, Python is sending data to fast to the Arduino but I'm not sure.

//Arduino Code
char Input[50];
int input_counter = 0;

float FBigx_coordinate = 0;
float FBigy_coordinate = 0;
float SBigx_coordinate = 0;
float SBigy_coordinate = 0;

float FSmallx_coordinate = 0;
float FSmally_coordinate = 0;
float SSmallx_coordinate = 0;
float SSmally_coordinate = 0;
float Robot_xcoordinate = 0;
float Robot_ycoordinate = 0;
float BotAngle = 0;
float angleOfFirstBL = 0;
float angleOfSecondBL = 0;
float angleOfFirstS = 0;
float angleOfSecondS = 0;
int Stopper = 0;

void setup() {
  // put your setup code here, to run once:
Serial.begin(115200);
}

void loop() {
  // put your main code here, to run repeatedly:
    ReadSerial();
  
  

}


void ReadSerial()
{
  while(Serial.available() > 0)
  {
    Input[input_counter]=Serial.read();
    if(Input[input_counter]== '\n') //newline indicates end of a message
    {
      if(Input[input_counter-1]== 'x') //message type is x, so handle that message
      {
        FBigx_coordinate = atof(Input);
        Serial.print("First Big Lego x: ");
        Serial.println(FBigx_coordinate);
        delay(100);
      }
      if(Input[input_counter-1] == 'y')
      {
        FBigy_coordinate = atof(Input);
        Serial.print("First Big Lego y: ");
        Serial.println(FBigy_coordinate);
        delay(100);
      }
      if(Input[input_counter-1]== 'q') //message type is x, so handle that message
      {
        SBigx_coordinate = atof(Input);
        Serial.print("Second Big Lego x: ");
        Serial.println(SBigx_coordinate);
        delay(100);
      }
      if(Input[input_counter-1] == 'w')
      {
        SBigy_coordinate = atof(Input);
        Serial.print("Second Big Lego y: ");
        Serial.println(SBigy_coordinate);
        delay(100);
      }
      if(Input[input_counter-1]== 'e') //message type is x, so handle that message
      {
        FSmallx_coordinate = atof(Input);
        Serial.print("First Small Lego x: ");
        Serial.println(FSmallx_coordinate);
        delay(100);
      }
      if(Input[input_counter-1] == 'r')
      {
        FSmally_coordinate = atof(Input);
        Serial.print("First Small Lego y: ");
        Serial.println(FSmally_coordinate);
        delay(100);
      }

       if(Input[input_counter-1]== 't') //message type is x, so handle that message
      {
        SSmallx_coordinate = atof(Input);
        Serial.print("Second Small Lego x: ");
        Serial.println(SSmallx_coordinate);
        delay(100);
      }
      if(Input[input_counter-1] == 'l')
      {
        SSmally_coordinate = atof(Input);
        Serial.print("Second Small Lego y: ");
        Serial.println(SSmally_coordinate);
        delay(100);
        
      }
      if(Input[input_counter-1] == 'u')
      {
        Robot_xcoordinate = atof(Input);
        Serial.print("Robot x: ");
        Serial.println(Robot_xcoordinate);
        delay(100);
        
      }
  if(Input[input_counter-1] == 'i')
      {
        Robot_ycoordinate = atof(Input);
        Serial.print("Robot y: ");
        Serial.println(Robot_ycoordinate);
        delay(100);
        
      }
        if(Input[input_counter-1] == 'o')
      {
        BotAngle = atof(Input);
        Serial.print("Robot Angle: ");
        Serial.println(BotAngle);
        delay(100);
        
      }
        if(Input[input_counter-1] == 'p')
      {
        angleOfFirstBL = atof(Input);
        Serial.print("Angle of First Big Lego: ");
        Serial.println(angleOfFirstBL);
        delay(100);
        
      }
     if(Input[input_counter-1] == 'a')
      {
        angleOfSecondBL = atof(Input);
        Serial.print("Angle of Second Big Lego: ");
        Serial.println(angleOfSecondBL);
        delay(100);
        
      }
         if(Input[input_counter-1] == 's')
      {
        angleOfFirstS = atof(Input);
        Serial.print("Angle of First Small Lego: ");
        Serial.println(angleOfFirstS);
        delay(100);
        
      }
         if(Input[input_counter-1] == 'd')
      {
        angleOfSecondS = atof(Input);
        Serial.print("Angle of Second Small Lego: ");
        Serial.println(angleOfSecondS);
        delay(100);
        
      }
      
      input_counter = 0;
    }
    else
    {
      input_counter++;
      
    }
    
  }
}
#Some of the Python Code, I didn't put all as it's a lot

# Standard imports
import cv2
import numpy as np;
import cv2.aruco as aruco
import math
import serial
import time
from time import sleep

# Read image


cam = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_COMPLEX
out = cv2.VideoWriter('c:\\outpy.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 10, (640, 480))

cam.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
cam.set(cv2.CAP_PROP_FPS, 1)

cv2.namedWindow("TEST", cv2.WINDOW_AUTOSIZE)
cv2.namedWindow("TEST2", cv2.WINDOW_AUTOSIZE)
cv2.startWindowThread()


ser = serial.Serial('COM3', 115200, timeout=1) # Establish the connection on a specific port
ser.set_buffer_size(rx_size = 2147483647 , tx_size = 2147483647 )
time.sleep(1)

found1 = 0
found2 = 0

while (1):
    ret, frame = cam.read()
    if not ret:
        break
# Setting up detector would go before the Detection below
    for i in range(0, len(Grkeypoints)):
        GRx = Grkeypoints[i].pt[0]  # i is the index of the blob you want to get the position
        GRy = Grkeypoints[i].pt[1]
        GRarea = Grkeypoints[i].size
        GRxy = np.array((GRx, GRy))
        GRxy = np.array((GRx, GRy))

        # First Big Lego found coordinates________________________________________________Important
        if (i == 0 and GRarea > 45 and GRarea < 100) and counts <= 50:
            FirstLegox = Grkeypoints[i].pt[0]
            FirstLegoy = Grkeypoints[i].pt[1]
            FirstLegoxy = (FirstLegox,FirstLegoy)
            found1 = 1

            # Send x and y Coordinates of first block to Arduino 
            ser.write("{:.2f}x\n".format(FirstLegox).encode()) 
            ser.write("{:.2f}y\n".format(FirstLegoy).encode())
            print(ser.readline().strip())
           
        # Second Big Lego Found Coordinates_________________________________________

            if (i == 1 and GRarea > 45 and GRarea < 100 and countA <= 50):
                SecondLegox = Grkeypoints[1].pt[0]
                SecondLegoy = Grkeypoints[1].pt[1]
                SecondLegoxy = (SecondLegox, SecondLegoy)
                found2 = 1
                 # Send x and y Coordinates of Second block to Arduino 
                ser.write("{:.2f}q\n".format(SecondLegox).encode())
                ser.write("{:.2f}w\n".format(SecondLegoy).encode())
                print(ser.readline().strip())

 
                #print("First x: ", FirstLegox, " : Second X: ", SecondLegox)
                #print("First Y: ", FirstLegoy, " : Second Y: ", SecondLegoy)


        cv2.putText(im_with_Grkeypoints, str(int(GRx)), (int(GRx), int(GRy)), font, 1, (0, 0, 0))
        cv2.putText(im_with_Grkeypoints, str(int(GRy)), (int(GRx), int(GRy + 30)), font, 1, (0, 0, 0))
        cv2.putText(im_with_Grkeypoints, str(int(GRarea)), (int(GRx), int(GRy + 60)), font, 1, (0, 0, 0))
    # frame = cv2.bitwise_and(frame, im_with_keypoints, mask=mask)
    # cv2.imshow('frame', frame)
    # cv2.imshow('canvas',canvas)
    cv2.imshow('reversemask', reversemask)
    cv2.imshow('blur', blur)
    cv2.imshow('result', result)
    cv2.imshow('Green Keypoints', im_with_Grkeypoints)

Can you connect the OpenCV device to a different device than the Arduino, to verify and possibly examine the output from it, and to see if it freezes? For example, a PC running a serial terminal emulator.

You should not have any delay()s in code that is receiving serial input

Have a look at the examples in Serial Input Basics - simple reliable non-blocking ways to receive data. There is also a parse example to illustrate how to extract numbers from the received text.

The technique in the 3rd example will be the most reliable. It is what I use for Arduino to Arduino and Arduino to PC communication.

You can send data in a compatible format with code like this (or the equivalent in any other programming language)

Serial.print('<'); // start marker
Serial.print(value1);
Serial.print(','); // comma separator
Serial.print(value2);
Serial.println('>'); // end marker

This Simple Python - Arduino demo uses similar code.

Have a look at how millis() is used to manage timing without blocking in Several Things at a Time.

And see Using millis() for timing. A beginners guide if you need more explanation.

If you can avoid using float values on your Arduino then do so as floating point maths is slow. Do the hard calculations in Python.

...R