Thanks again for the link and the more basic rundown ascociated. I got things working but I'm hoping to get a little more help. I got the python script to send data to the esp32 controlling a 16x32 Matrix panel as well as update continuously with the image recognition on the PC.
However the serial communication takes over a second to send the data. I'm hoping it can be sped up and I'm not running into limitations of the pyserial library. It's sending less than 1000 bits and the baud rate is 115200 so in theory it should be able to communicate much faster if the python script sends the information fast enough.
Demo Video
The image recognition isn't as good with the background I had but it mostly worked.
Here is the code. I tried to keep things seperated so it's easier to follow.
Notes about Arduino Script:
There is an I2C display. It is a part of this project but right now is just used to show how long different proceeses take.
The two functions doing serial input are:
recvWithStartEndMarkers();
showNewData();
They're not copies from https://forum.arduino.cc/t/serial-input-basics-updated/382007 but are based off of it.
Notes about Python Script:
I switched to multiprocessing instead of multithreading because the image recognition is pretty computationally intensive. (Helpful Source) I tried to keep things seperate so hopefully it's easier to weigh in on the serial part of it.
There are 3 tasks:
P1: Image Processing
P2: Send Serial to Arduino
P3: Check for quit command (had some other functionality removed)
There are 3 Queues:
arduinoQueue (currently unused)
bytesQueue (used to send an array of bytes objects from P1 to P2)
quitQueue (used to end all tasks safely)
I am not collecting serial information from the arduino (it made it easier to only access the com port in one process).
The image recoginition updates the bytesQueue once every 1.1 seconds (look for DTIME). This is just done so the bytesQueue doesn't stack up. If you set DTIME to say 0.5, the display still only updates once per second.
Python Code Sample Output:
Waiting for Arduino
Arduino Ready
INFO: Created TensorFlow Lite XNNPACK delegate for CPU.
Update
0.8003032207489014
Update
0.7430198192596436
Update
0.8796906471252441
Update
0.6871297359466553
Update
0.8116204738616943
Arduino Code
//MAIN
#include "FastLED.h"
#include <Adafruit_SSD1306.h>
//OLED SETUP
#define SCREEN_WIDTH 128 // OLED display width, in pixels
#define SCREEN_HEIGHT 32 // OLED display height, in pixels
#define OLED_RESET -1 // Reset pin # (or -1 if sharing Arduino reset pin)
#define SCREEN_ADDRESS 0x3C ///< See datasheet for Address; 0x3D for 128x64, 0x3C for 128x32
Adafruit_SSD1306 display(SCREEN_WIDTH, SCREEN_HEIGHT, &Wire, OLED_RESET);
//Onboard LED
#define LED_BUILTIN 2
//FAST LED SETUP
#define LED_PIN 5
#define NUM_LEDS 512
#define COLOR_ORDER GRB
#define MAX_POWER_MILLIAMPS 3500
//creates an array of length 512 to store LED info.
CRGBArray<NUM_LEDS> leds;
//Grid for Human Display
uint32_t gridIn[16] = {
0b00000000000000000000000000000000,
0b00000000000000000000000000000000,
0b00000000000000111110000000000000,
0b00000000000011111111000000000000,
0b00000000000111111111100000000000,
0b00000000000111111111100000000000,
0b00000000000111111111100000000000,
0b00000000000111111111100000000000,
0b00000000000111111111100000000000,
0b00000000001111111111100000000000,
0b00000000000111111111000000000000,
0b00000000000011111110000000000000,
0b00000000000001111110000000000000,
0b00000000000001111110000000000000,
0b00000000000001111111100000000000,
0b00000000000111111111111100000000,
};
//for calculating time b/w
uint32_t prevTime = 0;
long dTime = 0;
//used to update matrix
bool bit=0;
uint32_t row = 0;
//For Serial Communication
#define numChars 4
const uint32_t END = 1162757120;
const uint32_t GRID = 1196575044;
char receivedChars[numChars];
boolean newData = false;
boolean sameGrid = false;
boolean gridActive = false;
boolean newGrid = false;
void setup() {
setup_OLED_FastLED(); //sets up OLED screen and FastLED
//Blink the Onboard LED to show the script started
pinMode(LED_BUILTIN, OUTPUT);
digitalWrite(LED_BUILTIN, HIGH);
delay(500);
digitalWrite(LED_BUILTIN, LOW);
//Send OK to show script ready
Serial.begin(115200);
Serial.setTimeout(0.01);
Serial.println("OK");
human_setup();
}
void loop() {
//Send and recieve serial instructions
recvWithStartEndMarkers();
showNewData();
if(newGrid){
prevTime = millis();
newGrid = false;
//Serial.println("Update");
for (int j=0; j<16; j++){
row = gridIn[j];
for (int i=0; i<32; i++){
bit = row&1;
if(bit){
leds[XY(i, 15-j)] = 0x00003D;
}
else{
leds[XY(i, j)] = 0;
}
row >>= 1;
}
}
FastLED.show();
}
}
void recvWithStartEndMarkers() {
static boolean recvInProgress = false;
static byte ndx = 0;
char startMarker = '<';
char endMarker = '>';
char rc;
while (Serial.available() > 0 && newData == false) {
rc = Serial.read();
if (recvInProgress == true) {
if (rc != endMarker) {
receivedChars[ndx] = rc;
ndx++;
if (ndx > numChars) {
ndx = numChars - 1;
}
}
else {
recvInProgress = false;
ndx = 0;
newData = true;
}
}
else if (rc == startMarker) {
recvInProgress = true;
}
rc = 0;
}
if(ndx < numChars){
if(ndx==1){
receivedChars[1] = 0;
receivedChars[2] = 0;
receivedChars[3] = 0;
}
else if(ndx==2){
receivedChars[2] = 0;
receivedChars[3] = 0;
}
else if(ndx==3){
receivedChars[3] = 0;
}
}
}
void showNewData() {
static uint8_t rowIter = 0;
static uint32_t row = 0;
if (newData == true) {
newData = false;
row = (receivedChars[0]<<24)|(receivedChars[1]<<16)|(receivedChars[2]<<8)|(receivedChars[3]);
//checks if row =="END" using ascii
if(row == END){
gridActive = false;
newGrid=true;
display.clearDisplay();
display.setCursor(0, 0);
dTime = millis()-prevTime;
display.print(dTime);
display.display();
}
//while the grid is being sent
if(gridActive){
if(rowIter<16){
gridIn[rowIter] = row;
rowIter++;
}
}
//checks if row == "grid" using ascii
if(row == GRID){
prevTime = millis();
rowIter = 0;
gridActive = true;
}
}
}
uint16_t XY(uint8_t x, uint8_t y)
{
if(x<=31 && y<=15){
uint16_t i;
//check for first or second matrix
bool addAtEnd = false;
if (x>15){
x -= 16;
addAtEnd = true;
}
//&0x01 checks if odd
if(y & 0x01){
// Even rows run backwards
uint8_t reverseX = 15 - x;
i = (y * 16) + reverseX;
}
else {
// odd rows run forwards
i = (y * 16) + x;
}
if(addAtEnd){
i+=256;
}
/*
Serial.print("x: ");
Serial.print(x);
Serial.print(" y: ");
Serial.print(y);
Serial.print(" i: ");
Serial.println(i);*/
return i;
}
else{
//Serial.println(" bad val");
return NUM_LEDS;
}
}
void setup_OLED_FastLED(){
//Setup I2C Display**********************************************************
// SSD1306_SWITCHCAPVCC = generate display voltage from 3.3V internally
if(!display.begin(SSD1306_SWITCHCAPVCC, SCREEN_ADDRESS)) {
Serial.println("SSD1306 allocation failed");
for(;;); // Don't proceed, loop forever
}
// Show initial display buffer contents on the screen --
// the library initializes this with an Adafruit splash screen.
display.display();
delay(500); // Pause for 2 seconds
// Clear the buffer
display.clearDisplay();
display.setTextColor(SSD1306_WHITE);
display.setTextSize(3); // Draw 3X-scale text
//Setup FastLED************************************************************
FastLED.addLeds<WS2812B,LED_PIN,COLOR_ORDER>(leds, NUM_LEDS);
//.setCorrection(TypicalLEDStrip);
FastLED.setDither(150 < 255);
FastLED.setMaxPowerInVoltsAndMilliamps( 5, MAX_POWER_MILLIAMPS);
FastLED.setBrightness(180);
/*https://fastled.io/docs/group___color_enums.html#gadf6bcba67c9573665af20788c4431ae8
https://fastled.io/docs/color_8h_source.html*/
FastLED.setCorrection(0xE696DC);/*230, 150, 220*/
}
void human_setup(){
FastLED.clear();
display.setCursor(0, 0);
display.stopscroll();
display.clearDisplay();
display.setTextSize(2);
/*
display.println(F("Human "));
display.print(F("CV"));
display.setTextSize(3);
display.display();
display.startscrollright(0x00, 0x0F);
*/
draw_human();
}
void draw_human(){
for (int j=0; j<16; j++){
row = gridIn[j];
for (int i=0; i<32; i++){
bit = row&1;
if(bit){
leds[XY(i, 15-j)] = 0x00003D;
}
else{
leds[XY(i, j)] = 0;
}
row >>= 1;
}
}
FastLED.show();
}
Warning
This code may continue to run tasks after you run it so be careful.
Python Code
from multiprocessing import Process, Manager
import serial, time
import cv2
from mediapipe import solutions
import keyboard
def P1_buildGrid(bytesQueue, quitQueue):
#Grid of bits, 32 in each list, 16 lists
startGrid = [
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]
]
def gridToBytes(grid_f):
grid_func = []
#convert each row of the grid to a list of 4 bytes (for the 32 bits in each row)
for i, row in enumerate(grid_f):
#convert row into a string of binary values
sb = ''.join(row)[::-1]
#This could flip each row if desired[::-1]
row_updated = []
#seperate the row into 4 chunks of 8 bits each
#At one point considered adding this as a signal byte
row_updated.append(int(sb[0:8],2))
row_updated.append(int(sb[8:16],2))
row_updated.append(int(sb[16:24],2))
row_updated.append(int(sb[24:32],2))
#Actually convert to bytes
grid_func.append(bytes(row_updated))
return grid_func
#print("Waiting for Arduino")
time.sleep(1)
#print("Arduino Ready")
#MEDIAPIPE CONSTANTS------------------------------------
#CHANGE THESE
NumPixelsX = 32
NumPixelsY = 16
PX_ACCEPTANCE=0.5 #B/W 0-1, less means more pixels will turn on, more means less pixels will turn on
PX_CHOICE=0.2
#Defines update speed of bytes to arduino
DTIME = 1.1
#These must match the ratio of numPixels
#Actual Camera Dimensions are 480x640
CAM_HEIGHT = 320
CAM_LENGTH = 640
CAM_OFFSET_H = 80
PX_LEN = int(320/NumPixelsY)
PX_MIN = int((PX_LEN*PX_LEN)*PX_ACCEPTANCE)
#Previous Time
prevTime = time.time()
#Set Up Mediapipe------------------------
mp_seg = solutions.selfie_segmentation
# For webcam input:
cap = cv2.VideoCapture(0)
#model_selection: 0 or 1. 0 to select a general-purpose model, and 1 to
# select a model more optimized for landscape images.
with mp_seg.SelfieSegmentation(model_selection=0) as pose:
grid = startGrid
prevTime = time.time()
#MAIN LOOP---------------------------------------------------
while cap.isOpened():
#Update IMAGEPIPE --------------------------------------
success, image = cap.read()
if not success:
print("Ignoring empty camera frame.")
# If loading a video, use 'break' instead of 'continue'.
continue
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
image = cv2.flip(image, 1)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#Generate Mask
results = pose.process(image)
#Called to check a pixel from the mask
def check_mask(x,y):
px_x_end = (x+1)*PX_LEN
px_y = (y*PX_LEN) + CAM_OFFSET_H
px_y_end = (y+1)*PX_LEN + CAM_OFFSET_H
px_x=x*PX_LEN
px_count=0
try:
while(px_y<px_y_end):
px_x=x*PX_LEN
while(px_x<px_x_end):
if(results.segmentation_mask[px_y][px_x] > PX_CHOICE):
px_count+=1
px_x+=1
px_y+=1
if(px_count>PX_MIN):
return "1"
else:
return "0"
except:
return "0"
#UPDATE GRID
#Update only if enough time has passed
if(time.time()-prevTime >= DTIME):
for y, row in enumerate(grid):
for x, px in enumerate(row):
prev=grid[y][x]
grid[y][x]=check_mask(x,y)
bytesQueue.put(gridToBytes(grid))
#print(gridBytes)
prevTime = time.time()
if not quitQueue.empty():
break
cap.release()
#Update Serial-----------------------------------------
def P2_updateSerial(bytesQueue, arduinoQueue, quitQueue):
arduino = serial.Serial(port='COM8', baudrate=115200, timeout=0.01)
print("Waiting for Arduino")
time.sleep(2)
print("Arduino Ready")
def write(data):
arduino.write(b'<')
time.sleep(0.001)
arduino.write(data)
time.sleep(0.001)
arduino.write(b'>')
time.sleep(0.001)
while True:
#Output
if(not bytesQueue.empty()):
print("Update")
gridBytes = bytesQueue.get()
write(b'<')
write(b'GRID')
write(b'>')
prevTime = time.time()
for row in gridBytes:
write(row)
arduino.write(bytes('\n', encoding='utf-8'))
time.sleep(0.001)
print(time.time()-prevTime)
write(b'END')
#Input
if not arduinoQueue.empty():
print(arduinoQueue.get())
if not quitQueue.empty():
break
#Listents to arduino for input
def P3_listenToArduinoAndQuit(arduinoQueue, quitQueue):
#arduino = serial.Serial(port='COM8', baudrate=115200, timeout=0.1)
message = b''
while True:
"""
incoming = arduino.read()
if (incoming == b'\n'):
#arduinoQueue.put(message.decode('utf-8').strip().upper())
arduinoQueue.put(message)
message = b''
else:
if ((incoming != b'') and (incoming != b'\r')):
message += incoming
"""
if keyboard.is_pressed('q'):
for i in range(30):
quitQueue.put(1)
break
if __name__ == '__main__':
"""
Processes:
P1_buildGrid(bytesQueue)
P2_updateSerial(bytesQueue, arduinoQueue)
P3_listenToArduino(arduinoQueue)
"""
# Create an instance of the Manager
with Manager() as manager:
# Create a queue within the context of the manager
arduinoQueue = manager.Queue()
bytesQueue = manager.Queue()
quitQueue = manager.Queue()
#Create two instances of the Process class, one for each function
p1 = Process(target=P1_buildGrid, args=(bytesQueue, quitQueue,))
p2 = Process(target=P2_updateSerial, args=(bytesQueue, arduinoQueue, quitQueue,))
p3 = Process(target=P3_listenToArduinoAndQuit, args=(arduinoQueue, quitQueue,))
#Start processes
p1.start()
p2.start()
p3.start()
# Wait for processes to finish
p1.join()
p2.join()
p3.join()
