How to snchronize webcam recording and arduino using python?

Projects Programming
1

Hello,

I am sending serial data from my arduino with python. The sending starts once I press a joysticksbutton. In my python script, this works very well, that serial data is only send when the button is pressed.

Now I want python to simultaneously start the laptop camera to record, once the button is pressed. However this does not work, unfortunately. The camera immediatly starts to record when I run the python script and does not wait for the "button signal" like the serial data does.

I dont know if this is the roght forum to ask a python question but I'd still appreciate any help!

#include <math.h>

const int Led_Green = 9;  
const int Led_Yellow = 10;  
const int Gate1 = 5;
const int Gate2 = 6;
const int Joystick_X = A0; 
const int Joystick_Y = A1;
const int pushbutton = 3;
int JoystickValue_X = 0; 
int JoystickValue_Y = 0;
const int middle_X = 512;
const int middle_Y = 512;
int pushbuttonState;
int JoystickValue_X_Y;

int tempsensor1 = A2;
int tempsensor2 = A3;
int tempsensor3 = A4;

int readVal1 = 0;
int readVal2 = 0;
int readVal3 = 0;

double Temp1 = 0;
double Temp2 = 0;
double Temp3 = 0;
int TempLimit;

double offset_degree;

double offset_degree1 = -2.2; // RT=26.0°C, difference
double offset_degree2 = 0.7;
double offset_degree3 = -2.47;

unsigned long deltaTime;

bool running = true;

//Define experiment (0 = manual, 1 = vollgas, 2 = , 3 = )
int experiment = 1; 

//Define functions--------------------------------------------------------------------
void manual_function() {
      
      JoystickValue_X = analogRead (Joystick_X);                        // [0, 1023]
      JoystickValue_X = constrain(JoystickValue_X, middle_X, 1023);     // [512, 1023]
      JoystickValue_X = map(JoystickValue_X, middle_X, 1023, 0, 1023);   // [0, 100]

 
      JoystickValue_Y = analogRead (Joystick_Y);                        // [0, 1023]
      JoystickValue_Y = constrain(JoystickValue_Y, middle_Y, 1023);     // [512, 1023]
      JoystickValue_Y = map(JoystickValue_Y, middle_Y, 1023, 0, 1023);   // [0, 100]

        Serial.print(Temp1);   
        Serial.print(";");      
        Serial.print(Temp2);  
        Serial.print(";");   
        Serial.print(Temp3); 
        Serial.print(";");
  
        Serial.print(JoystickValue_X);
        Serial.print(";");
        Serial.println(JoystickValue_Y);          
}



//Funktion Temperatur offset und Umrechung-----------------------------------------------------
double Thermistor(int RawADC, int sensor_number)
{
    double Temp;
    double offset_degree;
    Temp = log(10000.0 * ((1024.0 / RawADC - 1)));
    Temp = 1 / (0.001129148 + (0.000234125 + (0.0000000876741 * Temp * Temp )) * Temp );
    Temp = Temp - 273.15;            // Konvertierung von Kelvin in Celsius

    switch (sensor_number) {
  case 1:
    offset_degree = offset_degree1;
    break;
  case 2:
    offset_degree = offset_degree2;
    break;
   case 3:
    offset_degree = offset_degree3;
    break;
  default:
    offset_degree = 0;   
    break; 
}
    return Temp+offset_degree;
}

 
//-----------------------------------------------------------------------------------------------

void case_function( int JoystickInput_X, int JoystickInput_Y, int TempLimit, int TempValue) {    
    if(deltaTime < 45000L) {
    JoystickValue_X = JoystickInput_X;
    JoystickValue_Y = JoystickInput_Y;
        Serial.print(Temp1);   
        Serial.print(";");      
        Serial.print(Temp2);  
        Serial.print(";");   
        Serial.print(Temp3); 
        Serial.print(";");
  
        Serial.print(JoystickValue_X);
        Serial.print(";");
        Serial.println(JoystickValue_Y);  
    }
    else if ((deltaTime >= 45000L) && (TempValue > TempLimit)) {
     JoystickValue_X = 0;
     JoystickValue_Y = 0;
        Serial.print(Temp1);   
        Serial.print(";");      
        Serial.print(Temp2);  
        Serial.print(";");   
        Serial.print(Temp3); 
        Serial.print(";");
  
        Serial.print(JoystickValue_X);
        Serial.print(";");
        Serial.println(JoystickValue_Y);  
    }
    else if ((deltaTime > 45000L) && (TempValue < TempLimit)) {
      Serial.print("Experiment done.");
      running = false;
      
    }
    
}
//---------------------------------------------------------------------------------

void setup() {
  pinMode(Joystick_X, INPUT); // X-axis
  pinMode(Joystick_Y, INPUT); // Y-axis
  pinMode(pushbutton,INPUT); // press button
  digitalWrite(pushbutton, INPUT_PULLUP);
  pushbuttonState = digitalRead(pushbutton);
      
  Serial.begin(9600); 
  while (digitalRead(pushbutton) == HIGH);

}

void loop() {
  if(running == true) {
    //currentTime = millis();
    //unsigned long deltaTime = CalculateDeltaTime();
    static unsigned long TimeZero = millis();
    deltaTime = millis() - TimeZero;
    
      Serial.print("start");
      Serial.print(";");
      Serial.print(deltaTime);
      Serial.print(";");
      
      readVal1 = analogRead(tempsensor1);
      Temp1 =  Thermistor(readVal1,1);

      readVal2 = analogRead(tempsensor2);
      Temp2 =  Thermistor(readVal2,2);

      readVal3 = analogRead(tempsensor3);
      Temp3 =  Thermistor(readVal3,3);

     switch(experiment) {
      case 0:
        manual_function();
        break;
    
    
       case 1:
         case_function(1023, 0, 18, Temp1);

         break;
    
       case 2:
         case_function(0, 1023, 23, Temp2);
         break;
    
       case 3:
         case_function(1023, 1023, 23, Temp3);
         break;
    
       case 4:
         case_function(1023, 0, 36, Temp1);
         break;
    
       case 5:
         case_function(0, 1023, 36, Temp2);
         break;
    
       case 6:
         case_function(1023, 1023, 36, Temp3);
         break;
     
 }
         if (JoystickValue_Y <= 0) {
      analogWrite(Led_Green, 0);
      analogWrite(Gate1, 0);
    }
  
    if (JoystickValue_Y > 0) {
      analogWrite(Led_Green, JoystickValue_Y/4);
      analogWrite(Gate1, JoystickValue_Y/4);
    }
    
  
   if (JoystickValue_X <= 0) {
      analogWrite(Led_Yellow, 0);
      analogWrite(Gate2, 0);
    }
  
   if (JoystickValue_X > 0) {
      analogWrite(Led_Yellow, JoystickValue_X/4);
      analogWrite(Gate2, JoystickValue_X/4);
    }
       delay(1000);
      }
}

my python scriot:

import serial
from datetime import datetime
import cv2
import csv
from RsInstrument import *
from time import sleep, localtime, strftime
import time
import gsv8
import usb.core
import pyvisa
import usb.util


# Serial Data from Arduino
ser = serial.Serial('COM6', 9600, timeout=1)

# Serial Data from Loadcell--------------------------------------------------------
#dev = gsv8.gsv8("COM9", 9600)  # load cell port


# Initialize and request instrument for all sessions via
"""
RsInstrument.assert_minimum_version('1.22.0')
USB_adr_NGE = 'USB0::0x0AAD::0x0197::5601.3800k03-103085::0::INSTR'
nge = RsInstrument(USB_adr_NGE)
idn = nge.query_str('*IDN?')
"""
while 'start' not in ser:
    while True:
        # Camera Start (Laptop)---------------------------------------------------------------------------------
        webcam = cv2.VideoCapture(0)
        frame_width = int(webcam.get(3))
        frame_height = int(webcam.get(4))
        size = (frame_width, frame_height)
        result_laptop = cv2.VideoWriter('NameAsDateTime_laptop.avi', cv2.VideoWriter_fourcc(*'MJPG'), 10, size)
        ret, frame = webcam.read()
        if ret == True:
           result_laptop.write(frame)
           cv2.imshow("Test", frame)
           key = cv2.waitKey(20) & 0xFF
           if key == ord("q"):
                webcam.release()
                break
        """
        # Camera Start (external)-------------------------------------------------------------------------------
        camera = cv2.VideoCapture(2)
        frame_width2 = int(camera.get(3))
        frame_height2 = int(camera.get(4))
        size2 = (frame_width2, frame_height2)
        result_camera = cv2.VideoWriter('NameAsDateTime_camera.avi', cv2.VideoWriter_fourcc(*'MJPG'), 10, size2)
        ret_cam, frame_cam = camera.read()
        image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        if ret_cam == True:
            result_camera.write(frame_cam)
            cv2.imshow("Test cam", image)
            key_cam = cv2.waitKey(20) & 0xFF
            if key_cam == ord("w"):
                camera.release()
                break
        """
        #get arduino data(temp)---------------------------------
        fileName = "temp_force_serialdata.csv"
        file = open(fileName, "a")
        getData = ser.readline()
        dataString = getData.decode('utf-8')
        data = dataString.rstrip()

        dateTimeObj = datetime.now()
        timestampStr = dateTimeObj.strftime("%d-%b-%Y (%H:%M:%S)")

        sensor_data = data.split(";")
        sensor_data.append(timestampStr)

        readings = data.split(";")
        sensor_data.append(readings)

        """
        # get load cell data-------------------------------------
        measurement = dev.ReadValue()
        measurement_str = measurement.toString()
        measurement_arr = measurement_str.split(";")
        force_x = measurement_arr[1].split(":")[2]
        force_y = measurement_arr[2].split(":")[1]
        force_z = measurement_arr[3].split(":")[1]
        
        #get power supply data-----------------------------------
        V1 = 3.6
        C1 = 1000
        nge.write('INSTrument:NSELect 1')  # Choose CH1
        nge.write('OUTPut:STATe 1')  # Switch CH1 ON
        nge.write('SOURce:VOLTage:LEVel:IMMediate:AMPLitude ' + str(V1))  # Set voltage level
        nge.write('SOURce:CURRent:LEVel:IMMediate:AMPLitude ' + str(C1))  # Set current level
        voltage1 = nge.query_str('MEASure:SCALar:VOLTage:DC?')  # ...Voltage
        current1 = nge.query_str('MEASure:SCALar:CURRent:DC?')  # ...Current
        """
        entry = timestampStr + ',' + ','.join(readings) + '\n'
        #entry = timestampStr + ',' + ','.join(readings) + ',' + force_x + ',' + force_y + ',' + force_z + '\n'
        #entry = timestampStr + ',' + ','.join(readings) + ',' + voltage1 + ', ' + current1 + '\n'
        print(entry)

        with open(fileName, 'a', encoding='UTF8') as f:
            f.write(entry)

        file.close()
        pass
2

Are you sure you want to indent the while True: after that ?

3

if I dont I get an indentation error

4

my point is you don't want to do

    while True:
        # Camera Start (Laptop)---------------------------------------------------------------------------------
        webcam = cv2.VideoCapture(0)
        frame_width = int(webcam.get(3))
        frame_height = int(webcam.get(4))
        size = (frame_width, frame_height)
        result_laptop = cv2.VideoWriter('NameAsDateTime_laptop.avi', cv2.VideoWriter_fourcc(*'MJPG'), 10, size)

while 'start' not in ser: (I assume you expect the Arduino to send 'start' when the button is pressed ?)

this is what triggers the camera right away.

Don't you want to wait and do nothing until you've received start?

5

yes thats exactly what I want to do, but I just dont know how

6

My experience is that I found best to use tasks in python to listen to the Serial port to collect commands (lines) asynchronously and deal with them

You could try to explore something like this (assuming your Arduino is on COM6)

Python:

#!/usr/bin/python3
import threading, queue, serial

arduinoQueue = queue.Queue()

def listenToArduino():
    message = b''
    while True:
        incoming = arduino.read()
        if (incoming == b'\n'):
            arduinoQueue.put(message.decode('utf-8').strip().upper())
            message = b''
        else:
            if ((incoming != b'') and (incoming != b'\r')):
                 message += incoming
                 
                 
def configureArduino():
    global arduino
    arduino = serial.Serial('COM6', 115200, timeout=.1)
    arduinoThread = threading.Thread(target=listenToArduino, args=())
    arduinoThread.daemon = True
    arduinoThread.start()


configureArduino()                                      # will reboot AVR based Arduinos

print("Waiting for Arduino")

while True:
    if not arduinoQueue.empty():
        message = arduinoQueue.get()
        if message == "START":
            break
        else:
            print("got unknown message from Arduino : " + message)
            
print("Arduino Ready")
print("Starting camera")

and you can try with this arduino code

const byte buttonPin = 3;

void setup() {
	pinMode(buttonPin, INPUT_PULLUP);
	Serial.begin(115200); 
	Serial.println(F("setup done, waiting for button press"));
  while (digitalRead(buttonPin) == HIGH) yield(); // active wait until the button is pressed

  // notify that we are ready to start
  Serial.println(F("START"));
}

void loop() {}

Note that opening the Serial port from the python side will reboot your arduino (obviously the Serial monitor should not be open in the IDE)

what the python code does is to collect (in upper cap) all the lines sent by the Arduino in a queue and the main part of the code checks if something is in the queue and if so compare it to START. when START is found, you exit the infinite wait loop and proceed with the video acquisition

running the python script in a terminal

$$ python3 ./test.py
Waiting for Arduino
got unknown message from Arduino : SETUP DONE, WAITING FOR BUTTON PRESS
<<HERE I PRESSED THE BUTTON>>
Arduino Ready
Starting camera
7 
while 'start' not in ser:
    pass
while True:
    etc.
8

not sure this works on the Serial port

9

Neither am I but I was just demonstrating how to get past the empty while statement.

10

OK pass or continue would do indeed

11

oh thanks for your help! I am not sure if I understand everything tho.. I dont know where to put my actual code

Would it be like:

import threading, queue, serial

arduinoQueue = queue.Queue()

def listenToArduino():
    message = b''
    while True:
        incoming = arduino.read()
        if (incoming == b'\n'):
            arduinoQueue.put(message.decode('utf-8').strip().upper())
            message = b''
        else:
            if ((incoming != b'') and (incoming != b'\r')):
                 message += incoming
                 
                 
def configureArduino():
    global arduino
    arduino = serial.Serial('COM6', 115200, timeout=.1)
    arduinoThread = threading.Thread(target=listenToArduino, args=())
    arduinoThread.daemon = True
    arduinoThread.start()


configureArduino()                                      # will reboot AVR based Arduinos

print("Waiting for Arduino")

while True:
    if not arduinoQueue.empty():
        message = arduinoQueue.get()
        if message == "START":
        
        [HERE IS MY CODE]

break
        else:
            print("got unknown message from Arduino : " + message)
            
print("Arduino Ready")
print("Starting camera")
12

don't indent, put your code at the same level where I had

print("Arduino Ready")
print("Starting camera")

➜

while True:
    if not arduinoQueue.empty():
        message = arduinoQueue.get()
        if message == "START":
            break
        else:
            print("got unknown message from Arduino : " + message)
            
print("Arduino Ready")
print("Starting camera")
[HERE IS YOUR CODE]
13

what exactly does F mean?

14

If I press the button. nothing happens, neither the data is send to python, nor the camera starts

15

it means store the string START in flash memory instead of SRAM as you don't have much of this.
the behavior is exactly as if you had typed Serial.println("START"); but with less memory used.

how is your button connected ? pin 3 -- button --- GND ?

16

right

18

okay, now it works that everything starts simultenously. However the code stops immediatly starting. Like I can see that the camera window pops up but then immediatly closes again

19

Post the code. You likely miss a while

Your initialization code goes here
while True: 
    Your iterative code here
20 
import serial
from datetime import datetime
import cv2
import threading, queue
import pyrealsense2
import csv
from RsInstrument import *
from time import sleep, localtime, strftime
import time
import gsv8
import usb.core
import pyvisa
import usb.util

arduinoQueue = queue.Queue()


def listenToArduino():
    message = b''
    while True:
        incoming = arduino.read()
        if (incoming == b'\n'):
            arduinoQueue.put(message.decode('utf-8').strip().upper())
            message = b''
        else:
            if ((incoming != b'') and (incoming != b'\r')):
                message += incoming


def configureArduino():
    global arduino
    arduino = serial.Serial('COM6', 115200, timeout=.1)
    arduinoThread = threading.Thread(target=listenToArduino, args=())
    arduinoThread.daemon = True
    arduinoThread.start()


configureArduino()  # will reboot AVR based Arduinos

print("Waiting for Arduino")

while True:
    if not arduinoQueue.empty():
        message = arduinoQueue.get()
        if message == "START":
            break
        else:
            print("got unknown message from Arduino : " + message)

print("Arduino Ready")
print("Starting camera")

# Serial Data from Arduino
#ser = serial.Serial('COM6', 9600, timeout=1)


# Serial Data from Loadcell--------------------------------------------------------
# dev = gsv8.gsv8("COM9", 9600)  # load cell port

"""
# Initialize and request instrument for all sessions via
RsInstrument.assert_minimum_version('1.22.0')
USB_adr_NGE = 'USB0::0x0AAD::0x0197::5601.3800k03-103085::0::INSTR'
nge = RsInstrument(USB_adr_NGE)
idn = nge.query_str('*IDN?')
"""

# Camera Start (Laptop)---------------------------------------------------------------------------------
webcam = cv2.VideoCapture(0)
frame_width = int(webcam.get(3))
frame_height = int(webcam.get(4))
size = (frame_width, frame_height)
result_laptop = cv2.VideoWriter('NameAsDateTime_laptop.avi', cv2.VideoWriter_fourcc(*'MJPG'), 10, size)
ret, frame = webcam.read()
if ret == True:
    result_laptop.write(frame)
    cv2.imshow("Test", frame)
    key = cv2.waitKey(20) & 0xFF
    if key_cam == ord("w"):


# Camera Start (external)-------------------------------------------------------------------------------
camera = cv2.VideoCapture(3)
frame_width2 = int(camera.get(3))
frame_height2 = int(camera.get(4))
size2 = (frame_width2, frame_height2)
result_camera = cv2.VideoWriter('NameAsDateTime_camera.avi', cv2.VideoWriter_fourcc(*'MJPG'), 10, size2)
ret_cam, frame_cam = camera.read()
# image = cv2.cvtColor(frame_cam, cv2.COLOR_BGR2GRAY)
if ret_cam == True:
    result_camera.write(frame_cam)
    cv2.imshow("Test cam", frame_cam)
    key_cam = cv2.waitKey(20) & 0xFF


# get arduino data(temp)---------------------------------
fileName = "temp_force_serialdata.csv"
file = open(fileName, "a")
getData = arduino.readline()
dataString = getData.decode('utf-8')
data = dataString.rstrip()

dateTimeObj = datetime.now()
timestampStr = dateTimeObj.strftime("%d-%b-%Y (%H:%M:%S)")

sensor_data = data.split(";")
sensor_data.append(timestampStr)

readings = data.split(";")
sensor_data.append(readings)

"""
# get load cell data-------------------------------------
measurement = dev.ReadValue()
measurement_str = measurement.toString()
measurement_arr = measurement_str.split(";")
force_x = measurement_arr[1].split(":")[2]
force_y = measurement_arr[2].split(":")[1]
force_z = measurement_arr[3].split(":")[1]


#get power supply data-----------------------------------
V1 = 3.6
C1 = 1000
nge.write('INSTrument:NSELect 1')  # Choose CH1
nge.write('OUTPut:STATe 1')  # Switch CH1 ON
nge.write('SOURce:VOLTage:LEVel:IMMediate:AMPLitude ' + str(V1))  # Set voltage level
nge.write('SOURce:CURRent:LEVel:IMMediate:AMPLitude ' + str(C1))  # Set current level
voltage1 = nge.query_str('MEASure:SCALar:VOLTage:DC?')  # ...Voltage
current1 = nge.query_str('MEASure:SCALar:CURRent:DC?')  # ...Current
"""
entry = timestampStr + ',' + ','.join(readings) + '\n'
# entry = timestampStr + ',' + ','.join(readings) + ',' + force_x + ',' + force_y + ',' + force_z + '\n'
# entry = timestampStr + ',' + ','.join(readings) + ',' + voltage1 + ', ' + current1 + '\n'
print(entry)

with open(fileName, 'a', encoding='UTF8') as f:
    f.write(entry)

file.close()
pass
21

shouldnt this while True, make the loop go on? Or do I need another one