Hello guys, I'm having problem with SocketIO connection betwwen ESP32 (client) and flask python (server). Here is my code:
import json
import mediapipe as mp
import cv2
import io
import pickle
import numpy as np
from PIL import Image
from flask import request
from flask import Flask, render_template
from flask import jsonify
from flask_socketio import SocketIO
from bluetooth import *
import socket
import subprocess
import base64
Trimodel = pickle.load(open('model', 'rb'))
hand_signals = ['Move', 'Stop']
ip = None
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(max_num_hands = 2, min_detection_confidence = 0.8, min_tracking_confidence = 0.5)
app = Flask(__name__)
socketio = SocketIO(app)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/test', methods=['POST'])
def test():
output = request.get_json()
imgdata = base64.b64decode(output)
img = Image.open(io.BytesIO(imgdata))
opencv_img = cv2.cvtColor(np.array(img), cv2.COLOR_BGR2RGB)
opencv_img = cv2.flip(opencv_img, 1)
opencv_img.flags.writeable = False
results = hands.process(opencv_img)
opencv_img.flags.writeable = True
result = None
if results.multi_hand_landmarks:
hands_keypoint = []
if len(results.multi_handedness) == 2:
for hand in results.multi_hand_landmarks:
hand = convert_landmark_list(hand)
hands_keypoint += hand
if (results.multi_handedness[0].classification[0].label != results.multi_handedness[1].classification[0].label):
if (results.multi_handedness[0].classification[0].label == 'Right'):
slice = int(len(hands_keypoint) / 2)
hands_keypoint = hands_keypoint[slice:] + hands_keypoint[:slice]
result = Trimodel.predict([hands_keypoint])
else:
for side in results.multi_handedness:
if side.classification[0].label == 'Left':
for hand in results.multi_hand_landmarks:
hand = convert_landmark_list(hand)
hands_keypoint += hand
hands_keypoint += [0.0] * 63
else:
hands_keypoint += [0.0] * 63
for hand in results.multi_hand_landmarks:
hand = convert_landmark_list(hand)
hands_keypoint += hand
result = Trimodel.predict([hands_keypoint])
if result is not None:
result = str(hand_signals[result[0]])
socketio.emit('reply', result)
return jsonify(result)
result = "Not found"
socketio.emit('reply', result)
return jsonify("Not found")
@app.route('/Scan', methods=['POST'])
def Scan():
nearby_devices = discover_devices(lookup_names=True)
return jsonify(nearby_devices)
@app.route('/ConnectBluetooth', methods=['POST'])
def ConnectBluetooth():
bd_addr = request.get_json()
port = 1
s = socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM)
try:
s.connect((bd_addr, port))
return ("Connected!")
except socket.error:
return ("{socket.error}")
finally:
data = subprocess.check_output(['netsh', 'wlan', 'show', 'interfaces']).decode('utf-8').split('\n')
ssid_line = [x for x in data if 'SSID' in x and 'BSSID' not in x]
if ssid_line:
ssid_list = ssid_line[0].split(':')
ssid = ssid_list[1].strip()
ssid_info = subprocess.check_output(['netsh', 'wlan', 'show', 'profile', ssid, 'key=clear']).decode('utf-8').split('\n')
password = [b.split(":")[1][1:-1] for b in ssid_info if "Key Content" in b]
password = password[0]
s.send(bytes(ssid, 'UTF-8'))
s.send(bytes("|", 'utf-8'))
s.send(bytes(password, 'utf-8'))
s.send(bytes("|", 'utf-8'))
s.send(bytes(ip, 'utf-8'))
s.close()
def convert_landmark_list (hand):
converted = []
for landmark in hand.landmark:
converted.append(landmark.x)
converted.append(landmark.y)
converted.append(landmark.z)
return converted
if __name__ == "__main__":
# app.run(host= '{}'.format(socket.gethostbyname(socket.gethostname())))
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('10.255.255.255', 1))
ip = s.getsockname()[0]
socketio.run(app, host='{}'.format(ip), port=5000)
As you can see, I want to send data to ESP32 in App.Route('/test'). Here is my ESP32 code to receive it:
#include <Arduino.h>
#include <WiFi.h>
#include <WiFiMulti.h>
#include <WiFiClientSecure.h>
#include <ArduinoJson.h>
#include <WebSocketsClient.h>
#include <SocketIOclient.h>
WiFiMulti WiFiMulti;
SocketIOclient socketIO;
#define USE_SERIAL Serial
void socketIOEvent(socketIOmessageType_t type, uint8_t *payload, size_t length) {
switch (type) {
case sIOtype_DISCONNECT:
USE_SERIAL.printf("[IOc] Disconnected!\n");
break;
case sIOtype_CONNECT:
USE_SERIAL.printf("[IOc] Connected to url: %s\n", payload);
// join default namespace (no auto join in Socket.IO V3)
socketIO.send(sIOtype_CONNECT, "/");
break;
case sIOtype_EVENT:
{
char *sptr = NULL;
int id = strtol((char *)payload, &sptr, 10);
USE_SERIAL.printf("[IOc] get event: %s id: %d\n", payload, id);
if (id) {
payload = (uint8_t *)sptr;
}
DynamicJsonDocument doc(1024);
DeserializationError error = deserializeJson(doc, payload, length);
if (error) {
USE_SERIAL.print(F("deserializeJson() failed: "));
USE_SERIAL.println(error.c_str());
return;
}
String eventName = doc[0];
USE_SERIAL.printf("[IOc] event name: %s\n", eventName.c_str());
// Message Includes a ID for a ACK (callback)
if (id) {
// creat JSON message for Socket.IO (ack)
DynamicJsonDocument docOut(1024);
JsonArray array = docOut.to<JsonArray>();
// add payload (parameters) for the ack (callback function)
JsonObject param1 = array.createNestedObject();
param1["now"] = millis();
// JSON to String (serializion)
String output;
output += id;
serializeJson(docOut, output);
// Send event
socketIO.send(sIOtype_ACK, output);
}
}
break;
case sIOtype_ACK:
USE_SERIAL.printf("[IOc] get ack: %u\n", length);
break;
case sIOtype_ERROR:
USE_SERIAL.printf("[IOc] get error: %u\n", length);
break;
case sIOtype_BINARY_EVENT:
USE_SERIAL.printf("[IOc] get binary: %u\n", length);
break;
case sIOtype_BINARY_ACK:
USE_SERIAL.printf("[IOc] get binary ack: %u\n", length);
break;
}
}
void setup() {
//USE_SERIAL.begin(921600);
USE_SERIAL.begin(115200);
//Serial.setDebugOutput(true);
USE_SERIAL.setDebugOutput(true);
USE_SERIAL.println();
USE_SERIAL.println();
USE_SERIAL.println();
for (uint8_t t = 4; t > 0; t--) {
USE_SERIAL.printf("[SETUP] BOOT WAIT %d...\n", t);
USE_SERIAL.flush();
delay(1000);
}
// this timeout is the interval of serial data read
WiFiMulti.addAP("cazone_staff", "lamviecnghiemtuc");
//WiFi.disconnect();
while (WiFiMulti.run() != WL_CONNECTED) {
delay(100);
}
String ip = WiFi.localIP().toString();
USE_SERIAL.printf("[SETUP] WiFi Connected %s\n", ip.c_str());
// server address, port and URL
socketIO.begin("192.168.1.187", 5000, "/socket.io/?EIO=3");
// event handler
socketIO.onEvent(socketIOEvent);
}
void loop() {
socketIO.loop();
}
Instead of receiving data as I thought, it keeps printing "Disconnected".
Hope you guys can help me to fix it out. Thank you!