I am working on a little project to control a little vehicle and I want to attach a little camera in the front, so I can have visual feedback on my phone. I am using a cheap ESP32 cam I found, because I thought it will be the easiest, it does work, but not as fast as I expected.
For now, I am using python and opencv to receive and process the jpeg data because is easy, but my framerate is not as good. The example they have with a webserver is faster and I wonder what is wrong.
Here is the code for the ESP32 camera:
#include "esp_camera.h"
#include <WiFi.h>
//
// WARNING!!! PSRAM IC required for UXGA resolution and high JPEG quality
// Ensure ESP32 Wrover Module or other board with PSRAM is selected
// Partial images will be transmitted if image exceeds buffer size
//
// Select camera model
//#define CAMERA_MODEL_WROVER_KIT // Has PSRAM
//#define CAMERA_MODEL_ESP_EYE // Has PSRAM
//#define CAMERA_MODEL_M5STACK_PSRAM // Has PSRAM
//#define CAMERA_MODEL_M5STACK_V2_PSRAM // M5Camera version B Has PSRAM
//#define CAMERA_MODEL_M5STACK_WIDE // Has PSRAM
//#define CAMERA_MODEL_M5STACK_ESP32CAM // No PSRAM
#define CAMERA_MODEL_AI_THINKER // Has PSRAM
//#define CAMERA_MODEL_TTGO_T_JOURNAL // No PSRAM
#include "camera_pins.h"
char* ssid = "....";
char* password = "......";
WiFiClient client;
IPAddress server(192,168,2,29);
bool bConnecting = true;
void setup() {
// put your setup code here, to run once:
Serial.begin(115200);
Serial.setDebugOutput(true);
Serial.println();
// Lets connect to wifi first
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED)
{
delay(500);
Serial.print(".");
}
// connection is done, print things
Serial.print("SSID: ");
Serial.println(WiFi.SSID());
IPAddress ip = WiFi.localIP();
Serial.print("IP Address: ");
Serial.println(ip);
// Now init camera
camera_config_t config;
config.ledc_channel = LEDC_CHANNEL_0;
config.ledc_timer = LEDC_TIMER_0;
config.pin_d0 = Y2_GPIO_NUM;
config.pin_d1 = Y3_GPIO_NUM;
config.pin_d2 = Y4_GPIO_NUM;
config.pin_d3 = Y5_GPIO_NUM;
config.pin_d4 = Y6_GPIO_NUM;
config.pin_d5 = Y7_GPIO_NUM;
config.pin_d6 = Y8_GPIO_NUM;
config.pin_d7 = Y9_GPIO_NUM;
config.pin_xclk = XCLK_GPIO_NUM;
config.pin_pclk = PCLK_GPIO_NUM;
config.pin_vsync = VSYNC_GPIO_NUM;
config.pin_href = HREF_GPIO_NUM;
config.pin_sscb_sda = SIOD_GPIO_NUM;
config.pin_sscb_scl = SIOC_GPIO_NUM;
config.pin_pwdn = PWDN_GPIO_NUM;
config.pin_reset = RESET_GPIO_NUM;
config.xclk_freq_hz = 20000000;
config.pixel_format = PIXFORMAT_JPEG;
// if PSRAM IC present, init with UXGA resolution and higher JPEG quality
// for larger pre-allocated frame buffer.
if(psramFound()){
config.frame_size = FRAMESIZE_UXGA;
config.jpeg_quality = 10;
config.fb_count = 1;
} else {
config.frame_size = FRAMESIZE_SVGA;
config.jpeg_quality = 12;
config.fb_count = 1;
}
#if defined(CAMERA_MODEL_ESP_EYE)
pinMode(13, INPUT_PULLUP);
pinMode(14, INPUT_PULLUP);
#endif
// camera init
esp_err_t err = esp_camera_init(&config);
if (err != ESP_OK) {
Serial.printf("Camera init failed with error 0x%x", err);
return;
}
sensor_t * s = esp_camera_sensor_get();
// initial sensors are flipped vertically and colors are a bit saturated
if (s->id.PID == OV3660_PID) {
s->set_vflip(s, 1); // flip it back
s->set_brightness(s, 1); // up the brightness just a bit
s->set_saturation(s, -2); // lower the saturation
}
// drop down frame size for higher initial frame rate
//FRAMESIZE_VGA, // 640x480
//FRAMESIZE_SVGA, // 800x600
//FRAMESIZE_XGA, // 1024x768
//FRAMESIZE_HD, // 1280x720
//FRAMESIZE_SXGA, // 1280x1024
//FRAMESIZE_UXGA, // 1600x1200
s->set_framesize(s, FRAMESIZE_XGA);
#if defined(CAMERA_MODEL_M5STACK_WIDE) || defined(CAMERA_MODEL_M5STACK_ESP32CAM)
s->set_vflip(s, 1);
s->set_hmirror(s, 1);
#endif
}
void loop() {
// put your main code here, to run repeatedly:
if(bConnecting)
{
if (client.connect(server, 80))
{
Serial.println("connected");
bConnecting = false;
}
}
else
{
if(client.connected())
{
String Data = client.readStringUntil(':');
camera_fb_t * fb = esp_camera_fb_get();
if (!fb)
{
Serial.println("Frame buffer could not be acquired");
return;
}
String message = String(fb->width) + "," + String(fb->height) + "," + String(fb->len);
Serial.println(message);
int len = fb->len;
client.write((byte*)&len, sizeof(int));
client.write((byte*)fb->buf, fb->len);
esp_camera_fb_return(fb);
}
else
{
Serial.println("lost connection with server");
bConnecting = true;
}
}
}
I decided to write the client side here, as you can see on the loop function, I am checking if the connection is alive, then read some data and finally send the image buffer. For that I went the easy way, I am sending the length as an int over the first 4 bytes, then I am simply sending the jpeg data.
Now lets see the python code:
# echo-server.py
import socket
import cv2
import numpy as np
HOST = "192.168.2.29" # Standard loopback interface address (localhost)
PORT = 80 # Port to listen on (non-privileged ports are > 1023)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((HOST, PORT))
s.listen()
conn, addr = s.accept()
with conn:
print(f"Connected by {addr}")
conn.sendall("handshake:".encode())
while True:
data = conn.recv(4)
if not data:
print("no len")
break
message_length = int.from_bytes(data, "little")
print("message received len: " + str(message_length))
jpeg_data = None
jpeg_data = recvall(conn, message_length)
if not jpeg_data:
print("received header but not jpeg")
break
image = cv2.imdecode(np.fromstring(bytes(jpeg_data), dtype=np.uint8), cv2.IMREAD_UNCHANGED)
cv2.imshow('image', image)
# send awnser back
conn.sendall("back:".encode())
if cv2.waitKey(1) & 0xFF == ord('q'):
break
On python I am creating a TCP server an listening for connections, once a connection is found, I am simply locking all to a while loop, in this loop I can read the first 4 bytes, get the length of the jpeg data and the simply process to get that data and repeat.
As you can see, the example is so minimal, I am really not sure how is possible that the demo works so fast.