So I'm trying to use a barebones ESP32 WROVER IE with an OV7670 (not esp32 cam)
I'm using JPEGENC and a base64 encoder to encode it into a base64 string which I then display on my webserver. This all works fine and dandy but the problem is that I am getting very weird results when using RGB565 (JPEGEnc only supports rgb565 or grayscale, but I want colour images)
I'm not sure why any of these problems are happening. My code: (sorry for making it very long, currently the only part that is actually used, where the camera capture and encoding happens, is in the function
cs_ei_get_buffer_encode_to_png on line 414):
/* Edge Impulse Arduino examples
* Copyright (c) 2022 EdgeImpulse Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
// These sketches are tested with 2.0.4 ESP32 Arduino Core
// https://github.com/espressif/arduino-esp32/releases/tag/2.0.4
/* Includes ---------------------------------------------------------------- */
#include <faec_detection_inferencing.h>
#include "edge-impulse-sdk/dsp/image/image.hpp"
#include "esp_camera.h"
#include <WiFi.h>
#include <JPEGENC.h>
#include <base64.hpp>
#include <ArduinoJson.h>
// Select camera model - find more camera models in camera_pins.h file here
// https://github.com/espressif/arduino-esp32/blob/master/libraries/ESP32/examples/Camera/CameraWebServer/camera_pins.h
#define CAMERA_MODEL_ESP_EYE // Has PSRAM
//#define CAMERA_MODEL_AI_THINKER // Has PSRAM
#define PWDN_GPIO_NUM 32
#define RESET_GPIO_NUM -1
#define XCLK_GPIO_NUM 0
#define SIOD_GPIO_NUM 26
#define SIOC_GPIO_NUM 27
#define Y9_GPIO_NUM 35
#define Y8_GPIO_NUM 34
#define Y7_GPIO_NUM 39
#define Y6_GPIO_NUM 36
#define Y5_GPIO_NUM 21
#define Y4_GPIO_NUM 19
#define Y3_GPIO_NUM 18
#define Y2_GPIO_NUM 5
#define VSYNC_GPIO_NUM 25
#define HREF_GPIO_NUM 23
#define PCLK_GPIO_NUM 22
//#error "Camera model not selected"
/* Constant defines -------------------------------------------------------- */
#define EI_CAMERA_RAW_FRAME_BUFFER_COLS 640
#define EI_CAMERA_RAW_FRAME_BUFFER_ROWS 480
#define EI_CAMERA_FRAME_BYTE_SIZE 3
/* Private variables ------------------------------------------------------- */
static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal
static bool is_initialised = false;
uint8_t *snapshot_buf; //points to the output of the capture
static camera_config_t camera_config = {
.pin_pwdn = PWDN_GPIO_NUM,
.pin_reset = RESET_GPIO_NUM,
.pin_xclk = XCLK_GPIO_NUM,
.pin_sscb_sda = SIOD_GPIO_NUM,
.pin_sscb_scl = SIOC_GPIO_NUM,
.pin_d7 = Y9_GPIO_NUM,
.pin_d6 = Y8_GPIO_NUM,
.pin_d5 = Y7_GPIO_NUM,
.pin_d4 = Y6_GPIO_NUM,
.pin_d3 = Y5_GPIO_NUM,
.pin_d2 = Y4_GPIO_NUM,
.pin_d1 = Y3_GPIO_NUM,
.pin_d0 = Y2_GPIO_NUM,
.pin_vsync = VSYNC_GPIO_NUM,
.pin_href = HREF_GPIO_NUM,
.pin_pclk = PCLK_GPIO_NUM,
//XCLK 20MHz or 10MHz for OV2640 double FPS (Experimental)
.xclk_freq_hz = 10000000,
.ledc_timer = LEDC_TIMER_0,
.ledc_channel = LEDC_CHANNEL_0,
.pixel_format = PIXFORMAT_GRAYSCALE, //YUV422,GRAYSCALE,RGB565,JPEG
.frame_size = FRAMESIZE_QVGA, //QQVGA-UXGA Do not use sizes above QVGA when not JPEG
.jpeg_quality = 12, //0-63 lower number means higher quality
.fb_count = 1, //if more than one, i2s runs in continuous mode. Use only with JPEG
.fb_location = CAMERA_FB_IN_PSRAM,
.grab_mode = CAMERA_GRAB_WHEN_EMPTY,
};
// Dual core tools
TaskHandle_t WebServerTask;
TaskHandle_t CamTask;
ei::signal_t current_signal;
/* Function definitions ------------------------------------------------------- */
bool ei_camera_init(void);
void ei_camera_deinit(void);
bool ei_camera_capture(uint32_t img_width, uint32_t img_height, uint8_t *out_buf);
/**
* @brief Arduino setup function
*/
void setup() {
// put your setup code here, to run once:
Serial.begin(115200);
//comment out the below line to start inference immediately after upload
while (!Serial)
;
Serial.println("Webserver is initializing on the second core.");
xTaskCreatePinnedToCore(
WebServer, /* Function to implement the task */
"Webserver", /* Name of the task */
10000, /* Stack size in words */
NULL, /* Task input parameter */
0, /* Priority of the task */
&WebServerTask, /* Task handle. */
1); /* Core where the task should run */
Serial.println("Edge Impulse Inferencing Demo");
if (ei_camera_init() == false) {
ei_printf("Failed to initialize Camera!\r\n");
} else {
ei_printf("Camera initialized\r\n");
}
ei_printf("\nStarting continious inference in 2 seconds...\n");
ei_sleep(2000);
}
/**
* @brief Get data and run inferencing
*
* @param[in] debug Get debug info if true
*/
void loop() {
// instead of wait_ms, we'll wait on the signal, this allows threads to cancel us...
if (ei_sleep(5) != EI_IMPULSE_OK) {
return;
}
/*
snapshot_buf = (uint8_t *)malloc(EI_CAMERA_RAW_FRAME_BUFFER_COLS * EI_CAMERA_RAW_FRAME_BUFFER_ROWS * EI_CAMERA_FRAME_BYTE_SIZE);
// check if allocation was successful
if (snapshot_buf == nullptr) {
ei_printf("ERR: Failed to allocate snapshot buffer!\n");
return;
}
ei::signal_t signal;
signal.total_length = EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT;
signal.get_data = &ei_camera_get_data;
current_signal = signal;
if (ei_camera_capture((size_t)EI_CLASSIFIER_INPUT_WIDTH, (size_t)EI_CLASSIFIER_INPUT_HEIGHT, snapshot_buf) == false) {
//ei_printf("Failed to capture image\r\n");
free(snapshot_buf);
return;
}
//Serial.println(String((char *)snapshot_buf));
// Run the classifier
ei_impulse_result_t result = { 0 };
EI_IMPULSE_ERROR err = run_classifier(&signal, &result, debug_nn);
if (err != EI_IMPULSE_OK) {
ei_printf("ERR: Failed to run classifier (%d)\n", err);
return;
}
// print the predictions
ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
result.timing.dsp, result.timing.classification, result.timing.anomaly);
#if EI_CLASSIFIER_OBJECT_DETECTION == 1
ei_printf("Object detection bounding boxes:\r\n");
for (uint32_t i = 0; i < result.bounding_boxes_count; i++) {
ei_impulse_result_bounding_box_t bb = result.bounding_boxes[i];
if (bb.value == 0) {
continue;
}
ei_printf(" %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\r\n",
bb.label,
bb.value,
bb.x,
bb.y,
bb.width,
bb.height);
}
// Print the prediction results (classification)
#else
ei_printf("Predictions:\r\n");
for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
ei_printf(" %s: ", ei_classifier_inferencing_categories[i]);
ei_printf("%.5f\r\n", result.classification[i].value);
}
#endif
// Print anomaly result (if it exists)
#if EI_CLASSIFIER_HAS_ANOMALY
ei_printf("Anomaly prediction: %.3f\r\n", result.anomaly);
#endif
#if EI_CLASSIFIER_HAS_VISUAL_ANOMALY
ei_printf("Visual anomalies:\r\n");
for (uint32_t i = 0; i < result.visual_ad_count; i++) {
ei_impulse_result_bounding_box_t bb = result.visual_ad_grid_cells[i];
if (bb.value == 0) {
continue;
}
ei_printf(" %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\r\n",
bb.label,
bb.value,
bb.x,
bb.y,
bb.width,
bb.height);
}
#endif
free(snapshot_buf);
*/
}
/**
* @brief Setup image sensor & start streaming
*
* @retval false if initialisation failed
*/
bool ei_camera_init(void) {
if (is_initialised) return true;
#if defined(CAMERA_MODEL_ESP_EYE)
pinMode(13, INPUT_PULLUP);
pinMode(14, INPUT_PULLUP);
#endif
//initialize the camera
esp_err_t err = esp_camera_init(&camera_config);
if (err != ESP_OK) {
Serial.printf("Camera init failed with error 0x%x\n", err);
return false;
}
sensor_t *s = esp_camera_sensor_get();
// initial sensors are flipped vertically and colors are a bit saturated
if (s->id.PID == OV3660_PID) {
s->set_vflip(s, 1); // flip it back
s->set_brightness(s, 1); // up the brightness just a bit
s->set_saturation(s, 0); // lower the saturation
}
#if defined(CAMERA_MODEL_M5STACK_WIDE)
s->set_vflip(s, 1);
s->set_hmirror(s, 1);
#elif defined(CAMERA_MODEL_ESP_EYE)
s->set_vflip(s, 1);
s->set_hmirror(s, 1);
s->set_awb_gain(s, 1);
#endif
is_initialised = true;
return true;
}
/**
* @brief Stop streaming of sensor data
*/
void ei_camera_deinit(void) {
//deinitialize the camera
esp_err_t err = esp_camera_deinit();
if (err != ESP_OK) {
ei_printf("Camera deinit failed\n");
return;
}
is_initialised = false;
return;
}
bool buffering_c = false;
/**
* @brief Capture, rescale and crop image
*
* @param[in] img_width width of output image
* @param[in] img_height height of output image
* @param[in] out_buf pointer to store output image, NULL may be used
* if ei_camera_frame_buffer is to be used for capture and resize/cropping.
*
* @retval false if not initialised, image captured, rescaled or cropped failed
*
*/
bool ei_camera_capture(uint32_t img_width, uint32_t img_height, uint8_t *out_buf) {
if (buffering_c == true) {
ei_printf("You cannot use ei_camera_capture on a thread while another thread is running it!\n");
return false;
}
buffering_c = true;
bool do_resize = false;
if (!is_initialised) {
//ei_camera_init();
//ei_printf("ERR: Camera is not initialized\r\n");
buffering_c = false;
return false;
}
camera_fb_t *fb = esp_camera_fb_get();
if (!fb) {
ei_printf("Camera capture failed\n");
buffering_c = false;
return false;
}
bool converted = fmt2rgb888(fb->buf, fb->len, PIXFORMAT_RGB565, snapshot_buf);
esp_camera_fb_return(fb);
if (!converted) {
ei_printf("Conversion failed\n");
buffering_c = false;
return false;
}
if ((img_width != EI_CAMERA_RAW_FRAME_BUFFER_COLS)
|| (img_height != EI_CAMERA_RAW_FRAME_BUFFER_ROWS)) {
do_resize = true;
}
if (do_resize) {
ei::image::processing::crop_and_interpolate_rgb888(
out_buf,
EI_CAMERA_RAW_FRAME_BUFFER_COLS,
EI_CAMERA_RAW_FRAME_BUFFER_ROWS,
out_buf,
img_width,
img_height);
}
buffering_c = false;
return true;
}
bool buffering = false;
static int ei_camera_get_data(size_t offset, size_t length, float *out_ptr) {
// if another call to get camera data is active, returns false
if (buffering == true) {
ei_printf("You cannot use ei_camera_get_data on a thread while another thread is running it!\n");
return 0;
}
buffering = true;
// we already have a RGB888 buffer, so recalculate offset into pixel index
size_t pixel_ix = offset * 3;
size_t pixels_left = length;
size_t out_ptr_ix = 0;
while (pixels_left != 0) {
// Swap BGR to RGB here
// due to https://github.com/espressif/esp32-camera/issues/379
out_ptr[out_ptr_ix] = (snapshot_buf[pixel_ix + 2] << 16) + (snapshot_buf[pixel_ix + 1] << 8) + snapshot_buf[pixel_ix];
// go to the next pixel
out_ptr_ix++;
pixel_ix += 3;
pixels_left--;
}
// disable buffering
buffering = false;
// and done!
return 0;
}
#if !defined(EI_CLASSIFIER_SENSOR) || EI_CLASSIFIER_SENSOR != EI_CLASSIFIER_SENSOR_CAMERA
#error "Invalid model for current sensor"
#endif
JPEGENC jpg;
JPEGENCODE jpe;
uint8_t *out_jpg = NULL;
String b64 = "";
// cs = custom functiom
void cs_ei_get_buffer_encode_to_png(void *pvParams) {
Serial.println("start1");
Serial.println("lololololol");
if (buffering_c == true) {
Serial.println("lololololololol");
ei_printf("You cannot use ei_camera_capture on a thread while another thread is running it!\n");
return;
}
Serial.println("start1");
Serial.println("start1");
buffering_c = true;
camera_fb_t *fb = esp_camera_fb_get();
Serial.println("got fb");
if (!fb) {
ei_printf("Camera capture failed\n");
buffering_c = false;
return;
}
/*
jpeg docs
int open(const char *szFilename, JPEGE_OPEN_CALLBACK *pfnOpen, JPEGE_CLOSE_CALLBACK *pfnClose, JPEGE_READ_CALLBACK *pfnRead, JPEGE_WRITE_CALLBACK *pfnWrite, JPEGE_SEEK_CALLBACK *pfnSeek);
int open(uint8_t *pOutput, int iBufferSize);
int close();
int encodeBegin(JPEGENCODE *pEncode, int iWidth, int iHeight, uint8_t ucPixelType, uint8_t ucSubSample, uint8_t ucQFactor);
int addMCU(JPEGENCODE *pEncode, uint8_t *pPixels, int iPitch);
int addFrame(JPEGENCODE *pEncode, uint8_t *pPixels, int iPitch);
int getLastError();
*/
size_t out_jpg_len1 = 0, out_jpg_len2;
out_jpg = (uint8_t *)ps_malloc(65536);
Serial.println("outjpg inited");
jpg.open(out_jpg, 65536);
Serial.println("jpg opened");
int rc = jpg.encodeBegin(&jpe, fb->width, fb->height, JPEGE_PIXEL_GRAYSCALE, JPEGE_SUBSAMPLE_420, JPEGE_Q_BEST);
Serial.println("jpg encoded");
if (rc != JPEGE_SUCCESS) {
ei_printf("Failed to encode JPEG");
buffering_c = false;
return;
}
// bytes per line = width * 2 (16 bits/pixel)
jpg.addFrame(&jpe, fb->buf, fb->width * 2);
Serial.println("jpg fb added");
int32_t iDataSize = jpg.close();
Serial.println("jpg closed");
Serial.println("Output file size =");
Serial.println(iDataSize, DEC);
esp_camera_fb_return(fb);
Serial.println("returned");
uint8_t temp;
unsigned char output[65536];
Serial.println("Base64 buffer created");
unsigned int base64_length = encode_base64((unsigned char *)out_jpg, iDataSize, output);
Serial.println("Encoded");
buffering_c = false;
b64 = String((char *)output);
Serial.println("lulzies");
vTaskDelete(NULL);
}
// helpers
//core 2
const char *ssid = "Ramialk_2020_2.4GEXT"; // CHANGE IT
const char *password = "0504915088"; // CHANGE IT
WiFiServer server(80);
String html = R"(
<!DOCTYPE html>
<html>
<head>
<title>ESP32 Camera Webserver</title>
<style>
</style>
<script>
const url = '/update'
let dataBegin = 'data:image/jpeg;base64,'
let data = ''
async function updateData() {
let image = document.getElementById('img')
const response = await fetch(url, {
method: "GET",
})
let json = await response.json()
console.log(json.image)
image.src = dataBegin + json.image
}
</script>
</head>
<body>
<h1>Hello World!</h1>
<img src='' alt='Image from OV7670 cam' id='img'><br>
<button onclick='updateData()'>Update Camera</button>
</body>
</html>
)";
// Current time
unsigned long currentTime = millis();
// Previous time
unsigned long previousTime = 0;
// Define timeout time in milliseconds (example: 2000ms = 2s)
const long timeoutTime = 2000;
String header;
void WebServer(void *pvParams) {
//Serial.begin(9600);
Serial.print("Connecting to ");
Serial.println(ssid);
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
// Print local IP address and start web server
Serial.println("");
Serial.println("WiFi connected.");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
server.begin();
// Start continous loop (required for XTaskCreatePinnedToCore)
for (;;) {
WiFiClient client = server.available();
if (client) {
currentTime = millis();
previousTime = currentTime;
Serial.println("New Client."); // print a message out in the serial port
String currentLine = ""; // make a String to hold incoming data from the client
while (client.connected() && currentTime - previousTime <= timeoutTime) { // loop while the client's connected
currentTime = millis();
if (client.available()) { // if there's bytes to read from the client,
char c = client.read(); // read a byte, then
Serial.write(c); // print it out the serial monitor
header += c;
if (c == '\n') { // if the byte is a newline character
// if the current line is blank, you got two newline characters in a row.
// that's the end of the client HTTP request, so send a response:
if (currentLine.length() == 0) {
// HTTP headers always start with a response code (e.g. HTTP/1.1 200 OK)
// and a content-type so the client knows what's coming, then a blank line:
if (header.indexOf("GET /update") >= 0) {
Serial.println("update");
// do not send HTML
// instead sends back the resolution with JSON
JsonDocument doc;
xTaskCreatePinnedToCore(
cs_ei_get_buffer_encode_to_png, /* Function to implement the task */
"CamTask", /* Name of the task */
100000, /* Stack size in words */
NULL, /* Task input parameter */
1, /* Priority of the task */
&CamTask, /* Task handle. */
1); /* Core where the task should run */
Serial.println("e");
configASSERT(CamTask);
Serial.println("help me");
/*if (CamTask != NULL) {
Serial.println("deletion");
vTaskDelete(CamTask);
}*/
Serial.println("Expecto patronum");
doc["image"] = b64;
// Use the handle to delete the task.
String out;
serializeJson(doc, out);
client.println(out);
//breaks from loop to prevent HTML from being sent
break;
}
client.println("HTTP/1.1 200 OK");
client.println("Content-type:text/html");
client.println("Connection: close");
client.println();
// turns the GPIOs on and off
client.print(html);
// The HTTP response ends with another blank line
client.println();
// Break out of the while loop
break;
} else { // if you got a newline, then clear currentLine
currentLine = "";
}
} else if (c != '\r') { // if you got anything else but a carriage return character,
currentLine += c; // add it to the end of the currentLine
}
}
}
// Clear the header variable
header = "";
// Close the connection
client.stop();
Serial.println("Client disconnected.");
Serial.println("");
}
}
}
I have triple checked my pin connection so I believe it isn't a problem