Memory Issues in Script on XiaoEsp32s3

Projects Programming
1

Hi All,
I am trying to port this AI Vision project from TechieSMS over to XIAOESP32-S3 Sense Board.

Basically I'm trying to capture a photo (no need to save it) and ask a preprogrammed question about it to OpenAI API and then print the output to the serial monitor.

I'm getting weird error message when it connects to WiFi. I've confirmed that it can connect to WiFi with other scripts.

Here's his project on ESP32-CAM board:
Youtube Video

Here's my code:

/*
  ESP32-CAM Image Analysis with OpenAI API and Capacitive Touch

  This code captures an image using the ESP32-CAM module, processes it, 
  and sends it to OpenAI's GPT-4o API for analysis. It uses capacitive touch 
  for triggering instead of a physical button.

  Tested with:
  - Arduino IDE version 2.3.2
  - ESP32 boards package version 3.0.0
  - ArduinoJson library version 7.1.0
  - Base64 library (default version with ESP32 boards package)

  Make sure to install these libraries and configure your environment 
  as specified above before running the code.
*/

#include <WiFi.h>
#include <HTTPClient.h>
#include <Base64.h>
#include "esp_camera.h"
#include <ArduinoJson.h>

// WiFi credentials
const char* ssid = "mywifi";
const char* password = "mywifipassword";

// OpenAI API key
const String apiKey = "API KEY";

// Question to be Asked about the image
String Question = "Summarize the image";

// Pin definitions for ESP32-CAM AI-Thinker module
#define PWDN_GPIO_NUM 32
#define RESET_GPIO_NUM -1
#define XCLK_GPIO_NUM 0
#define SIOD_GPIO_NUM 26
#define SIOC_GPIO_NUM 27
#define Y9_GPIO_NUM 35
#define Y8_GPIO_NUM 34
#define Y7_GPIO_NUM 39
#define Y6_GPIO_NUM 36
#define Y5_GPIO_NUM 21
#define Y4_GPIO_NUM 19
#define Y3_GPIO_NUM 18
#define Y2_GPIO_NUM 5
#define VSYNC_GPIO_NUM 25
#define HREF_GPIO_NUM 23
#define PCLK_GPIO_NUM 22

// Capacitive touch pin for ESP32
#define TOUCH_PIN A5 // T0 corresponds to GPIO4

#define BUZZER_PIN 2  // Buzzer connected to GPIO2

// Function to encode image to Base64
String encodeImageToBase64(const uint8_t* imageData, size_t imageSize) {
  return base64::encode(imageData, imageSize);
}

void setup() {
  Serial.begin(115200);
  WiFi.begin(ssid, password);

  pinMode(BUZZER_PIN, OUTPUT);  // Set Buzzer pin as output

  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.println("Connecting to WiFi...");
  }

  Serial.println("WiFi Connected!");

  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_JPEG;
  config.frame_size = FRAMESIZE_QVGA;
  config.jpeg_quality = 10;
  config.fb_count = 1;

  if (esp_camera_init(&config) != ESP_OK) {
    Serial.println("Camera init failed");
    return;
  }

  Serial.println("Camera Initialized");
}

void captureAndAnalyzeImage() {
  Serial.println("Capturing image...");

  camera_fb_t* fb = esp_camera_fb_get();  // Get the frame buffer
  if (!fb) {
    Serial.println("Camera capture failed");
    return;
  }

  Serial.println("Image captured");
  String base64Image = encodeImageToBase64(fb->buf, fb->len);

  beep();
  esp_camera_fb_return(fb);  // Return the frame buffer to free memory

  if (base64Image.isEmpty()) {
    Serial.println("Failed to encode the image!");
    return;
  }
  
  AnalyzeImage(base64Image);
}

void AnalyzeImage(const String& base64Image) {
  Serial.println("Sending image for analysis...");

  String result;
  String url = "data:image/jpeg;base64," + base64Image;

  DynamicJsonDocument doc(4096);
  doc["model"] = "gpt-4o";
  JsonArray messages = doc.createNestedArray("messages");
  JsonObject message = messages.createNestedObject();
  message["role"] = "user";
  JsonArray content = message.createNestedArray("content");
  JsonObject textContent = content.createNestedObject();
  textContent["type"] = "text";
  textContent["text"] = "Summarize the context of this image?";

  JsonObject imageContent = content.createNestedObject();
  imageContent["type"] = "image_url";
  JsonObject imageUrlObject = imageContent.createNestedObject("image_url");
  imageUrlObject["url"] = url;
  imageContent["image_url"]["detail"] = "auto";

  doc["max_tokens"] = 400;

  String jsonPayload;
  serializeJson(doc, jsonPayload);

  if (sendPostRequest(jsonPayload, result)) {
    Serial.print("[ChatGPT] Response: ");
    Serial.println(result);

    DynamicJsonDocument responseDoc(4096);
    deserializeJson(responseDoc, result);

    String responseContent = responseDoc["choices"][0]["message"]["content"].as<String>();
    Serial.println("[ChatGPT] Parsed response: " + responseContent);
  } else {
    Serial.print("[ChatGPT] Error: ");
    Serial.println(result);
  }
}

bool sendPostRequest(const String& payload, String& result) {
  HTTPClient http;
  http.begin("https://api.openai.com/v1/chat/completions");

  http.addHeader("Content-Type", "application/json");
  http.addHeader("Authorization", "Bearer " + apiKey);
  http.setTimeout(20000);

  Serial.print("Payload size: ");
  Serial.println(payload.length());

  int httpResponseCode = http.POST(payload);

  if (httpResponseCode > 0) {
    result = http.getString();
    Serial.println("HTTP Response Code: " + String(httpResponseCode));
    Serial.println("Response Body: " + result);
    http.end();
    return true;
  } else {
    result = "HTTP request failed, response code: " + String(httpResponseCode);
    Serial.println("Error Code: " + String(httpResponseCode));
    Serial.println("Error Message: " + http.errorToString(httpResponseCode));
    http.end();
    return false;
  }
}

void loop() {
  // Use capacitive touch instead of button press
  int touchValue = touchRead(TOUCH_PIN);
  if (touchValue < 30) {  // Adjust this threshold as needed for your environment
    Serial.println("Touch detected! Capturing image...");
    captureAndAnalyzeImage();
    delay(1000);  // Small delay to debounce touch
  }
}

void beep(){
  digitalWrite(2,HIGH);
  delay(300);
  digitalWrite(2,LOW);
  
}

The error I'm getting is:

 
Here's the error message: 
Connecting to WiFi...Guru Meditation Error: Core  1 panic'ed (LoadProhibited). Exception was unhandled.
20:44:32.042 -> 
20:44:32.042 -> Core  1 register dump:
20:44:32.042 -> PC      : 0x42050989  PS      : 0x00060230  A0      : 0x8204969e  A1      : 0x3fcab600  
20:44:32.042 -> A2      : 0x3fcb7004  A3      : 0x3fcab6d8  A4      : 0xffff8fff  A5      : 0x00000064  
20:44:32.042 -> A6      : 0x3c0d6e2c  A7      : 0x00000001  A8      : 0x3c0d6e84  A9      : 0x00000000  
20:44:32.042 -> A10     : 0x00000016  A11     : 0x00000060  A12     : 0x00000060  A13     : 0x00000064  
20:44:32.042 -> A14     : 0x00000000  A15     : 0x00000002  SAR     : 0x00000011  EXCCAUSE: 0x0000001c  
20:44:32.042 -> EXCVADDR: 0x00000000  LBEG    : 0x400570e8  LEND    : 0x400570f3  LCOUNT  : 0x00000000  
20:35:10.733 -> PC      : 0x42050989  PS      : 0x00060230  A0      : 0x8204969e  A1      : 0x3fcab570
20:35:10.733 -> A2      : 0x3fcb7004  A3      : 0x3fcab648  A4      : 0xffff8fff  A5      : 0x00000064
20:35:10.733 -> A6      : 0x3c0d6e2c  A7      : 0x00000001  A8      : 0x3c0d6e84  A9      : 0x00000000
20:35:10.733 -> A10     : 0x00000016  A11     : 0x00000060  A12     : 0x00000060  A13     : 0x00000064
20:35:10.733 -> A14     : 0x00000000  A15     : 0x00000002  SAR     : 0x00000015  EXCCAUSE: 0x0000001c
20:35:10.733 -> EXCVADDR: 0x00000000  LBEG    : 0x400570e8  LEND    : 0x400570f3  LCOUNT  : 0x00000000
20:35:10.733 -> 
20:35:10.733 -> 
20:35:10.733 -> Backtrace: 0x42050986:0x3fcab570 0x4204969b:0x3fcab5c0 0x42049222:0x3fcab5f0 0x420034a1:0x3fcab640 0x4200efb7:0x3fcab6d0 0x4037df46:0x3fcab6f0
20:35:10.733 -> 
20:35:10.733 -> 
20:35:10.733 ->
2

The pins used by the camera on the ESP32CAM and the camera used on the Seeed XIAO ESP32S3 board are different. Different processor of course too.

What the implications are for Arduino IDE 2, I don't know, I don't use it.

1 Like
3

Now take every 1st number of each pair in a backtrace (i.e. 0x42050986, 0x420...) and pass it to "addr2line" program:

addr2line.exe -pfiaC -e file.elf 0x4204969b

file.elf - is your compiled firmware. It is usually can be found in C:\Users\YourName... :slight_smile:

image
image1920×1080 219 KB

Running addr2line.exe for every backtrace number will show you how all this (a crash) happened.

4

No no no. Can not use ai-thinker pinout on S3. I am pretty sure thats the reason why your firmware crashes.

2 Likes
5

Attaching functional code below. Thanks for all who assisted!

/*
 SEEED STUDIO XIAOESP32-S3 Sense Image Analysis with OpenAI API and Capacitive Touch

  This code captures an image using the Xiao ESP32S3 module, processes it, 
  and sends it to OpenAI's GPT-4o API for analysis. It uses capacitive touch 
  for triggering instead of a physical button. Adjust touch threshold to prevent touch errors.

  Tested with:
  - Arduino IDE version 2.3.2
  - ESP32 boards package version 3.0.0
  - ArduinoJson library version 7.1.0
  - Base64 library (default version with ESP32 boards package)

  Make sure to install these libraries and configure your environment 
  as specified above before running the code.
*/

#include <WiFi.h>
#include <HTTPClient.h>
#include <Base64.h>
#include "esp_camera.h"
#include <ArduinoJson.h>
#define CAMERA_MODEL_XIAO_ESP32S3 // Has PSRAM - make sure you enable this in board settings before uploading
#include "camera_pins.h" //get this from seeedstudio's github.

// WiFi credentials 
const char* ssid = "mywifissid"; // replace with your wifi hotspot name
const char* password = "mywifipassword"; //replace with your wifi password

// OpenAI API key
const String apiKey = "myapikey"; //replace with your openai api key.

// Question to be Asked about the image
String Question = "Summarize the image";

// Pin definitions

#define BUZZER_PIN 2  // Buzzer connected to GPIO2 //untested.

// Function to encode image to Base64
String encodeImageToBase64(const uint8_t* imageData, size_t imageSize) {
  return base64::encode(imageData, imageSize);
}

int threshold = 20; //this may be too sensitive. lower the number if so.
bool touch1detected = false;

void gotTouch1(){
 touch1detected = true;
}

void setup() {
  Serial.begin(115200);
  WiFi.begin(ssid, password);
  touchAttachInterrupt(A5, gotTouch1, threshold);




  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.println("Connecting to WiFi...");
  }

  Serial.println("WiFi Connected!");

  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.frame_size = FRAMESIZE_UXGA;
  config.pixel_format = PIXFORMAT_JPEG; // for streaming
  //config.pixel_format = PIXFORMAT_RGB565; // for face detection/recognition
  config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
  config.fb_location = CAMERA_FB_IN_PSRAM;
  config.jpeg_quality = 12;
  config.fb_count = 1;

  if (esp_camera_init(&config) != ESP_OK) {
    Serial.println("Camera init failed");
    return;
  }

  Serial.println("Camera Initialized");


  
}

void captureAndAnalyzeImage() {
  Serial.println("Capturing image...");

  camera_fb_t* fb = esp_camera_fb_get();  // Get the frame buffer
  if (!fb) {
    Serial.println("Camera capture failed");
    return;
  }

  Serial.println("Image captured");
  String base64Image = encodeImageToBase64(fb->buf, fb->len);
  Serial.println(base64Image);

  beep();
  esp_camera_fb_return(fb);  // Return the frame buffer to free memory

  if (base64Image.isEmpty()) {
    Serial.println("Failed to encode the image!");
    return;
  }
  
  AnalyzeImage(base64Image);
}

void AnalyzeImage(const String& base64Image) {
  Serial.println("Sending image for analysis...");

  String result;
  String url = "data:image/jpeg;base64," + base64Image;

  DynamicJsonDocument doc(4096);
  doc["model"] = "gpt-4o-2024-08-06";
  JsonArray messages = doc.createNestedArray("messages");
  JsonObject message = messages.createNestedObject();
  message["role"] = "user";
  JsonArray content = message.createNestedArray("content");
  JsonObject textContent = content.createNestedObject();
  textContent["type"] = "text";
  textContent["text"] = "Summarize the context of this image?";

  JsonObject imageContent = content.createNestedObject();
  imageContent["type"] = "image_url";
  JsonObject imageUrlObject = imageContent.createNestedObject("image_url");
  imageUrlObject["url"] = url;
  imageContent["image_url"]["detail"] = "auto";

  doc["max_tokens"] = 400;

  String jsonPayload;
  serializeJson(doc, jsonPayload);

  if (sendPostRequest(jsonPayload, result)) {
    Serial.print("[ChatGPT] Response: ");
    Serial.println(result);

    DynamicJsonDocument responseDoc(4096);
    deserializeJson(responseDoc, result);

    String responseContent = responseDoc["choices"][0]["message"]["content"].as<String>();
    Serial.println("[ChatGPT] Parsed response: " + responseContent);
  } else {
    Serial.print("[ChatGPT] Error: ");
    Serial.println(result);
  }
}

bool sendPostRequest(const String& payload, String& result) {
  HTTPClient http;
  http.begin("https://api.openai.com/v1/chat/completions");

  http.addHeader("Content-Type", "application/json");
  http.addHeader("Authorization", "Bearer " + apiKey);
  http.setTimeout(20000);

  Serial.print("Payload size: ");
  Serial.println(payload.length());

  int httpResponseCode = http.POST(payload);

  if (httpResponseCode > 0) {
    result = http.getString();
    Serial.println("HTTP Response Code: " + String(httpResponseCode));
    Serial.println("Response Body: " + result);
    http.end();
    return true;
  } else {
    result = "HTTP request failed, response code: " + String(httpResponseCode);
    Serial.println("Error Code: " + String(httpResponseCode));
    Serial.println("Error Message: " + http.errorToString(httpResponseCode));
    http.end();
    return false;
  }
}

void loop() {
  
 if(touch1detected){
    touch1detected = false;
    Serial.println("Touch 1 detected");
    Serial.println("Touch detected! Capturing image...");
      captureAndAnalyzeImage();
  }
       
   
    
  }

 

void beep(){
  digitalWrite(2,HIGH);
  delay(300);
  digitalWrite(2,LOW);
  
}
6

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.