Hi everyone!
I turn to you as I am out of ideas about how to debug this!
I am running a TensorFlow lite model on the Nano 33 BLE sense. The model is used to recognise an object placed in front of the Nano and then print the index 1, 2 or 3 depending on the results from the Tensor Flow Lite Object Classifier code. The sketch runs as intended if I comment out all the BLE related code, I get either 1, 2 or 3 printed out on the serial monitor.
I have tested the BLE code by commenting out all the TensorFlow lite code and using the proximity sensor on the Nano, this also works as intended and I am able to see this data in the LighBlue app on my phone.
The problem I have is when I run the sketch with the TensorFlow Lite and the BLE together. I don't get any error messages, the code compiles and I see 0's printed on the Serial but as soon as I try to scan an item with Nano it makes the program crash - the Serial freezes and LED starts flashing on the Nano.
int CLASSES [4] = {3,2,1}; This used to be const char * CLASSES [] = {"3","2","1"} ; I had to change it to int to be able to use it with the buttonCharacteristic.writeValue(buttonValue);
I would much appreciate any suggestions or thoughts on what could be going wrong here.
Also maybe someone knows a way to debug Arduino code like in Thonny where you can see step-by-step what is happening in the program?
/*
Object classifier by color
--------------------------
Uses RGB color sensor input to Neural Network to classify objects
Outputs object class to serial using unicode emojis
Note: The direct use of C/C++ pointers, namespaces, and dynamic memory is generally
discouraged in Arduino examples, and in the future the TensorFlowLite library
might change to make the sketch simpler.
Hardware: Arduino Nano 33 BLE Sense board.
Created by Don Coleman, Sandeep Mistry
Adapted by Dominic Pajak
This example code is in the public domain.
*/
// Arduino_TensorFlowLite - Version: 0.alpha.precompiled
#include <TensorFlowLite.h>
#include <tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h>
#include <tensorflow/lite/experimental/micro/micro_error_reporter.h>
#include <tensorflow/lite/experimental/micro/micro_interpreter.h>
#include <tensorflow/lite/schema/schema_generated.h>
#include <tensorflow/lite/version.h>
#include <Arduino_APDS9960.h>
#include "model.h"
// global variables used for TensorFlow Lite (Micro)
tflite::MicroErrorReporter tflErrorReporter;
// pull in all the TFLM ops, you can remove this line and
// only pull in the TFLM ops you need, if would like to reduce
// the compiled size of the sketch.
tflite::ops::micro::AllOpsResolver tflOpsResolver;
const tflite::Model* tflModel = nullptr;
tflite::MicroInterpreter* tflInterpreter = nullptr;
TfLiteTensor* tflInputTensor = nullptr;
TfLiteTensor* tflOutputTensor = nullptr;
// Create a static memory buffer for TFLM, the size may need to
// be adjusted based on the model you are using
constexpr int tensorArenaSize = 8 * 1024;
byte tensorArena[tensorArenaSize];
// array to map gesture index to a name
int CLASSES[4] = {3, 2, 1};
// u8"\U0001F34E", blue 3 //
// u8"\U0001F34C", green 2//
// u8"\U0001F34A" red 1//
#define NUM_CLASSES (sizeof(CLASSES) / sizeof(CLASSES[0]))
#include <ArduinoBLE.h>
BLEService ledService("19B10010-E8F2-537E-4F6C-D104768A1214"); // create service
// create button characteristic and allow remote device to get notifications
BLEIntCharacteristic buttonCharacteristic("19B10012-E8F2-537E-4F6C-D104768A1214", BLERead | BLENotify);
void setup() {
Serial.begin(9600);
while (!Serial) {};
//begin initialization
if (!BLE.begin()) {
Serial.println("starting BLE failed!");
while (1);
}
if (!APDS.begin()) {
Serial.println("Error initializing APDS9960 sensor.");
}
// get the TFL representation of the model byte array
tflModel = tflite::GetModel(model);
if (tflModel->version() != TFLITE_SCHEMA_VERSION) {
Serial.println("Model schema mismatch!");
while (1);
}
// Create an interpreter to run the model
tflInterpreter = new tflite::MicroInterpreter(tflModel, tflOpsResolver, tensorArena, tensorArenaSize, &tflErrorReporter);
// Allocate memory for the model's input and output tensors
tflInterpreter->AllocateTensors();
// Get pointers for the model's input and output tensors
tflInputTensor = tflInterpreter->input(0);
tflOutputTensor = tflInterpreter->output(0);
// set the local name peripheral advertises
BLE.setLocalName("nano33");
// set the UUID for the service this peripheral advertises:
BLE.setAdvertisedService(ledService);
// add the characteristics to the service
ledService.addCharacteristic(buttonCharacteristic);
// add the service
BLE.addService(ledService);
buttonCharacteristic.writeValue(0);
// start advertising
BLE.advertise();
Serial.println("Bluetooth device active, waiting for connections...");
}
void loop() {
int r, g, b, p, c;
int buttonValue;
float sum;
//poll for BLE events
BLE.poll();
// check if both color and proximity data sample is available
while (!APDS.colorAvailable() || !APDS.proximityAvailable()) {}
// read the color and proximity sensor
APDS.readColor(r, g, b, c);
p = APDS.readProximity();
sum = r + g + b;
// check if there's an object close and well illuminated enough
if (p == 0 && c > 10 && sum > 0) {
// normalize
float redRatio = r / sum;
float greenRatio = g / sum;
float blueRatio = b / sum;
// input sensor data to tensorflow
tflInputTensor->data.f[0] = redRatio;
tflInputTensor->data.f[1] = greenRatio;
tflInputTensor->data.f[2] = blueRatio;
// run inferencing
TfLiteStatus invokeStatus = tflInterpreter->Invoke();
if (invokeStatus != kTfLiteOk) {
Serial.println("Invoke failed!");
while (1);
return;
}
// output results
int maxPercent = -1;
int maxIndex = -1;
for (int i = 0; i < NUM_CLASSES; i++) {
int percent = (int(tflOutputTensor->data.f[i] * 100));
if ( percent > maxPercent) { //maxPercent is currently highest percentage from the i that the for loop has gone through
maxPercent = percent;
maxIndex = i;
}
(CLASSES[i]);
(" ");
(int(tflOutputTensor->data.f[i] * 100));
("%\n");
}
Serial.print(CLASSES[maxIndex]);
buttonValue = CLASSES[maxIndex];
// wait for the object to be moved away
while (!APDS.proximityAvailable() || (APDS.readProximity() == 0)) {}
}
buttonCharacteristic.writeValue(buttonValue);
Serial.println(buttonValue);
}