I want to make a real-time inference of a DL model in Arduino Nano Ble Rev sens

Below is my sketch for a model built in tensor flow and converted to Tflite to make inference in Arduino nano of gesture recognition task.

#include <Arduino_BMI270_BMM150.h>
#include <ArduTFLite.h>
#include "model.h"

const int windowSize = 50;  // Number of timesteps
const int features = 3;     // 3 features: x, y, z
constexpr int tensorArenaSize = 64 * 1024; // Increased tensor arena size
alignas(16) byte tensorArena[tensorArenaSize];

const char* gestures[5] = {
    "horizontal shake",
    "vertical shake",
    "letter S",
    "circle",
    "letter M"
};

float inputWindow[windowSize][features]; // Buffer for 50 timesteps, each with 3 features
int currentIndex = 0;  // Tracks the current position in the buffer

const int expectedInputSize = windowSize * features;  // Manually set the input tensor size
const int outputSize = 5; // Expected number of output classes

// Preprocess input values by normalizing
void preprocessInput(float& value, float offset, float scale) {
    value = (value + offset) / scale;
}

void setup() {
    Serial.begin(9600);  // Faster serial output
    while (!Serial);

    if (!IMU.begin()) {
        Serial.println("Failed to initialize IMU!");
        while (1);
    }
    Serial.println("IMU initialized successfully.");

    Serial.println("Initializing model...");
    if (!modelInit(model, tensorArena, tensorArenaSize)) {
        Serial.println("Model initialization failed!");
        while (1);
    }
    Serial.println("Model initialization done.");

    // Debugging: Print expected model input size
    Serial.print("Expected model input size: ");
    Serial.println(expectedInputSize);
}

void loop() {
    float aX, aY, aZ; 

    if (IMU.accelerationAvailable()) {
        IMU.readAcceleration(aX, aY, aZ);

        // Normalize sensor data
        preprocessInput(aX, 4.0, 8.0);
        preprocessInput(aY, 4.0, 8.0);
        preprocessInput(aZ, 4.0, 8.0);

        // Store in the input window buffer
        inputWindow[currentIndex][0] = aX;
        inputWindow[currentIndex][1] = aY;
        inputWindow[currentIndex][2] = aZ;

        // Update the index in a circular manner
        currentIndex = (currentIndex + 1) % windowSize;
        Serial.println(currentIndex);

        // Run inference once the buffer is full
        if (currentIndex == 0) {
            // Ensure we do not exceed the model's input tensor size
            int actualInputSize = windowSize * features;

            if (expectedInputSize != actualInputSize) {
                Serial.println("Error: Model input size mismatch!");
                Serial.print("Expected: "); Serial.println(expectedInputSize);
                Serial.print("Actual: "); Serial.println(actualInputSize);
                return;
            }

            // Set model input from the input window
            for (int i = 0; i < windowSize; i++) {
                for (int j = 0; j < features; j++) {
                    int index = i * features + j;
                    if (index >= expectedInputSize) { // Prevent out-of-range access
                        Serial.println("Error: Input tensor index out of range!");
                        return;
                    }
                    modelSetInput(inputWindow[i][j], index);
                }
            }

            // Run inference
            if (!modelRunInference()) {
                Serial.println("Error: Model inference failed!");
                return;
            }

            // Retrieve model output
            float maxValue = -1.0;
            int maxIndex = -1;
            float totalOutput = 0.0;
            float gestureOutputs[outputSize];

            Serial.println("Inference Output:");

            for (int i = 0; i < outputSize; i++) {
                // Ensure output index is within range
                if (i >= outputSize) {
                    Serial.println("Error: Output tensor index out of range!");
                    continue;
                }

                float output = modelGetOutput(i);
                gestureOutputs[i] = output;
                totalOutput += output;

                Serial.print("Output[");
                Serial.print(i);
                Serial.print("] (");
                Serial.print(gestures[i]);
                Serial.print("): ");
                Serial.println(output, 6);

                // Find highest confidence gesture
                if (output > maxValue) {
                    maxValue = output;
                    maxIndex = i;
                }
            }

            // Calculate and display probabilities for each gesture
            Serial.println("Gesture Probabilities:");
            for (int i = 0; i < outputSize; i++) {
                float probability = (totalOutput > 0) ? (gestureOutputs[i] / totalOutput) * 100 : 0;
                Serial.print(gestures[i]);
                Serial.print(": ");
                Serial.print(probability, 2);
                Serial.println("%");
            }

            // Print the detected gesture
            if (maxIndex >= 0) {
                Serial.print("Detected Gesture: ");
                Serial.println(gestures[maxIndex]);
            } else {
                Serial.println("No gesture detected.");
            }

            Serial.println();
            delay(3000); // Wait before the next prediction cycle
        }
    }
}

However, I ran into an error the error describe below anytime the above sketch is uploaded to Arduino IDE
Output[2] (letter S): -1.000000

Output tensor index out of range!

Output[3] (circle): -1.000000

Output tensor index out of range!

Output[4] (letter M): -1.000000

Gesture Probabilities:

horizontal shake: 0.00%

vertical shake: 0.00%

letter S: 0.00%

circle: 0.00%

letter M: 0.00%

Detected Gesture: horizontal shake

Did you write this?

I doubt if TFLite is still supported. Most, if not all of the Arduino effort was abandoned several years ago and the Github repository hasn't been updated for over 2 years.

You are much better off learning this stuff using a laptop and Python. There are countless tutorials.

I built model model in python on google colab. I only convert the model to tflite for inference on the arduino nano. but the model is built using sliding window. because I collected the data using arduino nano's accelerometer capturing 5 distinct gestures each gesture has 6 samples.

Yes

            for (int i = 0; i < outputSize; i++) {
                // Ensure output index is within range
                if (i >= outputSize) {
                    Serial.println("Error: Output tensor index out of range!");
                    continue;
                }

Then you should have no problem explaining how the if (i >= outputSize) { block gets executed then, as it apparently does from the output you've shown. Because I've looked at your code snippet and it's not apparent to me how i is ever greater than or equal to outputSize in the loop. Are you passing it by reference to the not present modelGetOutput function and changing it there? There doesn't seem to be anywhere in the loop itself where it's modified outside of the for statement.

I look forward to your answer.

For clarification some of the methods were added by chatgpt like however, below is the original code

#include <Arduino_BMI270_BMM150.h>
#include <ArduTFLite.h>
#include "model.h"

const int windowSize = 50; // Number of timesteps
const int features = 3;    // 3 features: x, y, z
constexpr int tensorArenaSize = 32 * 1024; // Tensor arena size for TFLite
alignas(16) byte tensorArena[tensorArenaSize];

const char* gestures[5] = {
    "horizontal shake",
    "vertical shake",
    "letter S",
    "circle",
    "letter M"
};

float inputWindow[windowSize][features]; 
int currentIndex = 0;                   

void preprocessInput(float& value, float offset, float scale) {
    value = (value + offset) / scale;
}

void setup() {
    Serial.begin(9600);
    while (!Serial);

    if (!IMU.begin()) {
        Serial.println("Failed to initialize IMU!");
        while (1);
    }
    Serial.println("IMU initialized successfully.");

    Serial.println("Initializing model...");
    if (!modelInit(model, tensorArena, tensorArenaSize)) {
        Serial.println("Model initialization failed!");
        while (1);
    }
    Serial.println("Model initialization done.");
}

void loop() {
    float aX, aY, aZ;

    if (IMU.accelerationAvailable()) {
        IMU.readAcceleration(aX, aY, aZ);

        // Normalize sensor data
        preprocessInput(aX, 4.0, 8.0); // Normalize x-axis
        preprocessInput(aY, 4.0, 8.0); // Normalize y-axis
        preprocessInput(aZ, 4.0, 8.0); // Normalize z-axis

        // Store in the input window buffer
        inputWindow[currentIndex][0] = aX;
        inputWindow[currentIndex][1] = aY;
        inputWindow[currentIndex][2] = aZ;


        // Update the index in a circular manner
        currentIndex = (currentIndex + 1) % windowSize;
        Serial.println(currentIndex);
        // Run inference once the buffer is full
        if (currentIndex == 0) {
            // Set model input from the input window
            for (int i = 0; i < windowSize; i++) {
                for (int j = 0; j < features; j++) {
                    modelSetInput(inputWindow[i][j], i * features + j);
                }
            }

            if (!modelRunInference()) {
                Serial.println("RunInference Failed!");
                return;
            }

            // Find the gesture with the highest confidence
            float maxValue = -1.0;
            int maxIndex = -1;

            for (int i = 0; i < 5; i++) {
                float output = modelGetOutput(i);
                Serial.print("Output[");
                Serial.print(i);
                Serial.print("] (");
                Serial.print(gestures[i]);
                Serial.print("): ");
                Serial.println(output, 6);

                if (output > maxValue) {
                    maxValue = output;
                    maxIndex = i;
                }
            }

            // Print the detected gesture
            if (maxIndex >= 0) {
                Serial.print("Detected Gesture: ");
                Serial.println(gestures[maxIndex]);
            } else {
                Serial.println("No gesture detected.");
            }

            Serial.println();

            delay(3000);
        }
    }
}

Also one of the problem is that this is my first time programming in Arduino. Thank you

"For clarification", is it? So you were less than truthful when you said that you wrote it.

I'm done here. Good-bye.

1 Like

Please as you can see from the code I did the most part of the coding. I only used chatgpt when I encountered the error and used their version. However, I saved by main code somewhere for things like this. Please help if you can.

I assume your problem is based on the fact that the sketch samples data even if there is no significant acceleration at all.

If you analyze the library example

https://github.com/spaziochirale/ArduTFLite/blob/main/examples/ArduinoNano33BLE_GestureClassifier/ArduinoNano33BLE_GestureClassifier.ino

you will find a relevant constant declaration and this function at the beginning of loop():

// see constant declaration!
const float accelerationThreshold = 2.5; // Threshold (in G values) to detect a "gesture" start

// ... later in the example ...

  // wait for a significant movement
  while (true) {
    if (IMU.accelerationAvailable()) {
      // read linear acceleration
      IMU.readAcceleration(aX, aY, aZ);

      // compute absolute value of total acceleration
      float aSum = fabs(aX) + fabs(aY) + fabs(aZ);

      // if total absolute acceleration is over the threshold a gesture has started
      if (aSum >= accelerationThreshold) {
        samplesRead = 0; // init samples counter
        break; // exit from waiting cycle
      }
    }
  }

Samples are only taken when the sum of the absolute values of aX, aY and aZ reach or exceed a given threshold. Using fabs() is mandatory as the values can be negative.

Your sketch seems to be a modified version of the example but this essential part is missing ...

In your sketch are several unnecessary (but not hindering) lines:

  • This can be deleted since expectedInputSize and actualInputSize are calculated based on the same constants windowSize and features. They will always be equal (unless you expect a defect controller or compiler):
// const int windowSize = 50;  // Number of timesteps
// const int features = 3;     // 3 features: x, y, z
//  const int expectedInputSize = windowSize * features; 

int actualInputSize = windowSize * features;

            if (expectedInputSize != actualInputSize) {
                Serial.println("Error: Model input size mismatch!");
                Serial.print("Expected: "); Serial.println(expectedInputSize);
                Serial.print("Actual: "); Serial.println(actualInputSize);
                return;
            }
  • This if clause does not make sense:
// const int outputSize = 5; // Expected number of output classes

            for (int i = 0; i < outputSize; i++) {
                // Ensure output index is within range
                if (i >= outputSize) {
                    Serial.println("Error: Output tensor index out of range!");
                    continue;
                }

As outputSize is a constant and i is only controlled by the for-loop the value of i can not become equal or greater than outputSize. The if-clause can be deleted.

If you apply the functionality as in the example the problems should be removed.

Good luck!
ec2021

Thank you. I'll work on it right away give you feedback.

Hello after modifying the sample code to this

#include <Arduino_BMI270_BMM150.h> // IMU Sensor Library for Arduino Nano 33 BLE Rev.2

#include <ArduTFLite.h>

#include "model.h"

const float accelerationThreshold = 2.5; // Threshold (in G values) to detect a "gesture" start
const int numSamples = 50; // Number of samples for a single gesture

int samplesRead; // sample counter 
const int inputLength = 150; // dimension of input tensor (3 values * 50 samples)

constexpr int tensorArenaSize = 8 * 1024;
alignas(16) byte tensorArena[tensorArenaSize];

// a simple table to map gesture labels
const char* GESTURES[] = {
  "Horizontal Shake",
  "Vertical Shake",
  "Letter S",
  "Circle",
  "Letter M"
};

#define NUM_GESTURES (sizeof(GESTURES) / sizeof(GESTURES[0]))

void setup() {
  Serial.begin(9600);
  while (!Serial);

  // init IMU sensor
  if (!IMU.begin()) {
    Serial.println("IMU sensor init failed!");
    while (true); // stop program here.
  }

  // print IMU sampling frequencies
  Serial.print("Accelerometer sampling frequency = ");
  Serial.print(IMU.accelerationSampleRate());
  Serial.println(" Hz");
  Serial.println();
  Serial.println("Init model..");
  if (!modelInit(model, tensorArena, tensorArenaSize)){
    Serial.println("Model initialization failed!");
    while(true);
  }
  Serial.println("Model initialization done.");
}

void loop() {
  float aX, aY, aZ;

  // wait for a significant movement
  while (true) {
    if (IMU.accelerationAvailable()) {
      // read linear acceleration
      IMU.readAcceleration(aX, aY, aZ);

      // compute absolute value of total acceleration
      float aSum = fabs(aX) + fabs(aY) + fabs(aZ);

      // if total absolute acceleration is over the threshold a gesture has started
      if (aSum >= accelerationThreshold) {
        samplesRead = 0; // init samples counter
        break; // exit from waiting cycle
      }
    }
  }

  // reading cycle of all samples for current gesture
  while (samplesRead < numSamples) {
    // check if a sample is available
    if (IMU.accelerationAvailable()) {
      // read acceleration and gyroscope values
      IMU.readAcceleration(aX, aY, aZ);
      // normalize sensor data because model was trained using normalized data
      aX = (aX + 4.0) / 8.0;
      aY = (aY + 4.0) / 8.0;
      aZ = (aZ + 4.0) / 8.0;
      
      // put the 3 values of current sample in the proper position
      // in the input tensor of the model
      modelSetInput(aX,samplesRead * 3 + 0);
      modelSetInput(aY,samplesRead * 3 + 1);
      modelSetInput(aZ,samplesRead * 3 + 2); 
      
      samplesRead++;
      
      // if all samples are got, run inference
      if (samplesRead == numSamples) {
        if(!modelRunInference()){
          Serial.println("RunInference Failed!");
          return;
        }

        // get output values and print as percentage
        for (int i = 0; i < NUM_GESTURES; i++) {
          Serial.print(GESTURES[i]);
          Serial.print(": ");
          Serial.print(modelGetOutput(i)*100, 2);
          Serial.println("%");
        }
        Serial.println();
      }
    }
  }
}

I got the below output
Accelerometer sampling frequency = 99.84 Hz

Init model..

Model initialization done.

Input tensor index out of range!
Letter S: Output tensor index out of range!

-100.00%

Circle: Output tensor index out of range!

-100.00%

Letter M: Output tensor index out of range!

-100.00%

f you look in the library, you will find this

float modelGetOutput(int index) {
    if (tflOutputTensor == nullptr || index >= tflOutputTensor->bytes / sizeof(float)) {
        Serial.println("Output tensor index out of range!");
        return -1;
    }

    return tflOutputTensor->data.f[index];
}

So it may be a problem due to a lack of memory allocated to tensorArenaSize ...

Now you have allocated

constexpr int tensorArenaSize = 8 * 1024;

while you had 8 times more bytes in your post #1 .

constexpr int tensorArenaSize = 64 * 1024; // Increased tensor arena size

However, this is just a guess !!!!

I'm not familiar with ArduTFlite that seems to be wrapped around flite-micro

https://github.com/tuanhe/tflite-micro/tree/master

Please note that I will not have time for an in-depth analysis of this quite extensive library.

You may try to increase the buffer size as mentioned above.

If it does not help you may also check that the model you are using really fits to your application. If you use the model.h as delivered with the example code I assume it has been set up and trained for just two(!) gestures not five ...

Thank you for your time and replies. However, I'm using this sketch for my trained model.h. However, I will increase the buffer size and see if it will work for me.