PDM and I2C issues with Edge Impulse

Projects Programming
1

Hello there everyone,

I have a trained model for audio classification, which works woderfully by itself.
And then I have an I2C set-up between the Nicla Sense ME(master) and Arduino Nano 33 Ble Sense(or Seeed Xiao Sense, tried with both, as slave). Which also works alone.

Here is the code for the I2C working between the 2 devices, a very simple i2c communication:
Master:

#include <Arduino.h>
#include "Nicla_System.h"
#include <Wire.h>

void setup() {
  Wire.begin(); // join i2c bus (address optional for master)
  Serial.begin(9600);
}

void loop() {
  while (Serial.available() > 0) {
    char incomingCharacter = Serial.read();
    if (incomingCharacter == '1') {
      Wire.beginTransmission(8); // transmit to device #8
      Wire.write(5);              // sends one byte
      Wire.endTransmission();    // stop transmitting

      delay(100);
    }
  }
}

And here for the slave:

#include <Wire.h>

void setup() {
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  Serial.begin(9600);    // start serial for output

}

void loop() {

}

// function that executes whenever data is received from master
void receiveEvent(int howMany) {
  while (Wire.available()){
   int x = Wire.read();    // receive byte as an integer
   if (x == 5){
    Serial.println("Hello World!");
   }
  }
}

Again, all of this code works.

Then my trained model had the void loop() edited so that it starts the recording and outputs the highest accuracy label and its accuracy, only after I send a 1 in the serial monitor (I am running record+classify, not continous):

while (Serial.available() > 0) {
    char incomingCharacter = Serial.read();
    if (incomingCharacter == '1') {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
        }
      }
    }
  }
}

This code works wonderfully as well. Nothing else, with the exception of the shown void loop has been changed.

Then if I try to move to I2C, I have the master the same, but the slave with 2 different methods, like this:

void setup()
{
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  Serial.begin(115200);    // start serial for output

}

void loop()
{

}

void receiveEvent(int howMany) {
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
          //Wire.write(int(ix));
        }
      }
    }
  }
}

void requestEvent() {
  Wire.write(label); // respond with message of 1 byte
}

And also with the following method:

void setup()
{
  Wire.begin(8);                // join i2c bus with address #8
  Serial.begin(115200);    // start serial for output
}

void loop()
{
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
          Wire.write(int(ix));
        }
      }
    }
  }
}

Neither work, it receives the byte, prints it out, prints that it starts recording and it seems to stop when it needs to start the recording:

Screenshot (1211)
Screenshot (1211)773×1064 9.82 KB

Here is the full code:

#define EIDSP_QUANTIZE_FILTERBANK   0

/* Includes ---------------------------------------------------------------- */
#include <PDM.h>
#include <Wire.h>
#include <a1210_samples_cnn_voice_rec_inferencing.h>

/** Audio buffers, pointers and selectors */
typedef struct {
  int16_t *buffer;
  uint8_t buf_ready;
  uint32_t buf_count;
  uint32_t n_samples;
} inference_t;

static inference_t inference;
static signed short sampleBuffer[2048];
static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal

int label = 0;

/**
   @brief      Arduino setup function
*/
void setup()
{
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  Serial.begin(115200);    // start serial for output

}

/**
   @brief      Arduino main function. Runs the inferencing loop.
*/
void loop()
{
}

void receiveEvent(int howMany) {
  while (Wire.available()) {
    int inByte = Wire.read();    // receive byte as an integer
    Serial.println(inByte);
    if (inByte == 5) {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }


      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
          label = int(ix);
        }
      }
    }
  }
}

void requestEvent() {
  Wire.write(label);
}

/**
   @brief      PDM buffer full callback
               Get data and call audio thread callback
*/
static void pdm_data_ready_inference_callback(void)
{
  int bytesAvailable = PDM.available();

  // read into the sample buffer
  int bytesRead = PDM.read((char *)&sampleBuffer[0], bytesAvailable);

  if (inference.buf_ready == 0) {
    for (int i = 0; i < bytesRead >> 1; i++) {
      inference.buffer[inference.buf_count++] = sampleBuffer[i];

      if (inference.buf_count >= inference.n_samples) {
        inference.buf_count = 0;
        inference.buf_ready = 1;
        break;
      }
    }
  }
}

/**
   @brief      Init inferencing struct and setup/start PDM

   @param[in]  n_samples  The n samples

   @return     { description_of_the_return_value }
*/
static bool microphone_inference_start(uint32_t n_samples)
{
  inference.buffer = (int16_t *)malloc(n_samples * sizeof(int16_t));

  if (inference.buffer == NULL) {
    return false;
  }

  inference.buf_count  = 0;
  inference.n_samples  = n_samples;
  inference.buf_ready  = 0;

  // configure the data receive callback
  PDM.onReceive(&pdm_data_ready_inference_callback);

  PDM.setBufferSize(4096);

  // initialize PDM with:
  // - one channel (mono mode)
  // - a 16 kHz sample rate
  if (!PDM.begin(1, EI_CLASSIFIER_FREQUENCY)) {
    ei_printf("Failed to start PDM!");
    microphone_inference_end();

    return false;
  }

  // set the gain, defaults to 20
  PDM.setGain(127);

  return true;
}

/**
   @brief      Wait on new data

   @return     True when finished
*/
static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;


  while (inference.buf_ready == 0) {
    delay(10);

  }
  return true;
}

/**
   Get raw audio signal data
*/
static int microphone_audio_signal_get_data(size_t offset, size_t length, float *out_ptr)
{
  numpy::int16_to_float(&inference.buffer[offset], out_ptr, length);

  return 0;
}

/**
   @brief      Stop PDM and release buffers
*/
static void microphone_inference_end(void)
{
  PDM.end();
  free(inference.buffer);
}

What makes it to be stuck is the while loop in the following function, more specifically the while loop:

static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;


  while (inference.buf_ready == 0) {
    delay(10);

  }
  return true;
}

I have written on the Edge Impulse forums as well, and I was pointed towards interrupting the I2C communication before the while happens, or using different mbed threads. As well as asking here as it seems to be an I2C issue.

I have tried using the Wire.end() and Wire.begin(8); with different methods (with end before the while loop and begin after). but nothing seems to work.
Meanwhile in regards to multiple Mbed threads, I have 0 experience on how I can do that.

I have tried 2 Arduino Nano 33 BLE Sense, 3 Seeed Xiao Sense, all have the same behaviour.
I also tried adding 4.7k pull-up resistors on the SDA/SCL pins (Although there are internal pull-up resistors already and the code without the voice recognition does work).

I would really appreaciate any help or ideas.

2

inference.buf_ready will never change. while goes forever.

3

This is the original code where interence.buf_ready changes, the function is the same, no changes:

#define EIDSP_QUANTIZE_FILTERBANK   0

/* Includes ---------------------------------------------------------------- */
#include <PDM.h>
#include <a1210_samples_cnn_voice_rec_inferencing.h>

/** Audio buffers, pointers and selectors */
typedef struct {
    int16_t *buffer;
    uint8_t buf_ready;
    uint32_t buf_count;
    uint32_t n_samples;
} inference_t;

static inference_t inference;
static signed short sampleBuffer[2048];
static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal

/**
 * @brief      Arduino setup function
 */
void setup()
{
    // put your setup code here, to run once:
    Serial.begin(115200);

    Serial.println("Edge Impulse Inferencing Demo");

    // summary of inferencing settings (from model_metadata.h)
    ei_printf("Inferencing settings:\n");
    ei_printf("\tInterval: %.2f ms.\n", (float)EI_CLASSIFIER_INTERVAL_MS);
    ei_printf("\tFrame size: %d\n", EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE);
    ei_printf("\tSample length: %d ms.\n", EI_CLASSIFIER_RAW_SAMPLE_COUNT / 16);
    ei_printf("\tNo. of classes: %d\n", sizeof(ei_classifier_inferencing_categories) / sizeof(ei_classifier_inferencing_categories[0]));

    if (microphone_inference_start(EI_CLASSIFIER_RAW_SAMPLE_COUNT) == false) {
        ei_printf("ERR: Failed to setup audio sampling\r\n");
        return;
    }
}

/**
 * @brief      Arduino main function. Runs the inferencing loop.
 */
void loop()
{
    ei_printf("Starting inferencing in 2 seconds...\n");

    delay(2000);

    ei_printf("Recording...\n");

    bool m = microphone_inference_record();
    if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
    }

    ei_printf("Recording done\n");

    signal_t signal;
    signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
    signal.get_data = &microphone_audio_signal_get_data;
    ei_impulse_result_t result = { 0 };

    EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
    if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
    }

    // print the predictions
    ei_printf("Predictions ");
    ei_printf("(DSP: %d ms., Classification: %d ms., Anomaly: %d ms.)",
        result.timing.dsp, result.timing.classification, result.timing.anomaly);
    ei_printf(": \n");
    for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        ei_printf("    %s: %.5f\n", result.classification[ix].label, result.classification[ix].value);
    }
#if EI_CLASSIFIER_HAS_ANOMALY == 1
    ei_printf("    anomaly score: %.3f\n", result.anomaly);
#endif
}

/**
 * @brief      PDM buffer full callback
 *             Get data and call audio thread callback
 */
static void pdm_data_ready_inference_callback(void)
{
    int bytesAvailable = PDM.available();

    // read into the sample buffer
    int bytesRead = PDM.read((char *)&sampleBuffer[0], bytesAvailable);

    if (inference.buf_ready == 0) {
        for(int i = 0; i < bytesRead>>1; i++) {
            inference.buffer[inference.buf_count++] = sampleBuffer[i];

            if(inference.buf_count >= inference.n_samples) {
                inference.buf_count = 0;
                inference.buf_ready = 1;
                break;
            }
        }
    }
}

/**
 * @brief      Init inferencing struct and setup/start PDM
 *
 * @param[in]  n_samples  The n samples
 *
 * @return     { description_of_the_return_value }
 */
static bool microphone_inference_start(uint32_t n_samples)
{
    inference.buffer = (int16_t *)malloc(n_samples * sizeof(int16_t));

    if(inference.buffer == NULL) {
        return false;
    }

    inference.buf_count  = 0;
    inference.n_samples  = n_samples;
    inference.buf_ready  = 0;

    // configure the data receive callback
    PDM.onReceive(&pdm_data_ready_inference_callback);

    PDM.setBufferSize(4096);

    // initialize PDM with:
    // - one channel (mono mode)
    // - a 16 kHz sample rate
    if (!PDM.begin(1, EI_CLASSIFIER_FREQUENCY)) {
        ei_printf("Failed to start PDM!");
        microphone_inference_end();

        return false;
    }

    // set the gain, defaults to 20
    PDM.setGain(127);

    return true;
}

/**
 * @brief      Wait on new data
 *
 * @return     True when finished
 */
static bool microphone_inference_record(void)
{
    inference.buf_ready = 0;
    inference.buf_count = 0;

    while(inference.buf_ready == 0) {
        delay(10);
    }

    return true;
}

/**
 * Get raw audio signal data
 */
static int microphone_audio_signal_get_data(size_t offset, size_t length, float *out_ptr)
{
    numpy::int16_to_float(&inference.buffer[offset], out_ptr, length);

    return 0;
}

/**
 * @brief      Stop PDM and release buffers
 */
static void microphone_inference_end(void)
{
    PDM.end();
    free(inference.buffer);
}

#if !defined(EI_CLASSIFIER_SENSOR) || EI_CLASSIFIER_SENSOR != EI_CLASSIFIER_SENSOR_MICROPHONE
#error "Invalid model for current sensor."
#endif
4

not really

5

Here's the original:
static bool microphone_inference_record(void)

static bool microphone_inference_record(void)
{
    inference.buf_ready = 0;
    inference.buf_count = 0;

    while(inference.buf_ready == 0) {
        delay(10);
    }

    return true;
}

And here is the same one copy-pasted that I use in my code:
static bool microphone_inference_record(void)

static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;


  while (inference.buf_ready == 0) {
    delay(10);

  }
  return true;
}

It's literally the same function.

It seems you have misunderstood my point. The ISSUE in my code that WORKS is caused by the I2C interrups and PDM interrupts that happen together, it is a concurrency issue.

IF I change my code from I2C to UART for example, it works perfectly well, however I must use I2C. The interrupts are caused by the static bool microphone_inference_record(void) function which also uses the static bool microphone_inference_start(uint32_t n_samples) and static void microphone_inference_end(void).

6

it is a concurrency issue. really?

7

If you don't have a solution, and you haven't used the PDM library with Edge Impulse, in order to know how it works, I do not understand why the need to reply.

8

Such things happen in event handlers. Simply set a flag in the handler and do everything in loop() if the flag is set.

9

Hey @DrDiettrich, thanks for your reply, I have tested something similar, since I posted the issue here.
Here is the code (this is not the whole code there are some other functions specific to the voice recording and recognition but are unimportant here):

int label = 0;
/**
   @brief      Arduino setup function
*/
bool startInferencing = false;

void setup()
{

  Serial.begin(115200);    // start serial for output
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
}

void loop()
{
  Serial.println("I am waiting");

  while (startInferencing == false){
    Serial.println("I am blocking the code execution");
  }
  Wire.end();

  Serial.println("Beginning Edge Impulse");

  if (microphone_inference_start(EI_CLASSIFIER_RAW_SAMPLE_COUNT) == false) {
    ei_printf("ERR: Failed to setup audio sampling\r\n");
    return;
  }
  ei_printf("Starting inferencing in 1 second...\n");
  delay(1000);

  ei_printf("Recording...\n");

  bool m = microphone_inference_record();
  if (!m) {
    ei_printf("ERR: Failed to record audio...\n");
    return;
  }

  ei_printf("Recording done\n");

  signal_t signal;
  signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
  signal.get_data = &microphone_audio_signal_get_data;
  ei_impulse_result_t result = { 0 };

  EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
  if (r != EI_IMPULSE_OK) {
    ei_printf("ERR: Failed to run classifier (%d)\n", r);
    return;
  }

  // print the predictions
  ei_printf("Predictions ");
  ei_printf("(DSP: %d ms., Classification: %d ms., Anomaly: %d ms.)",
            result.timing.dsp, result.timing.classification, result.timing.anomaly);
  ei_printf(": \n");
  for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
    ei_printf("    %s: %.5f\n", result.classification[ix].label, result.classification[ix].value);
    if (result.classification[ix].value > 0.5) {
      Serial.println("The label is: " + String(ix));
      label = int(ix);
    }
  }

  microphone_inference_end();


  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  startInferencing = false;
}

void receiveEvent(int howMany) {
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      Serial.println("I am in the if section");
      startInferencing = true;
    }
  }
  Serial.println("I have exited the while");
}

void requestEvent() {
  Serial.println("Info has been requested");
  Wire.write(label); // respond with message of 1 byte
  startInferencing = false;
}

And this is the MASTER code, quite simple:

// Include Arduino Wire library for I2C
#include <Arduino.h>
#include "Nicla_System.h"
#include <Wire.h>

void setup() {
  Wire.begin(); // join i2c bus (address optional for master)
  Serial.begin(9600);
}

void loop() {

  while (Serial.available() > 0) {
    char incomingCharacter = Serial.read();
    if (incomingCharacter == '1') {
      Wire.beginTransmission(8); // transmit to device #8
      Wire.write(5);              // sends one byte
      Wire.endTransmission();    // stop transmitting

      delay(500);
      Wire.requestFrom(8, 1);    // request 1 byte from slave device #8
      while (Wire.available()) { // slave may send less than requested
        int c = Wire.read(); // receive a byte as character

        Serial.print("Label Received : ");
        Serial.println(c);         // print the character
      }
    }
  }
  delay(2000);
}

You can see the "startInferencing" flag that I use. It only works the first time though (only the onReceive function, it doesn't even reach the onRequest). By "it works" I mean, it receives the byte, it sets the flag, it starts the recording, it performs the classification, Serial.prints the classification, then it goes back to the main loop and gets stuck in the while false loop, and stays there, not listening to anything.

I have tried many different ways of checking for info and breaking the while loop, none worked. Any suggestions are appreacited. it seems that if I fix that, everything should work accordingly.

10

Stop using unnecessary loops and delay()s and Serial.print()s.

if (startInferencing) {...}

Do you really want to receive anything any more?

11

The wire.end is necessary, the way the PDM library is used by Edge Impulse, it creates interrupts, and when combined with the I2C interrupts, there is an issue. So the I2C is stopped, to perform the MIC recording, then the wire begins again. I do not need/will not be receiving anything throughout the recording phase.

As for the delays and Serial prints, they were used to check where the code was frozen, many of the Serial.prints come already from the Edge Impulse classifier. The Serial prints will be fully removed once the I2C communication between both devices works accordingly.

I cannot use an if with my code unless I flip the way the flag works. I will be trying that.

12

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.