Nano 33 BLE sense voice control with sensors

Hardware Nano Family Nano 33 BLE Sense
1

I have a project using the nano 33 BLE sense as a microcontroller to display characters on a screen depending on values from a sensor, but I also want to include voice recognition with picovoice.
I've currently got it working so that if the sensor is a certain value and the voice control is recognised, then an image is displayed on the screen as well as it speaking back; However, I want the screen to constantly be displaying images depending on the sensor values and not just when the voice recognition is activated.

I've tried putting if statements for the sensor in the void loop, but when it runs, the voice recognition won't work, and if the sensor code is in a separate void, the soundboard isn't recognised.

I was wondering if anyone had advice on how to get these working together.

2

Welcome to the forum.

Joining project together is usually a more difficult task because many example projects use bad programming practices like delay and waiting for stuff to happen. That makes them often easier to understand for beginner but harder to combine.

Try modifying your individual project first so that loop runs as often as possible. Do not use delay, do not wait for anything to happen, store a state and move on if things take longer. That way you can extend the code without breaking what you have. Control timing using millis().

If you want some more specific advice. Post your code using code tags. Click </> above before you insert the code. Format your code properly before posting.

3 

#include <Picovoice_EN.h>

#include "pv_porcupine_params.h"
#include "Plant-health_3.h"

#include "Adafruit_Soundboard.h"


#define SFX_RST 2

Adafruit_Soundboard sfx = Adafruit_Soundboard(&Serial1, NULL, SFX_RST);

//LUX SENSOR
#include "Adafruit_VEML7700.h"

Adafruit_VEML7700 veml = Adafruit_VEML7700();

//SCREEN

#include <SPI.h>
#include <Wire.h>
#include <Adafruit_GFX.h>

#include <Adafruit_SSD1306.h>
#include "Facial_expressions.h"

#define SSD1306_128_64
#define SCREEN_WIDTH 128 // OLED display width, in pixels
#define SCREEN_HEIGHT 64 // OLED display height, in pixels

#define OLED_RESET     -1 // Reset pin # (or -1 if sharing Arduino reset pin)
#define SCREEN_ADDRESS 0x3D ///< See datasheet for Address; 0x3D for 128x64, 0x3C for 128x32
Adafruit_SSD1306 display(SCREEN_WIDTH, SCREEN_HEIGHT, &Wire, OLED_RESET);

#define MEMORY_BUFFER_SIZE (70 * 1024)

//OGG files on adafruit soundboard

char feelinggood[] = "GOOD   OGG";
char howareyou[] = "GOODMHRUOGG";
char goodmorning[] = "GOODM   OGG";
char goodnight[] = "GOODN   OGG";
char imsad[] = "SAD     OGG";
char imhappy[] = "HAPPY   OGG";
char toodark[] = "TDARK   OGG";
char toolight[] = "TLIGHT  OGG";
char needwater[] = "WATER   OGG";
char nobitdark[] = "NBDARK  OGG";
char nobitsad[] = "NBSAD   OGG";
char feelhappy[] = "FEELH   OGG";
char notoolight[] = "NTLIGHT OGG";
char yesbitbright[] = "BRIGHT  OGG";
char yesbitdark[] = "YBDARK  OGG";
char niceandlight[] = "NANDL   OGG";

void volUp(void) {
  Serial.println("Vol up...");
  uint16_t v;
  if (! (v = sfx.volUp()) ) {
    Serial.println("Failed to adjust");
  } else {
    Serial.print("Volume: "); Serial.println(v);
  }
}

//FOLLOWING PICOVOICE CODE 
static const char* ACCESS_KEY = "7F2tl+le3iIZcJft7R6OTbwx0YdwtExk/2DJbli7LfjakRs9eOxcEQ==";

//AccessKey string obtained from Picovoice Console (https://picovoice.ai/console/)

static pv_picovoice_t *handle = NULL;

static int8_t memory_buffer[MEMORY_BUFFER_SIZE] __attribute__((aligned(16)));

static const float PORCUPINE_SENSITIVITY = 0.75f;
static const float RHINO_SENSITIVITY = 0.5f;

static void wake_word_callback(void) {
  Serial.println("Wake word detected!");
  
          display.clearDisplay();
          display.drawBitmap(0, 0, happy_face, 128, 64, WHITE);
          display.display();
  sfx.playTrack(goodmorning);
}


static void inference_callback(pv_inference_t *inference) {
  Serial.println("{");
  Serial.print("    is_understood : ");
  Serial.println(inference->is_understood ? "true" : "false");
  if (inference->is_understood) {
    Serial.print("    intent : ");
    Serial.println(inference->intent);
    if (inference->num_slots > 0) {
      Serial.println("    slots : {");
      for (int32_t i = 0; i < inference->num_slots; i++) {
        Serial.print("        ");
        Serial.print(inference->slots[i]);
        Serial.print(" : ");
        Serial.println(inference->values[i]);

    while (veml.readLux() <= 500)                                // Too dark
  {

                  if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "How are you") {
         sfx.playTrack(toodark);
           display.clearDisplay();
          display.drawBitmap(0, 0, sad_face2, 128, 64, WHITE);
          display.display();
        return;
          }
          else {
              display.clearDisplay();
              display.drawBitmap(0, 0, fear_face, 128, 64, WHITE);
              display.display();
              return;
          }
  }
  
        if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "Happy" || String(inference->values[i]) == "Ok" || String(inference->values[i]) == "Healthy ") {
          sfx.playTrack(feelinggood);
          display.clearDisplay();
          display.drawBitmap(0, 0, happy_face, 128, 64, WHITE);
          display.display();
                  return;
        }

        if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "Sad") {
          sfx.playTrack(toodark);
          display.clearDisplay();
          display.drawBitmap(0, 0, sad_face2, 128, 64, WHITE);
          display.display();
                  return;
        }

        if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "Warm" || String(inference->values[i]) == "Hot") {
          sfx.playTrack(toolight);
          display.clearDisplay();
          display.drawBitmap(0, 0, hot_face, 128, 64, WHITE);
          display.display();
                  return;
        }

        if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "Wrong" || String(inference->values[i]) == "Upsetting") {
          sfx.playTrack(toodark);
          display.clearDisplay();
          display.drawBitmap(0, 0, fear_face, 128, 64, WHITE);
          display.display();
                  return;
        }
/*
        if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "How are you" && veml.readLux()> 2500) {
          sfx.playTrack(imsad);
          display.clearDisplay();
          display.drawBitmap(0, 0, sad_face, 128, 64, WHITE);
          display.display();

        }

                if (String(inference->intent) == "Planthealth" && String(inference->values[i]) == "How are you" &&  veml.readLux() <= 2500) {
          sfx.playTrack(toodark);
          display.clearDisplay();
          display.drawBitmap(0, 0, fear_face, 128, 64, WHITE);
          display.display();

        }
*/
        if (String(inference->intent) == "Conversation" && String(inference->values[i]) == "Hello" || String(inference->values[i]) == "Hi" ) {
          sfx.playTrack(goodmorning);
          display.clearDisplay();
          display.drawBitmap(0, 0, happy_face, 128, 64, WHITE);
          display.display();
                  return;

        }

        if (String(inference->intent) == "Conversation" && String(inference->values[i]) == "Good morning") {
          sfx.playTrack(goodmorning);

        }

        if (String(inference->intent) == "Conversation" && String(inference->values[i]) == "Good night") {
          sfx.playTrack(goodnight);

        }
      }
    }

    Serial.println("    }");
  }
  Serial.println("}\n");
  pv_inference_delete(inference);
}


void setup() {

  pinMode(SFX_RST, INPUT);
  
//LUX SENSOR

  while (!Serial) { delay(10); }
  Serial.begin(9600);
  Serial.println("Adafruit VEML7700 Test");

  if (!veml.begin()) {
    Serial.println("Sensor not found");
    while (1);
  }
  Serial.println("Sensor found");

  veml.setGain(VEML7700_GAIN_1_8);
  veml.setIntegrationTime(VEML7700_IT_25MS);

  Serial.print(F("Gain: "));
  switch (veml.getGain()) {
    case VEML7700_GAIN_1: Serial.println("1"); break;
    case VEML7700_GAIN_2: Serial.println("2"); break;
    case VEML7700_GAIN_1_4: Serial.println("1/4"); break;
    case VEML7700_GAIN_1_8: Serial.println("1/8"); break;
  }

  Serial.print(F("Integration Time (ms): "));
  switch (veml.getIntegrationTime()) {
    case VEML7700_IT_25MS: Serial.println("25"); break;
    case VEML7700_IT_50MS: Serial.println("50"); break;
    case VEML7700_IT_100MS: Serial.println("100"); break;
    case VEML7700_IT_200MS: Serial.println("200"); break;
    case VEML7700_IT_400MS: Serial.println("400"); break;
    case VEML7700_IT_800MS: Serial.println("800"); break;
  }
  
  // SCREEN
  Serial.begin(9600);
  Wire.begin();
  delay(100);

  Serial.begin(9600);
  display.begin(SSD1306_SWITCHCAPVCC, 0x3C);


  // Show initial display buffer contents on the screen --
  // the library initializes this with an Adafruit splash screen.
  display.display();
  delay(2000); // Pause for 2 seconds

  // Clear the buffer
  display.clearDisplay();

  // Show the display buffer on the screen. You MUST call display() after
  // drawing commands to make them visible on screen!
  display.display();
  delay(2000);
  
//SOUNDBOARD

  Serial.begin(9600);
  while (!Serial);

  Serial.println("Adafruit Sound Board!");

  // softwareserial at 9600 baud
  // ss.begin(9600);
  Serial1.begin(9600);

  if (!sfx.reset()) {
    Serial.println("Not found");
    while (1);
  }
  Serial.println("SFX board found");

  pv_status_t status = pv_audio_rec_init();
  if (status != PV_STATUS_SUCCESS) {
    Serial.print("Audio init failed with ");
    Serial.println(pv_status_to_string(status));
    while (1);
  }

  status = pv_picovoice_init(
             ACCESS_KEY,
             MEMORY_BUFFER_SIZE,
             memory_buffer,
             sizeof(KEYWORD_ARRAY),
             KEYWORD_ARRAY,
             PORCUPINE_SENSITIVITY,
             wake_word_callback,
             sizeof(CONTEXT_ARRAY),
             CONTEXT_ARRAY,
             RHINO_SENSITIVITY,
             true,
             inference_callback,
             &handle);
  if (status != PV_STATUS_SUCCESS) {
    Serial.print("Picovoice init failed with ");
    Serial.println(pv_status_to_string(status));
    while (1);
  }

  const char *rhino_context = NULL;
  status = pv_picovoice_context_info(handle, &rhino_context);
  if (status != PV_STATUS_SUCCESS) {
    Serial.print("retrieving context info failed with");
    Serial.println(pv_status_to_string(status));
    while (1);
  }
  Serial.println("Wake word: Hey Sol");
  Serial.println(rhino_context);
//  volUp();
//  volUp();
//  volUp();
}

void loop()
{
          
  Serial.print("Lux: "); Serial.println(veml.readLux());
  Serial.print("White: "); Serial.println(veml.readWhite());
  Serial.print("Raw ALS: "); Serial.println(veml.readALS());

  const int16_t *buffer = pv_audio_rec_get_new_buffer();
  if (buffer) {
    const pv_status_t status = pv_picovoice_process(handle, buffer);
    if (status != PV_STATUS_SUCCESS) {
      Serial.print("Picovoice process failed with ");
      Serial.println(pv_status_to_string(status));
     while (1);
    }
  }

  if (veml.readLux()> 500  && veml.readLux() <= 2500)          // shady
  {
  display.clearDisplay();
  display.drawBitmap(0, 0, bored, 128, 64, WHITE);
  display.display();
  }

    if (veml.readLux()> 2500  && veml.readLux() <= 10000)     //Low light
  {
  display.clearDisplay();
  display.drawBitmap(0, 0, sad_face2, 128, 64, WHITE);
  display.display();

  }
    if (veml.readLux()> 10000  && veml.readLux() <= 20000)    //indirect sunlight or morning light 
  {
  display.clearDisplay();
  display.drawBitmap(0, 0,  happy_face, 128, 64, WHITE);
  display.display();
  }
    if (veml.readLux() > 20000)                                                  // direct sunlight
  {
  display.clearDisplay();
  display.drawBitmap(0, 0, hot_face, 128, 64, WHITE);
  display.display();
  }
}
4

The code above demonstrates how when the if statements in the void loop are met, the voice recognition won't work, but having them in the static void means that the desired result isn't shown unless it is told to by the voice control. I haven't used static voids before, but they were in the given example code for the picovoice.

5

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.