Here's the build output:
I have made a few changes since the start of the post but the primary issue (no blue leds when the DrinkHandler() State==Drinking) is still present. I noted there's no need for a timeout on the cometHandler(), because the various States have their own timeouts or otherwise exit when a button is pushed, therefore I just pulled that out. Here's the full sketch:
/* Define the number of slices per model window.
For more info: https://docs.edgeimpulse.com/docs/continuous-audio-sampling */
#define EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW 6
#include <PDM.h>
#include <DogDrink_inferencing.h>
#include <ezButton.h>
#include <Adafruit_NeoPixel.h>
const int ButtonTimeout = 500; // Time in millis to detect types of button click
const int FeedTimeout = 5000; // Time in milliseconds that PB will be dispensed
const int DrinkTimeout = 15000; // Time in milliseconds after drinking starts that PB is enabled
const int sampleTimeout = 250; // Interval after which we will check the inference results
int consistentDrinking = 0;
int isDrinking = 0;
int drinkCounter = 0;
int drinkLimit = 4; // How many isDrinking detected before considered consistent.
int drinkStart = 0;
int drinkDwell = 5000; // How long to be listening for more drink signals after the first
int endStop = 0;
int backStop = 0;
int cometStop = 0;
enum {Static, Feeding, Drinking, Opening, Offline};
int State = Static;
enum {Waiting, Counting};
int ButtonState = Waiting;
int drinkState = Waiting;
enum {Recording, Inferencing};
int pdmState = Recording;
enum {Red,Green,Blue};
int color = Red;
int Bright = 128;
enum {show, pause};
int ledState = show;
int Position = 0;
int Direction = 1;
float fadePct = .20;
int cometSize = 3;
int pixelTimeout = 40; // miliseconds between pixel events
unsigned long cometStartTime = 0;
unsigned long pixelStartTime = 0;
unsigned long sampleStart = 0;
enum ButtonType {None, Click, DoubleClick, LongPress};
int ButtonType = None;
int ButtonCount = -1;
int ButtonOutput = None;
char ClickMsg[] = "CLICK";
char LongPressMsg[] = "LONGPRESS";
char DoubleClickMsg[] = "(DOUBLECLICK) ";
char StaticMsg[] = "Nothing is happening.";
char FeedingMsg[] = "Feeding... ";
char OpeningMsg[] = "Opening... ";
char DrinkingMsg[] = "Drinking... ";
char ScoopMsg[] = "scoop button pressed.";
char StopMsg[] = "stopped by endstop.";
char UserStopMsg[] = "stopped by user.";
char DrinkTimerMsg[] = "Drink timer started ... ";
char FeedTimerMsg[] = "Feed timer started ... ";
char FeedTimeoutMsg[] = "Feed timer elapsed.";
char DrinkTimeoutMsg[] = "Drink timer elapsed.";
unsigned int Tones[16]
= {415, 329, 369, 246, 246, 369, 415, 329, 415, 369, 329, 246, 246, 369, 415, 311};
unsigned int Durations[16]
= {700, 700, 700, 700, 700, 700, 700, 700, 700, 700, 700, 700, 800, 900, 1000, 2000};
unsigned int Pauses[16]
= {20, 20, 20, 1400, 20, 20, 20, 1400, 20, 20, 20, 1500, 40, 50, 60, 2100};
int lastStep = 16;
unsigned long ButtonMillis = 0;
unsigned long FeedMillis = 0;
unsigned long DrinkMillis = 0;
unsigned long ToneMillis = 0;
unsigned long sampleMillis = 0;
const int Buzzer = D0;
const int MotEnable = D8;
const int MotForward = D7;
const int MotBack = D6;
const int LedEnable = D10; // Provides power to the LED throught the motor drive
const int LedPin = D1;
#define numLEDs 17 // Number of NeoPixels
Adafruit_NeoPixel strip(numLEDs, LedPin, NEO_GRBW + NEO_KHZ800);
ezButton UserButton (D3);
ezButton ScoopButton (D4);
ezButton StopButton (D2);
/** Audio buffers, pointers and selectors */
typedef struct {
signed short *buffers[2];
unsigned char buf_select;
unsigned char buf_ready;
unsigned int buf_count;
unsigned int n_samples;
} inference_t;
static inference_t inference;
static bool record_ready = false;
static signed short *sampleBuffer;
static bool debug_nn = false; // Set true to see features from the raw signal
static int print_results = -(EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW);
// Arduino setup function
extern "C" {
#include <hal/nrf_pdm.h>
}
/** PDM clock frequency calculation based on 32MHz clock and decimation filter ratio 80
more clock gen info https://infocenter.nordicsemi.com/index.jsp?topic=%2Fps_nrf5340%2Fpdm.html
param sampleRate in Hz, return uint32_t clk value */
static uint32_t pdm_clock_calculate(uint64_t sampleRate)
{ const uint64_t PDM_RATIO = 80ULL;
const uint64_t CLK_32MHZ = 32000000ULL;
uint64_t clk_control = 4096ULL * (((sampleRate * PDM_RATIO) * 1048576ULL) / (CLK_32MHZ + ((sampleRate * PDM_RATIO) / 2ULL)));
return (uint32_t)clk_control; }
void setup() {
UserButton.setDebounceTime(10);
UserButton.setCountMode(COUNT_RISING); // only increments count when released
StopButton.setDebounceTime(10);
ScoopButton.setDebounceTime(10);
pinMode( MotEnable, OUTPUT);
pinMode( MotForward, OUTPUT);
pinMode( MotBack, OUTPUT);
pinMode( Buzzer, OUTPUT);
pinMode( LedEnable, OUTPUT);
pinMode( LedPin, OUTPUT);
strip.begin(); // Initialize NeoPixel strip object (REQUIRED)
strip.show(); // Initialize all pixels to 'off'
strip.setBrightness(Bright); // Set BRIGHTNESS to about 1/5 (max = 255)
// put your setup code here, to run once:
Serial.begin(115200);
// comment out the below line to cancel the wait for USB connection (needed for native USB)
int serialAttempts = 5;
for (int attempt = 0; attempt < serialAttempts && !Serial; ++attempt)
{ delay(30); }
if(Serial) Serial.println("Edge Impulse Inferencing Demo");
// summary of inferencing settings (from model_metadata.h)
if(Serial){
ei_printf("Inferencing settings:\n");
ei_printf("\tInterval: %.2f ms.\n", (float)EI_CLASSIFIER_INTERVAL_MS);
ei_printf("\tFrame size: %d\n", EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE);
ei_printf("\tSample length: %d ms.\n", EI_CLASSIFIER_RAW_SAMPLE_COUNT / 16);
ei_printf("\tNo. of classes: %d\n", sizeof(ei_classifier_inferencing_categories) /
sizeof(ei_classifier_inferencing_categories[0]));
}
run_classifier_init();
if (microphone_inference_start(EI_CLASSIFIER_SLICE_SIZE) == false) {
ei_printf("ERR: Could not allocate audio buffer (size %d), this could be due to the window length of your model\r\n", EI_CLASSIFIER_RAW_SAMPLE_COUNT);
return;
}
}
void loop()
{
UserButton.loop();
StopButton.loop();
ScoopButton.loop();
ButtonHandler();
DrinkHandler();
}
/* PDM buffer full callback, Get data and call audio thread callback */
static void pdm_data_ready_inference_callback(void)
{
int bytesAvailable = PDM.available();
// read into the sample buffer
int bytesRead = PDM.read((char *)&sampleBuffer[0], bytesAvailable);
if (record_ready == true) {
for (int i = 0; i<bytesRead>> 1; i++) {
inference.buffers[inference.buf_select][inference.buf_count++] = sampleBuffer[i];
if (inference.buf_count >= inference.n_samples) {
inference.buf_select ^= 1;
inference.buf_count = 0;
inference.buf_ready = 1;
}
}
}
}
/* Init inferencing struct and setup/start PDM
@param[in] n_samples The n samples
@return { description_of_the_return_value } */
static bool microphone_inference_start(uint32_t n_samples)
{
inference.buffers[0] = (signed short *)malloc(n_samples * sizeof(signed short));
if (inference.buffers[0] == NULL) {
return false;
}
inference.buffers[1] = (signed short *)malloc(n_samples * sizeof(signed short));
if (inference.buffers[1] == NULL) {
free(inference.buffers[0]);
return false;
}
sampleBuffer = (signed short *)malloc((n_samples >> 1) * sizeof(signed short));
if (sampleBuffer == NULL) {
free(inference.buffers[0]);
free(inference.buffers[1]);
return false;
}
inference.buf_select = 0;
inference.buf_count = 0;
inference.n_samples = n_samples;
inference.buf_ready = 0;
// configure the data receive callback
PDM.onReceive(&pdm_data_ready_inference_callback);
PDM.setBufferSize(4096);
// initialize PDM with:
// - one channel (mono mode)
// - a 16 kHz sample rate
if (!PDM.begin(1, 16000)) {
if(Serial) { ei_printf("Failed to start PDM!"); }
microphone_inference_end();
return false;
}
/* Calculate sample rate from sample interval */
uint32_t audio_sampling_frequency = (uint32_t)(1000.f / EI_CLASSIFIER_INTERVAL_MS);
if(audio_sampling_frequency != 16000) {
nrf_pdm_clock_set((nrf_pdm_freq_t)pdm_clock_calculate(audio_sampling_frequency));
}
// set the gain, defaults to 20
PDM.setGain(80);
record_ready = true;
return true;
}
/** Get raw audio signal data */
static int microphone_audio_signal_get_data(size_t offset, size_t length, float *out_ptr)
{
numpy::int16_to_float(&inference.buffers[inference.buf_select ^ 1][offset], out_ptr, length);
return 0;
}
/** @brief Stop PDM and release buffers */
static void microphone_inference_end(void)
{
PDM.end();
free(inference.buffers[0]);
free(inference.buffers[1]);
free(sampleBuffer);
}
#if !defined(EI_CLASSIFIER_SENSOR) || EI_CLASSIFIER_SENSOR != EI_CLASSIFIER_SENSOR_MICROPHONE
#error "Invalid model for current sensor."
#endif
void DrinkHandler(){
switch (State) {
case Offline: break; // Beeping or Playing a melody with delays
case Static: // Listening for drink sounds ...
strip.clear();
digitalWrite(LedEnable, LOW);
if ( ButtonOutput == Click and !endStop){
digitalWrite(MotEnable, HIGH);
FeedMillis = millis();
State = Feeding;
PrintMessage(FeedingMsg,0);
break;
}
if ( ButtonOutput == LongPress and !backStop ){
digitalWrite(MotEnable, HIGH);
State = Opening;
PrintMessage(OpeningMsg,0);
break;
}
if ( consistentDrinking == 1 ) {
PrintMessage(DrinkingMsg,0);
DrinkMillis = millis();
State = Drinking;
consistentDrinking = 0;
break;
}
else {
countDrinks();
}
break;
case Drinking: // a treat is possible
cometHandler(Blue);
if ( ButtonOutput == Click and !endStop ){
digitalWrite(MotEnable, HIGH);
FeedMillis = millis();
State = Feeding;
PrintMessage(FeedingMsg,0);
break;
}
if ( ScoopButton.isPressed() and !endStop ) {
PrintMessage(ScoopMsg,1);
digitalWrite(MotEnable, HIGH);
FeedMillis = millis();
State = Feeding;
PrintMessage(FeedingMsg,0);
break;
}
if ( millis() > DrinkMillis + DrinkTimeout ){
PrintMessage(DrinkTimeoutMsg,1);
DrinkMillis = 0;
State = Static;
PrintMessage(StaticMsg,1);
break;
}
else break;
case Feeding:
backStop = 0;
RunMotor(0);
cometHandler(Green);
if ( ButtonOutput == DoubleClick ){
StopMotor();
PrintMessage(UserStopMsg,1);
PrintMessage(StaticMsg,1);
Beep(1000,2); // Sets State to Offline for beep, then back to Static
break;
}
if ( StopButton.isPressed() ) {
StopMotor();
endStop = 1;
PrintMessage(StopMsg,1);
PrintMessage(StaticMsg,1);
PlaySong(); // Sets State to Offline for song, then back to Static
break;
}
if ( millis() > FeedMillis + FeedTimeout ) {
StopMotor();
FeedMillis = 0;
PrintMessage(FeedTimeoutMsg,1);
State = Static;
PrintMessage(StaticMsg,1);
break;
}
else break;
case Opening:
cometHandler(Red); // Red comets while backing up, enough time to get to end
endStop = 0;
RunMotor(1);
if ( ButtonOutput == DoubleClick ){
StopMotor();
PrintMessage(UserStopMsg,1);
PrintMessage(StaticMsg,1);
Beep(1000,2); // Sets State to Offline for song, then back to Static
break;
}
if ( StopButton.isPressed() ) {
StopMotor();
backStop = 1;
PrintMessage(StopMsg,1);
PrintMessage(StaticMsg,1);
Beep(1000,1); // Sets State to Offline for song, then back to Static
break;
}
break;
}
}
void ButtonHandler(){
switch (ButtonState){
case Waiting: // No keys being pressed
ButtonOutput = None;
if ( UserButton.isPressed() ) {
ButtonMillis = millis();
ButtonState = Counting;
UserButton.resetCount();
ButtonCount = 0;
}
break;
case Counting: // Check how many clicks
ButtonCount = UserButton.getCount();
if ( millis() > ButtonMillis + ButtonTimeout ) {
if (ButtonCount == 0) {ButtonOutput = LongPress; PrintMessage(LongPressMsg,1);}
if (ButtonCount == 1) {ButtonOutput = Click; PrintMessage(ClickMsg,1);}
if (ButtonCount >= 2) {ButtonOutput = DoubleClick; PrintMessage(DoubleClickMsg,0);}
ButtonMillis = 0;
ButtonCount = -1;
ButtonState = Waiting;
}
break;
}
}
void checkDrinking(){
if (inference.buf_ready == 0) {
return;
}
else if (inference.buf_ready == 1) {
signal_t signal;
signal.total_length = EI_CLASSIFIER_SLICE_SIZE;
signal.get_data = µphone_audio_signal_get_data;
ei_impulse_result_t result = {0};
EI_IMPULSE_ERROR r = run_classifier_continuous(&signal, &result, debug_nn);
if (r != EI_IMPULSE_OK) ei_printf("ERR: Failed to run classifier (%d)\n", r);
if (++print_results >= (EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW)) {
if(result.classification[2].value > .7) { // noise signal
digitalWrite(LED_RED,HIGH); // turn off any red
digitalWrite(LED_BLUE,HIGH); // turn off any blue
digitalWrite(LED_GREEN,HIGH); // turn off any green
isDrinking = 0;
}
else if (result.classification[1].value > .7) { // eat signal
digitalWrite(LED_BLUE,HIGH); // turn off any blue
digitalWrite(LED_GREEN,HIGH); // turn off any green
digitalWrite(LED_RED,LOW); // turn on RED
isDrinking = 1;
}
else if (result.classification[0].value > .4) { // drink signal
digitalWrite(LED_RED,HIGH); // turn off any red
digitalWrite(LED_GREEN,HIGH); // turn off any green
digitalWrite(LED_BLUE,LOW); // turn on BLUE
isDrinking = 1;
}
if(Serial){
for (size_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
Serial.print(result.classification[i].label);
Serial.print(":");
Serial.print(result.classification[i].value);
Serial.print(", ");
Serial.print(" ");
}
Serial.print("drinkCounter:");
Serial.println(drinkCounter);
}
print_results =0;
}
inference.buf_ready = 0;
}
}
void countDrinks(){
checkDrinking();
switch (drinkState){
case Waiting: // Not sensing drinking
if ( isDrinking == 1 ) {
drinkStart = millis();
drinkCounter = 0;
drinkState = Counting;
}
break;
case Counting: // Check how many drink detections
if (isDrinking == 1) {
drinkCounter++;
isDrinking = 0;
}
if ( millis() > drinkStart + drinkDwell ) {
if (drinkCounter >= drinkLimit) {
consistentDrinking = 1;
Serial.println("Consistent Drinking detected... ");
}
drinkStart = 0;
drinkCounter = 0;
drinkState = Waiting;
}
break;
}
}
void RunMotor(int Direction){
if ( Direction == 0) { digitalWrite(MotForward, HIGH); }
if ( Direction == 1) { digitalWrite(MotBack, HIGH); }
}
void StopMotor(){
digitalWrite(MotForward, LOW);
digitalWrite(MotBack, LOW);
digitalWrite(MotEnable, LOW);
}
void PrintMessage(char Message[],int LineBreak){
if(Serial) {
if (LineBreak == 1) Serial.println(Message);
if (LineBreak == 0) Serial.print(Message);
}
}
void PlaySong() {
State = Offline;
for ( int i ; i < lastStep; i++ ){
tone(Buzzer,Tones[i],Durations[i]);
delay(Durations[i]+Pauses[i]);
noTone(Buzzer);
}
State = Static;
}
void Beep( int freq, int count){
State = Offline;
for ( int i ; i < count; i++ ){
tone(Buzzer,freq,500);
delay(200);
}
noTone(Buzzer);
State = Static;
}
void cometHandler(int color){
digitalWrite(LedEnable, HIGH); // enable the LED through the driver
switch (ledState){
case show:
for (int i = 0; i < cometSize; i++)
switch (color){
case Red:
strip.setPixelColor( Position+i, Bright, 0, 0 );
break;
case Green:
strip.setPixelColor( Position+i, 0, Bright, 0 );
break;
case Blue:
strip.setPixelColor( Position+i, 0, 0, Bright );
break;
}
strip.show();
pixelStartTime = millis();
Position += Direction;
ledState = pause;
for (int j = 0; j < numLEDs; j++)
if (random(10) > 5) fadePixel(j,color,fadePct);
break;
case pause:
if(millis() > pixelStartTime + pixelTimeout){
if (Position == (numLEDs - cometSize) || Position == 0) Direction *= -1;
ledState = show;
}
break;
}
}
void fadePixel(int pixel, int color, float Amt){
long RGB = strip.getPixelColor(pixel);
int R = (uint8_t)((RGB >> 16) & 0xff);
int G = (uint8_t)((RGB >> 8) & 0xff);
int B = (uint8_t)(RGB & 0xff);
int faded = 0;
switch (color){
case Red:
faded = int(Amt*R);
strip.setPixelColor( pixel, faded, G, B );
break;
case Green:
faded = int(Amt*G);
strip.setPixelColor( pixel, R, faded, B );
break;
case Blue:
faded = int(Amt*B);
strip.setPixelColor( pixel, R, G, faded );
break;
}
}
but will push through it.