Edge Impulse and I2C

Hello there everyone,

I have a trained model for audio classification, which works woderfully by itself.
And then I have an I2C set-up between the Nicla Sense ME(master) and Arduino Nano 33 Ble Sense(or Seeed Xiao Sense, tried with both, as slave). Which also works alone.

Here is the code for the I2C working between the 2 devices, a very simple i2c communication:
Master:

#include <Arduino.h>
#include "Nicla_System.h"
#include <Wire.h>

void setup() {
  Wire.begin(); // join i2c bus (address optional for master)
  Serial.begin(9600);
}

void loop() {
  while (Serial.available() > 0) {
    char incomingCharacter = Serial.read();
    if (incomingCharacter == '1') {
      Wire.beginTransmission(8); // transmit to device #8
      Wire.write(5);              // sends one byte
      Wire.endTransmission();    // stop transmitting

      Wire.requestFrom(8, 1);    // request 1 byte from slave device #8
      while (Wire.available()) { // slave may send less than requested
        int c = Wire.read(); // receive a byte as character

        Serial.print("Slave value : ");
        Serial.println(c);         // print the character
      }

      delay(100);
    }
  }
}

And here for the slave:

#include <Wire.h>

void setup() {
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  Serial.begin(9600);    // start serial for output

}

void loop() {

}

// function that executes whenever data is received from master
void receiveEvent(int howMany) {
  while (Wire.available()){
   int x = Wire.read();    // receive byte as an integer
   if (x == 5){
    Serial.println("Hello World!");
   }
  }
}

// function that executes whenever data is requested by master
void requestEvent() {
  Wire.write(8); // respond with message of 1 byte
}

Again, all of this code works.

Then my trained model had the void loop() edited so that it starts the recording and outputs the highest accuracy label and its accuracy, only after I send a 1 in the serial monitor (I am running record+classify, not continous):

  while (Serial.available() > 0) {
    char incomingCharacter = Serial.read();
    if (incomingCharacter == '1') {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
        }
      }
    }
  }
}

This code works wonderfully as well. Nothing else, with the exception of the shown void loop has been changed.

Then if I try to move to I2C, I have the master the same, but the slave with 2 different methods, like this:

void setup()
{
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  Serial.begin(115200);    // start serial for output

}

void loop()
{

}

void receiveEvent(int howMany) {
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
          //Wire.write(int(ix));
        }
      }
    }
  }
}

void requestEvent() {
  Wire.write(label); // respond with message of 1 byte
}

And also with the following method:

void setup()
{
  Wire.begin(8);                // join i2c bus with address #8
  Serial.begin(115200);    // start serial for output
}

void loop()
{
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      ei_printf("Starting...\n");
      delay(350);
      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions

      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        if (result.classification[ix].value > 0.5) {
          Serial.println("Highest Prediction Classifier is: " + String(result.classification[ix].label) + "\nAccuracy is: " + String(result.classification[ix].value, 2));
          Wire.write(int(ix));
        }
      }
    }
  }
}

Neither work, it receives the byte, prints it out, prints that it starts recording and it seems to stop when it needs to start the recording.

Any help would be appreciated!

Update:
What makes it to be stuck is the while loop in the following function:

static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;

  while (inference.buf_ready == 0) {
    delay(10);
  }

  return true;
}

If I remove the while loop, the device works, however, it doesn’t actually record for 1.6 seconds (my window), it just immediately starts/stops recording and then classifies, of course wrong. Then sends the classification back.

Any clues why this could happen and why I2C messes up with that function?

Hi @Ciprian,

This is more an Arduino related question but the reason is most likely linked to a concurrency issue between the I2C and microphone inference interrupts.

One clean way would be to run I2C and inferencing on different (mbed) threads.

Otherwise, you can probably just call microphone_inference_start (and microphone_inference_end when inferencing is done) after receiving the incomingCharacter.If it doesn’t work, you may need to close the I2C while running inferencing and reopen it after.

Aurelien

@aurel Many thanks for your swift reply. I have added the post here as the PDM and I2C libraries work by themselves, and so I thought the issue may be generated by the implementation of EI’s library. I could be very wrong of course.

I have tried different ways of implementing the closing/opening of the I2C.
I have tried it in the microphone_interference_record like following:
1)

static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;

  Wire.end();
  while (inference.buf_ready == 0) {
    delay(10);
  }

  Wire.begin(8);
  return true;
}

I have also tried the wire.end and begin in the onReceive function, as well as moving everything to the void loop{} function.
Nothing worked.

Then I tried the micrphone_interference_record with an if instead of a while:

static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;

  if (inference.buf_ready == 0) {
    delay(10);
  }

  return true;
}

Now this technically works, but the interference is immediate it doesn’t record for the needed time.
I have also tried different delays for both the if and while implementations.

I have also tried something like this to execute the break:

static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;


  while (inference.buf_ready == 0) {
    delay(10);
    if (inference.buf_ready != 0) {
      break;
    }
  }

  return true;
}

I have also tried removing the: bool m = microphone_inference_record();
And recalling: microphone_inference_start(uint32_t n_samples) & microphone_inference_end(void). And it has the same behavior of immediately stopping the recording, so not running for the full window.
Moreover, I am a bit confused on what should be given as n_samples, I have tried with different values 1, 50, 100, 1000, 10000, 50000. Nothing changes from what I have seen.

As for the “clean approach” with multiple mbed threads, I have 0 knowledge about Mbed and its inner workings, so I got no clue on how to do that.

I have also used 2 different Nano 33 Ble Sense and 3 Xiao Sense, all with the same behaviour.
Even moved to the continous interferencing and it didn’t work.

I have been trying things for the past 7h, I really cannot get it to work. I cannot use UART, so I am thinking of using SPI, although that would complicate things for no reason when I2C is simpler.

Any other ideas in trying to get it to work?

The (PDM) interrupt callback is within the microphone_inference_start function.

I would try something like this, haven’t tested it though:

bool startInferencing = false;

void setup()
{

  Serial.begin(115200);    // start serial for output
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
}

void loop()
{
  
  while (startInferencing == false) {}
  Wire.end();

  if (microphone_inference_start(EI_CLASSIFIER_RAW_SAMPLE_COUNT) == false) {
    ei_printf("ERR: Failed to setup audio sampling\r\n");
    return;
  }
  ei_printf("Starting inferencing in 1 second...\n");
  delay(1000);
  
  ei_printf("Recording...\n");

  bool m = microphone_inference_record();
  if (!m) {
      ei_printf("ERR: Failed to record audio...\n");
      return;
  }

  ei_printf("Recording done\n");

  signal_t signal;
  signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
  signal.get_data = &microphone_audio_signal_get_data;
  ei_impulse_result_t result = { 0 };

  EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
  if (r != EI_IMPULSE_OK) {
      ei_printf("ERR: Failed to run classifier (%d)\n", r);
      return;
  }

  // print the predictions
  ei_printf("Predictions ");
  ei_printf("(DSP: %d ms., Classification: %d ms., Anomaly: %d ms.)",
      result.timing.dsp, result.timing.classification, result.timing.anomaly);
  ei_printf(": \n");
  for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
      ei_printf("    %s: %.5f\n", result.classification[ix].label, result.classification[ix].value);
  }
  
  microphone_inference_end();
  startInferencing = false;
  
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  
}

void receiveEvent(int howMany) {
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      startInferencing = true;
    }
  }
}

void requestEvent() {
  Wire.write(label); // respond with message of 1 byte
}

Hey @aurel thanks for the in-depth solution.
I have tested it, and it seems to only work once (the classification, it doesn’t get to the sending out the byte part). After adding some Serial.prints it seems that the code gets stuck in the while loop and stays there forever. Btw without anything in the while loop {}, it doesn’t work ever, a simple Serial.println, makes it work the first time, then it gets stuck in the loop.

This is the full code that I am using on the SLAVE that can at least work once, before getting stuck in the while loop:

#include <Wire.h>
#include <PDM.h>
#include <a1210_samples_cnn_voice_rec_inferencing.h>

/** Audio buffers, pointers and selectors */
typedef struct {
  int16_t *buffer;
  uint8_t buf_ready;
  uint32_t buf_count;
  uint32_t n_samples;
} inference_t;

static inference_t inference;
static signed short sampleBuffer[2048];
static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal

int label = 0;
/**
   @brief      Arduino setup function
*/
bool startInferencing = false;

void setup()
{

  Serial.begin(115200);    // start serial for output
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
}

void loop()
{
  Serial.println("I am waiting");

  while (startInferencing == false){
    Serial.println("I am blocking the code execution");
  }
  Wire.end();

  Serial.println("Beginning Edge Impulse");

  if (microphone_inference_start(EI_CLASSIFIER_RAW_SAMPLE_COUNT) == false) {
    ei_printf("ERR: Failed to setup audio sampling\r\n");
    return;
  }
  ei_printf("Starting inferencing in 1 second...\n");
  delay(1000);

  ei_printf("Recording...\n");

  bool m = microphone_inference_record();
  if (!m) {
    ei_printf("ERR: Failed to record audio...\n");
    return;
  }

  ei_printf("Recording done\n");

  signal_t signal;
  signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
  signal.get_data = &microphone_audio_signal_get_data;
  ei_impulse_result_t result = { 0 };

  EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
  if (r != EI_IMPULSE_OK) {
    ei_printf("ERR: Failed to run classifier (%d)\n", r);
    return;
  }

  // print the predictions
  ei_printf("Predictions ");
  ei_printf("(DSP: %d ms., Classification: %d ms., Anomaly: %d ms.)",
            result.timing.dsp, result.timing.classification, result.timing.anomaly);
  ei_printf(": \n");
  for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
    ei_printf("    %s: %.5f\n", result.classification[ix].label, result.classification[ix].value);
    if (result.classification[ix].value > 0.5) {
      Serial.println("The label is: " + String(ix));
      label = int(ix);
    }
  }

  microphone_inference_end();


  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
  startInferencing = false;
}

void receiveEvent(int howMany) {
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      Serial.println("I am in the if section");
      startInferencing = true;
    }
  }
  Serial.println("I have exited the while");
}

void requestEvent() {
  Serial.println("Info has been requested");
  Wire.write(label); // respond with message of 1 byte
  startInferencing = false;
}

/**
   @brief      PDM buffer full callback
               Get data and call audio thread callback
*/
static void pdm_data_ready_inference_callback(void)
{
  int bytesAvailable = PDM.available();

  // read into the sample buffer
  int bytesRead = PDM.read((char *)&sampleBuffer[0], bytesAvailable);

  if (inference.buf_ready == 0) {
    for (int i = 0; i < bytesRead >> 1; i++) {
      inference.buffer[inference.buf_count++] = sampleBuffer[i];

      if (inference.buf_count >= inference.n_samples) {
        inference.buf_count = 0;
        inference.buf_ready = 1;
        break;
      }
    }
  }
}

/**
   @brief      Init inferencing struct and setup/start PDM

   @param[in]  n_samples  The n samples

   @return     { description_of_the_return_value }
*/
static bool microphone_inference_start(uint32_t n_samples)
{
  inference.buffer = (int16_t *)malloc(n_samples * sizeof(int16_t));

  if (inference.buffer == NULL) {
    return false;
  }

  inference.buf_count  = 0;
  inference.n_samples  = n_samples;
  inference.buf_ready  = 0;

  // configure the data receive callback
  PDM.onReceive(&pdm_data_ready_inference_callback);

  PDM.setBufferSize(4096);

  // initialize PDM with:
  // - one channel (mono mode)
  // - a 16 kHz sample rate
  if (!PDM.begin(1, EI_CLASSIFIER_FREQUENCY)) {
    ei_printf("Failed to start PDM!");
    microphone_inference_end();

    return false;
  }

  // set the gain, defaults to 20
  PDM.setGain(127);

  return true;
}

/**
   @brief      Wait on new data

   @return     True when finished
*/
static bool microphone_inference_record(void)
{
  inference.buf_ready = 0;
  inference.buf_count = 0;

  while (inference.buf_ready == 0) {
    delay(10);
  }

  return true;
}

/**
   Get raw audio signal data
*/
static int microphone_audio_signal_get_data(size_t offset, size_t length, float *out_ptr)
{
  numpy::int16_to_float(&inference.buffer[offset], out_ptr, length);

  return 0;
}

/**
   @brief      Stop PDM and release buffers
*/
static void microphone_inference_end(void)
{
  PDM.end();
  free(inference.buffer);
}

#if !defined(EI_CLASSIFIER_SENSOR) || EI_CLASSIFIER_SENSOR != EI_CLASSIFIER_SENSOR_MICROPHONE
#error "Invalid model for current sensor."
#endif

And this is the MASTER code:

// Include Arduino Wire library for I2C
#include <Arduino.h>
#include "Nicla_System.h"
#include <Wire.h>

void setup() {
  Wire.begin(); // join i2c bus (address optional for master)
  Serial.begin(9600);
}

void loop() {

  while (Serial.available() > 0) {
    char incomingCharacter = Serial.read();
    if (incomingCharacter == '1') {
      Wire.beginTransmission(8); // transmit to device #8
      Wire.write(5);              // sends one byte
      Wire.endTransmission();    // stop transmitting

      delay(500);
      Wire.requestFrom(8, 1);    // request 1 byte from slave device #8
      while (Wire.available()) { // slave may send less than requested
        int c = Wire.read(); // receive a byte as character

        Serial.print("Label Received : ");
        Serial.println(c);         // print the character
      }
    }
  }
  delay(2000);
}

If I change:

while (startInferencing == false) {}

To:

while (startInferencing == false);

It reacts to both the onreceive and onrequest signals, but it doesn’t return back to the void loop().

I have tried the following changes with the original while (startInferencing == false) {}, to have it check if the byte has arrived at break:

while (startInferencing == false) {
    Serial.println("Code is being stopped");
    receiveEvent(1);
  }
while (startInferencing == false) {
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
}

Interestingly, if the main loop code gets moved to the if condition for onReceive, it bricks the arduino and it doesn’t get detected by the USB anymore. I bricked 2 devices for testing purposes. I believe I can just reflash the firmware and fix this.
This code bricks it:

void setup()
{

  Serial.begin(115200);    // start serial for output
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
}

void loop()
{
  Serial.println("I am waiting");

  while (startInferencing == false)
  {
    receiveEvent(1);
  }


}

void receiveEvent(int howMany) {
  Wire.begin(8);                // join i2c bus with address #8
  while (Wire.available()) {
    int x = Wire.read();    // receive byte as an integer
    Serial.println(x);
    if (x == 5) {
      Serial.println("I am in the if section");
      Wire.end();

      Serial.println("Beginning Edge Impulse");

      if (microphone_inference_start(EI_CLASSIFIER_RAW_SAMPLE_COUNT) == false) {
        ei_printf("ERR: Failed to setup audio sampling\r\n");
        return;
      }
      ei_printf("Starting inferencing in 1 second...\n");
      delay(1000);

      ei_printf("Recording...\n");

      bool m = microphone_inference_record();
      if (!m) {
        ei_printf("ERR: Failed to record audio...\n");
        return;
      }

      ei_printf("Recording done\n");

      signal_t signal;
      signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
      signal.get_data = &microphone_audio_signal_get_data;
      ei_impulse_result_t result = { 0 };

      EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
      if (r != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", r);
        return;
      }

      // print the predictions
      ei_printf("Predictions ");
      ei_printf("(DSP: %d ms., Classification: %d ms., Anomaly: %d ms.)",
                result.timing.dsp, result.timing.classification, result.timing.anomaly);
      ei_printf(": \n");
      for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
        ei_printf("    %s: %.5f\n", result.classification[ix].label, result.classification[ix].value);
        if (result.classification[ix].value > 0.5) {
          Serial.println("The label is: " + String(ix));
          label = int(ix);
        }
      }

      microphone_inference_end();


      Wire.begin(8);                // join i2c bus with address #8
      Wire.onRequest(requestEvent); // register event
      Wire.onReceive(receiveEvent); // register event
      startInferencing = false;
    }
  }
  Serial.println("I have exited the while");
}

void requestEvent() {
  Serial.println("Info has been requested");
  Wire.write(label); // respond with message of 1 byte
  startInferencing = false;
}

Ignoring the bricking code, and going back to the semi-working one. It seems like now the only thing needed to fix is the while loop freezing the whole code, Any suggestions?

I have been playing a bit more with it and the following code works:

#include <Wire.h>
#include <PDM.h>
#include <a1210_samples_cnn_voice_rec_inferencing.h>

/** Audio buffers, pointers and selectors */
typedef struct {
  int16_t *buffer;
  uint8_t buf_ready;
  uint32_t buf_count;
  uint32_t n_samples;
} inference_t;

static inference_t inference;
static signed short sampleBuffer[2048];
static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal

int label = 0;
/**
   @brief      Arduino setup function
*/
bool startInferencing = false;

void setup()
{

  Serial.begin(115200);    // start serial for output
  Wire.begin(8);                // join i2c bus with address #8
  Wire.onRequest(requestEvent); // register event
  Wire.onReceive(receiveEvent); // register event
}

void loop()
{
  if (startInferencing == true) {
    label = 10;
        
    if (microphone_inference_start(EI_CLASSIFIER_RAW_SAMPLE_COUNT) == false) {
      label = 11;
      startInferencing = false;
      return;
    }
    ei_printf("Starting...\n");
    delay(350);
    ei_printf("Recording...\n");

    bool m = microphone_inference_record();
    if (!m) {
      label = 12;
      startInferencing = false;
      return;
    }

    ei_printf("Recording done\n");

    signal_t signal;
    signal.total_length = EI_CLASSIFIER_RAW_SAMPLE_COUNT;
    signal.get_data = &microphone_audio_signal_get_data;
    ei_impulse_result_t result = { 0 };

    EI_IMPULSE_ERROR r = run_classifier(&signal, &result, debug_nn);
    if (r != EI_IMPULSE_OK) {
      label = 13;
      startInferencing = false;
      return;
    }

    // print the predictions
    for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
      if (result.classification[ix].value > 0.5) {
        Serial.println("The label is: " + String(ix));
        label = ix;
      }
    }

    microphone_inference_end();
    startInferencing = false;
  }
  delay(100);
}

void receiveEvent(int howMany) {
  startInferencing = true;
}

void requestEvent() {
  Serial.println("Info has been requested");
  Wire.write(label); // respond with message of 1 byte
}

One important thing is to have the Request from the master happening after 3 seconds (the recording window is 1650ms + 30ms classification + for loop time to go through all the labels and choose the one that is higher than 50% thanks to the softmax function).

I have set it up to work as follows:

  • Label 0-9 = 10 voice commands => dependant on the command spoken
  • Label 10 = no classification higher than 50% certainty (can happen with low voice volume or high environment noise) => send label 10 so the user is informed to repeat the command
  • Label 11-13 = software/hardware issues

Many many thanks @aurel for the support provided! You guys here at EI rock, I’ve never got such fast support anywhere else. Have a nice one!

1 Like