Upon deployment, model performs well. After a few more tries, model starts to output only one class

Help
, ,
#1

Question/Issue:
Good day, everyone.

Our system collects sign language gesture data from a doppler radar sensor for 4 seconds, each sample is expected to have 4000 values from the sensor. Data collection on edge impulse looks great. Training and testing accuracy of the system on edge impulse is high. Performing live classification of the system on edge impulse also shows good results. For the impulse, we are processing it with spectrogram.

Our problem is that after deploying the trained model to an esp32, it works fine at the start, classifying the gestures performed on the sensor reliably. However, as we repeatedly try to run inferencing, the model starts to output only one class regardless of the gesture performed.

We have tried to plot the raw values from the sensor during inference and it looks the same as the training data that we have on edge impulse. We noticed that on a fresh upload, meaning the esp32 is wiped of everything before upload, is how we can reproduce this event. We also delete the deployed arduino library on the computer that we use before we upload it again to the esp32.

Has anyone experienced this? Any help would be greatly appreciated!

Thanks!

Project ID:
[673283]

Reproducibility:

  • [ x] Always
  • [ ] Sometimes
  • [ ] Rarely

Logs/Attachments:
image

Im providing the code we use to interface other modules of our system below:

#include <PENTAVALENT-FSL-RECOG_v3_inferencing.h>
#include <ADS1115_WE.h>
#include <Wire.h>
#include <Adafruit_SSD1306.h>
#include <Adafruit_GFX.h>
#include <Adafruit_SH110X.h>
#include <SPI.h>
#include <DFRobot_DF1201S.h>
#include “HardwareSerial.h”
#define RX 16
#define TX 17

HardwareSerial DF1201SSerial(1);
DFRobot_DF1201S DF1201S;

#define FREQUENCY_HZ EI_CLASSIFIER_FREQUENCY
#define INTERVAL_MS (1000 / (FREQUENCY_HZ + 1)) // Adjust this if using micros, 1000 → 1000000

#define SCREEN_WIDTH 128
#define SCREEN_HEIGHT 64
#define OLED_RESET -1
#define OLED_Address 0x3C

Adafruit_SH1106G display = Adafruit_SH1106G(SCREEN_WIDTH, SCREEN_HEIGHT, &Wire, OLED_RESET);

ADS1115_WE adc = ADS1115_WE(&Wire, 0x48);

static unsigned long last_interval_ms = 0;
// to classify 1 frame of data you need EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE values
float features[EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE];
// keep track of where we are in the feature array
size_t feature_ix = 0;

int raw_feature_get_data(size_t offset, size_t length, float* out_ptr) {
memcpy(out_ptr, features + offset, length * sizeof(float));
return 0;
}

const int redLED = 12;
const int grnLED = 14;
const int buttonPin = 13;
int buttonState = 0;
unsigned long lastButtonPress = 0; // For debouncing
const unsigned long debounceDelay = 50; // Debounce delay in milliseconds

typedef void (*DisplayConfigFunc)();

const char* classLabels[] = {
“Good Morning!”,
“How are you?”,
“Idle”,
“Im fine!”,
“Take care!”,
“Nice to meet you!”,
“See you tomorrow!”,
“Thank you!”,
“Today”,
“Understand”,
“You’re welcome!”
};

const int numClasses = 11;

void print_inference_result(ei_impulse_result_t result);

void setup() {
Serial.begin(115200);
Wire.begin();
Wire.setClock(400000);

pinMode(buttonPin, INPUT_PULLUP); //Start Button
pinMode(redLED, OUTPUT);
pinMode(grnLED, OUTPUT);

if (!adc.init()) { //ADS1115
Serial.println(“ADS1115 not connected!”);
}
adc.setVoltageRange_mV(ADS1115_RANGE_0256);
adc.setCompareChannels(ADS1115_COMP_0_GND);
adc.setConvRate(ADS1115_860_SPS);
adc.setMeasureMode(ADS1115_CONTINUOUS);

display.begin(OLED_Address, true); //OLED
display.display();
/*
#if (defined ESP32)
DF1201SSerial.begin(115200, SERIAL_8N1, RX, TX);
#else
DF1201SSerial.begin(115200);
#endif
while (!DF1201S.begin(DF1201SSerial)) {
Serial.println(“Init failed, please check the wire connection!”);
delay(1000);
}
DF1201S.setVol(15);
DF1201S.switchFunction(DF1201S.MUSIC);
DF1201S.setPlayMode(DF1201S.SINGLE);

delay(2000);
display.clearDisplay();
*/
}

void loop() {

static bool waitingForButton = true; // State variable

if (waitingForButton) {
Serial.println(“Press to start signing!”);
waitingForButton = false; // Only print once, until the button is pressed
/*display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 8);
display.setTextWrap(0);
display.setCursor(16, 8);
display.println(“Press to”);
display.setCursor(0, 24);
display.setTextWrap(0);
display.setCursor(34, 24);
display.println(“start”);
display.setCursor(0, 40);
display.setTextWrap(0);
display.setCursor(16, 40);
display.println(“signing!”);
display.display();
*/
digitalWrite(redLED, HIGH);
digitalWrite(grnLED, LOW);
}

buttonState = digitalRead(buttonPin);

if (buttonState == LOW && (millis() - lastButtonPress > debounceDelay)) {
lastButtonPress = millis();

/*display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 16);
display.setTextWrap(0);
display.setCursor(22, 16);
display.println("Signing");
display.setCursor(0, 32);
display.setTextWrap(0);
display.setCursor(-2, 32);
display.println("in progress");
display.display();
*/
delay(3000);
display.clearDisplay();
waitingForButton = true;
digitalWrite(redLED, LOW);
digitalWrite(grnLED, HIGH);


while (feature_ix < EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE) {  // Collect data until the buffer is full
  unsigned long current_time = millis();
  if (current_time - last_interval_ms >= 1) {
    last_interval_ms = current_time;
    features[feature_ix++] = adc.getResult_mV();

    Serial.print(adc.getResult_mV());
    // Serial.print(", ");  // Indicate data collection
  }
}

Serial.println("\nData collection complete. Running inference...");

ei_impulse_result_t result = { 0 };

signal_t features_signal;
features_signal.total_length = sizeof(features) / sizeof(features[0]);
features_signal.get_data = &raw_feature_get_data;

EI_IMPULSE_ERROR res = run_classifier(&features_signal, &result, false);
if (res != EI_IMPULSE_OK) {
  ei_printf("ERR: Failed to run classifier (%d)\n", res);
  return;
}

ei_printf("run_classifier returned: %d\r\n", res);
print_inference_result(result);
feature_ix = 0;

}
}
/*
void GoodMorning() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 16);
display.setTextWrap(0);
display.setCursor(40, 16);
display.println(“Good”);
display.setCursor(0, 32);
display.setTextWrap(0);
display.setCursor(16, 32);
display.println(“Morning!”);
display.display();
}
void HowAreYou() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 16);
display.setTextWrap(0);
display.setCursor(22, 16);
display.println(“How are”);
display.setCursor(0, 32);
display.setTextWrap(0);
display.setCursor(28, 32);
display.println(" you? ");
display.display();
}
void Idle() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 23);
display.setTextWrap(0);
display.setCursor(40, 23);
display.println(“Idle”);
display.display();
}
void ImFine() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 24);
display.setTextWrap(0);
display.setCursor(10, 24);
display.println(“I’m fine!”);
display.display();
}
void TakeCare() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 24);
display.setTextWrap(0);
display.setCursor(4, 24);
display.println(“Take care!”);
display.display();
}
void NicetoMeetYou() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 16);
display.setTextWrap(0);
display.setCursor(22, 16);
display.println(“Nice to”);
display.setCursor(0, 32);
display.setTextWrap(0);
display.setCursor(10, 32);
display.println(“meet you!”);
display.display();
}
void SeeYouTom() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 16);
display.setTextWrap(0);
display.setCursor(22, 16);
display.println(“See you”);
display.setCursor(0, 32);
display.setTextWrap(0);
display.setCursor(10, 32);
display.println(“tomorrow!”);
display.display();
}
void ThankYou() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 24);
display.setTextWrap(0);
display.setCursor(4, 24);
display.println(“Thank you!”);
display.display();
}
void Today() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 24);
display.setTextWrap(0);
display.setCursor(34, 24);
display.println(“Today”);
display.display();
}
void Understand() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 24);
display.setTextWrap(0);
display.setCursor(4, 24);
display.println(“Understand”);
display.display();
}
void YoureWelcome() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(2);
display.setFont(NULL);
display.setCursor(0, 16);
display.setTextWrap(0);
display.setCursor(28, 16);
display.println(“You’re”);
display.setCursor(0, 32);
display.setTextWrap(0);
display.setCursor(16, 32);
display.println(“Welcome!”);
display.display();
}

DisplayConfigFunc ClassConfigs[11] = {
GoodMorning,
HowAreYou,
Idle,
ImFine,
TakeCare,
NicetoMeetYou,
SeeYouTom,
ThankYou,
Today,
Understand,
YoureWelcome
};

void callDisplayConfig(int configIndex) {
if (configIndex >= 0 && configIndex < 11) {
ClassConfigsconfigIndex; // Call the function at the specified index
delay(5000);
display.clearDisplay();
} else {
Serial.println(“Invalid configuration index!”);
}
}
*/
void print_inference_result(ei_impulse_result_t result) {

// Print how long it took to perform inference
ei_printf("Timing: DSP %d ms, inference %d ms, anomaly %d ms\r\n",
          result.timing.dsp,
          result.timing.classification,
          result.timing.anomaly);

// Print the prediction results (object detection)

#if EI_CLASSIFIER_OBJECT_DETECTION == 1
ei_printf(“Object detection bounding boxes:\r\n”);
for (uint32_t i = 0; i < result.bounding_boxes_count; i++) {
ei_impulse_result_bounding_box_t bb = result.bounding_boxes[i];
if (bb.value == 0) {
continue;
}
ei_printf(" %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\r\n",
bb.label,
bb.value,
bb.x,
bb.y,
bb.width,
bb.height);
}

// Print the prediction results (classification)

#else
ei_printf(“Predictions:\r\n”);
for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
ei_printf(" %s: “, ei_classifier_inferencing_categories[i]);
ei_printf(”%.5f\r\n", result.classification[i].value);
}
/*
int maxIndex = 0;
float maxPercentage = result.classification[0].value;
for (int i = 1; i < numClasses; i++) {
if (result.classification[i].value > maxPercentage) {
maxPercentage = result.classification[i].value;
maxIndex = i;
}
}
DF1201S.playFileNum(maxIndex + 1);

callDisplayConfig(maxIndex);

*/
#endif

// Print anomaly result (if it exists)

#if EI_CLASSIFIER_HAS_ANOMALY
ei_printf(“Anomaly prediction: %.3f\r\n”, result.anomaly);
#endif
}

#2

try to balance your dataset. there might be bias thats why it focus on one output. also try to use two processing blocks instead of only using 1