Wrong Classification On Images Loaded From SD Card

Help
, ,
#1

Question/Issue:
I have created the Arduino Library Zip file using Edge Impulse for my ESP32. Library simply contains the tflite model trained on my local machine with 100% test accuracy in my usecase, later this model was uploaded on Edge Impulse and on testing it was generating perfect results. But after integrating in the script for ESP32 it is not performing well.

Project ID:
537006

Context/Use case:
My use case is to load an image from SD Card, perform the inference and print the results. But the results are incorrect as i got correct results for same images on Edge Impulse dashboard.

Steps Taken:

  1. Prepared the following script
#include <TestProj_inferencing.h>
#include "edge-impulse-sdk/dsp/image/image.hpp"
#include <SD_MMC.h> // For accessing the SD card
#include <JPEGDecoder.h> // For decoding the image from SD card
#define IMAGE_WIDTH 96
#define IMAGE_HEIGHT 96
#define IMAGE_CHANNELS 3 // Assuming the model uses RGB images

static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal
uint8_t *snapshot_buf; // Buffer to hold the image data

// File path to the image on the SD card
const char* imagePath = "/image.jpg"; // Make sure the image is on the SD card with this path

void setup()
{
    // Start serial communication
    Serial.begin(115200);
    while (!Serial);

    Serial.println("Edge Impulse Inferencing with Image from SD Card");

    // Initialize the SD card
    if (!SD_MMC.begin()) {
        Serial.println("SD Card Mount Failed");
        return;
    }

    // Allocate memory for the image data (ensure buffer size matches model input requirements)
    snapshot_buf = (uint8_t*)malloc(IMAGE_WIDTH * IMAGE_HEIGHT * IMAGE_CHANNELS);
    
    if (snapshot_buf == nullptr) {
        Serial.println("Failed to allocate memory for image");
        return;
    }

    ei_printf("\nStarting inference in 2 seconds...\n");
    ei_sleep(2000);
}
void loop()
{
    // Load the image from the SD card
    if (!loadImageFromSD(imagePath)) {
        ei_printf("Failed to load image from SD card\r\n");
        free(snapshot_buf);
        return;
    }

    // Run inference on the loaded image
    ei::signal_t signal;
    signal.total_length = IMAGE_WIDTH * IMAGE_HEIGHT;
    signal.get_data = &ei_camera_get_data;

    ei_impulse_result_t result = { 0 };
    EI_IMPULSE_ERROR err = run_classifier(&signal, &result, debug_nn);
    if (err != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", err);
        return;
    }

    // Print the inference results
    ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
                result.timing.dsp, result.timing.classification, result.timing.anomaly);

#if EI_CLASSIFIER_OBJECT_DETECTION == 1
    ei_printf("Object detection bounding boxes:\r\n");
    for (uint32_t i = 0; i < result.bounding_boxes_count; i++) {
        ei_impulse_result_bounding_box_t bb = result.bounding_boxes[i];
        if (bb.value == 0) {
            continue;
        }
        ei_printf("  %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\r\n",
                bb.label,
                bb.value,
                bb.x,
                bb.y,
                bb.width,
                bb.height);
    }
#else
    ei_printf("Predictions:\r\n");
    for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
        ei_printf("  %s: ", ei_classifier_inferencing_categories[i]);
        ei_printf("%.5f\r\n", result.classification[i].value);
    }
#endif

#if EI_CLASSIFIER_HAS_ANOMALY
    ei_printf("Anomaly prediction: %.3f\r\n", result.anomaly);
#endif

    // Free memory for the snapshot buffer
    free(snapshot_buf);
    while (1);  // Stop after inference is done
}

// Function to load an image from the SD card
bool loadImageFromSD(const char* path)
{
    File imageFile = SD_MMC.open(path);

    if (!imageFile) {
        Serial.println("Failed to open the image file");
        return false;
    }

    // Decode the JPEG image into the buffer
    if (JpegDec.decodeSdFile(imageFile)) {
        // Copy decoded image data into snapshot_buf
        if (JpegDec.width == IMAGE_WIDTH && JpegDec.height == IMAGE_HEIGHT) {
            for (int i = 0; i < JpegDec.width * JpegDec.height * IMAGE_CHANNELS; i++) {
                snapshot_buf[i] = JpegDec.pImage[i];  // Copy pixel data into buffer
            }
            Serial.println("Image loaded successfully");
            return true;
        } else {
            Serial.println("Image dimensions do not match expected model input size");
            return false;
        }
    } else {
        Serial.println("Failed to decode JPEG image");
        return false;
    }
}


// Function to provide image data to the classifier
static int ei_camera_get_data(size_t offset, size_t length, float *out_ptr)
{
    // Process the RGB888 buffer to get the data
    size_t pixel_ix = offset * IMAGE_CHANNELS;  // Calculate pixel index (3 bytes per pixel)
    size_t pixels_left = length;
    size_t out_ptr_ix = 0;

    while (pixels_left != 0) {
        // Normalize RGB values and store into the output pointer (in range 0.0 - 1.0)
        out_ptr[out_ptr_ix] = ((float)snapshot_buf[pixel_ix + 0] / 255.0f);  // Red
        out_ptr[out_ptr_ix + 1] = ((float)snapshot_buf[pixel_ix + 1] / 255.0f);  // Green
        out_ptr[out_ptr_ix + 2] = ((float)snapshot_buf[pixel_ix + 2] / 255.0f);  // Blue
        out_ptr_ix += 3;  // Move to the next pixel
        pixel_ix += 3;
        pixels_left -= 3;
    }

    return 0; // Done
}

[Describe what actually happened]
Wrong Classification may be due to incorrect image loading method but i am totally new to this environment. I’ll appreciate your help asap.

Environment:

  • Platform: ESP32-Cam
  • Build Environment Details: Aurdino IDE
  • OS Version: [e.g., Windows 11]
  • Edge Impulse Version (Firmware): latest
1 Like