Counting objects using fomo on esp-eye

Help
#1

Question/Issue: I want to count objects in an taken image and I have follows the step in this tutorial: Count objects using FOMO - Edge Impulse Documentation

I am using ESP-EYE with ESP-IDF 4.4, and I have download this source code “GitHub - edgeimpulse/firmware-espressif-esp32: Edge Impulse firmware for the Espressif ESP-EYE(ESP32) Development board” then modify it like the tutorial:

ei_run_camera_impulse.cpp:
/* Edge Impulse ingestion SDK

  • Copyright (c) 2022 EdgeImpulse Inc.
  • Permission is hereby granted, free of charge, to any person obtaining a copy
  • of this software and associated documentation files (the “Software”), to deal
  • in the Software without restriction, including without limitation the rights
  • to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  • copies of the Software, and to permit persons to whom the Software is
  • furnished to do so, subject to the following conditions:
  • The above copyright notice and this permission notice shall be included in
  • all copies or substantial portions of the Software.
  • THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  • IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  • FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  • AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  • LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  • OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  • SOFTWARE.
    */

/* Include ----------------------------------------------------------------- */
#include “model-parameters/model_metadata.h”

#if defined(EI_CLASSIFIER_SENSOR) && EI_CLASSIFIER_SENSOR == EI_CLASSIFIER_SENSOR_CAMERA

#include “edge-impulse-sdk/classifier/ei_run_classifier.h”
#include “edge-impulse-sdk/dsp/image/image.hpp”
#include “ei_camera.h”
#include “firmware-sdk/at_base64_lib.h”
#include “firmware-sdk/jpeg/encode_as_jpg.h”
#include “stdint.h”
#include “ei_device_espressif_esp32.h”
#include “ei_run_impulse.h”

#include “esp_timer.h”

#define DWORD_ALIGN_PTR(a) ((a & 0x3) ?(((uintptr_t)a + 0x4) & ~(uintptr_t)0x3) : a)

typedef enum {
INFERENCE_STOPPED,
INFERENCE_WAITING,
INFERENCE_SAMPLING,
INFERENCE_DATA_READY
} inference_state_t;

static inference_state_t state = INFERENCE_STOPPED;
static uint64_t last_inference_ts = 0;

static bool debug_mode = false;
static bool continuous_mode = false;

static uint8_t *snapshot_buf = nullptr;
static uint32_t snapshot_buf_size;

static ei_device_snapshot_resolutions_t snapshot_resolution;
static ei_device_snapshot_resolutions_t fb_resolution;

static bool resize_required = false;
static uint32_t inference_delay;

// Define the top of the image and the number of columns
static int TOP_Y = 50;
static int NUM_COLS = 5;
static int COL_WIDTH = EI_CLASSIFIER_INPUT_WIDTH / NUM_COLS;
static int MAX_ITEMS = 10;

// Define the factor of the width/height which determines the threshold
// for detection of the object’s movement between frames:
static float DETECT_FACTOR = 1.5;

// Initialize variables
std::vector count(NUM_COLS, 0);
int countsum =0;
int notfoundframes = 0;
std::vector<std::vector<ei_impulse_result_bounding_box_t> > previous_blobs(NUM_COLS);

static int ei_camera_get_data(size_t offset, size_t length, float *out_ptr)
{
// we already have a RGB888 buffer, so recalculate offset into pixel index
size_t pixel_ix = offset * 3;
size_t pixels_left = length;
size_t out_ptr_ix = 0;

while (pixels_left != 0) {
    out_ptr[out_ptr_ix] = (snapshot_buf[pixel_ix] << 16) + (snapshot_buf[pixel_ix + 1] << 8) + snapshot_buf[pixel_ix + 2];

    // go to the next pixel
    out_ptr_ix++;
    pixel_ix+=3;
    pixels_left--;
}

// and done!
return 0;

}

void ei_run_impulse(void)
{
switch(state) {
case INFERENCE_STOPPED:
// nothing to do
return;
case INFERENCE_WAITING:
if(ei_read_timer_ms() < (last_inference_ts + inference_delay)) {
return;
}
state = INFERENCE_DATA_READY;
break;
case INFERENCE_SAMPLING:
case INFERENCE_DATA_READY:
if(continuous_mode == true) {
state = INFERENCE_WAITING;
}
break;
default:
break;
}

uint8_t *jpeg_image;
uint32_t jpeg_image_size = 0;

EiCameraESP32 *camera = static_cast<EiCameraESP32*>(EiCameraESP32::get_camera());

ei_printf("Taking photo...\n");

if(camera->ei_camera_capture_jpeg(&jpeg_image, &jpeg_image_size) == false) {
    ei_printf("ERR: Failed to take a snapshot!\n");
    return;
}

snapshot_buf = (uint8_t*)ei_malloc(snapshot_buf_size);

// check if allocation was successful
if(snapshot_buf == nullptr) {
    ei_printf("ERR: Failed to allocate snapshot buffer!\n");
    return;
}

if(camera->ei_camera_jpeg_to_rgb888(jpeg_image, jpeg_image_size, snapshot_buf) == false) {
    ei_printf("ERR: Failed to decode JPEG image\n");
    ei_free(snapshot_buf);
    ei_free(jpeg_image);
    return;
}

ei_free(jpeg_image);
jpeg_image_size = 0;

int64_t fr_start = esp_timer_get_time();

if (resize_required) {
    ei::image::processing::crop_and_interpolate_rgb888(
        snapshot_buf,
        fb_resolution.width,
        fb_resolution.height,
        snapshot_buf,
        snapshot_resolution.width,
        snapshot_resolution.height);
}
int64_t fr_end = esp_timer_get_time();

if (debug_mode) {
    ei_printf("Time resizing: %d\n", (uint32_t)((fr_end - fr_start)/1000));
}

ei::signal_t signal;
signal.total_length = EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT;
signal.get_data = &ei_camera_get_data;

// print and discard JPEG buffer before inference to free some memory
if (debug_mode) {
    ei_printf("Begin output\n");
    ei_printf("Framebuffer: ");
    // base64_encode((const char*)jpeg_image, jpeg_image_size, &ei_putchar);
    int ret = encode_rgb888_signal_as_jpg_and_output_base64(&signal, EI_CLASSIFIER_INPUT_WIDTH, EI_CLASSIFIER_INPUT_HEIGHT);
    ei_printf("\r\n");
    if(ret != 0) {
        ei_printf("ERR: Failed to encode frame as JPEG (%d)\n", ret);
    }
}

// run the impulse: DSP, neural network and the Anomaly algorithm
ei_impulse_result_t result = { 0 };

EI_IMPULSE_ERROR ei_error = run_classifier(&signal, &result, false);
if (ei_error != EI_IMPULSE_OK) {
    ei_printf("ERR: Failed to run impulse (%d)\n", ei_error);
    ei_free(snapshot_buf);
    return;
}
ei_free(snapshot_buf);

// print the predictions
ei_printf(“Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms., Count: %d ): \n”,
result.timing.dsp, result.timing.classification,result.timing.anomaly, countsum);

#if EI_CLASSIFIER_OBJECT_DETECTION == 1
bool bb_found = result.bounding_boxes[0].value > 0;
std::vector<std::vector<ei_impulse_result_bounding_box_t> > current_blobs(NUM_COLS);
for (size_t ix = 0; ix < result.bounding_boxes_count; ix++) {
auto bb = result.bounding_boxes[ix];
if (bb.value == 0) {
continue;
}
// Check which column the blob is in
int col = int(bb.x / COL_WIDTH);
// Check if blob is within DETECT_FACTOR*h of a blob detected in the previous frame and treat as the same object
for (auto blob : previous_blobs[col]) {
if (abs(int(bb.x - blob.x)) < DETECT_FACTOR * (bb.width + blob.width) && abs(int(bb.y - blob.y)) < DETECT_FACTOR * (bb.height + blob.height)) {
// Check this blob has “moved” across the Y threshold
if (blob.y >= TOP_Y && bb.y < TOP_Y) {
// Increment count for this column if blob has left the top of the image
count[col]++;
countsum++;
}
}
}
// Add current blob to list
current_blobs[col].push_back(bb);
//ei_printf(" %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\n", bb.label, bb.value, bb.x, bb.y, bb.width, bb.height);
}
previous_blobs = std::move(current_blobs);
ei_printf(" Count: %d\n",countsum);
if (bb_found) {
ei_printf(" Count: %d\n",countsum);
notfoundframes = 0;
}
else {
notfoundframes ++;
if (notfoundframes == 1){
ei_printf(" No objects found\n");
}
else {
ei_printf(" Count: %d\n",countsum);
}
}

#else
for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
ei_printf(" %s: %.5f\n", result.classification[ix].label,
result.classification[ix].value);

}

#if EI_CLASSIFIER_HAS_ANOMALY == 1
ei_printf(" anomaly score: %.3f\n", result.anomaly);
#endif
#endif

if (debug_mode) {
    ei_printf("\r\n----------------------------------\r\n");
    ei_printf("End output\r\n");
}

if(continuous_mode == false) {
    ei_printf("Starting inferencing in %d seconds...\n", inference_delay / 1000);
}

}

void ei_start_impulse(bool continuous, bool debug, bool use_max_uart_speed)
{
snapshot_resolution.width = EI_CLASSIFIER_INPUT_WIDTH;
snapshot_resolution.height = EI_CLASSIFIER_INPUT_HEIGHT;

debug_mode = debug;
continuous_mode = continuous;

EiDeviceESP32* dev = static_cast<EiDeviceESP32*>(EiDeviceESP32::get_device());
EiCameraESP32 *camera = static_cast<EiCameraESP32*>(EiCameraESP32::get_camera());

// check if minimum suitable sensor resolution is the same as
// desired snapshot resolution
// if not we need to resize later
fb_resolution = camera->search_resolution(snapshot_resolution.width, snapshot_resolution.height);

if (snapshot_resolution.width != fb_resolution.width || snapshot_resolution.height != fb_resolution.height) {
    resize_required = true;
}

if (!camera->init(snapshot_resolution.width, snapshot_resolution.height)) {
    ei_printf("Failed to init camera, check if camera is connected!\n");
    return;
}

snapshot_buf_size = fb_resolution.width * fb_resolution.height * 3;

// summary of inferencing settings (from model_metadata.h)
ei_printf("Inferencing settings:\n");
ei_printf("\tImage resolution: %dx%d\n", EI_CLASSIFIER_INPUT_WIDTH, EI_CLASSIFIER_INPUT_HEIGHT);
ei_printf("\tFrame size: %d\n", EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE);
ei_printf("\tNo. of classes: %d\n", sizeof(ei_classifier_inferencing_categories) / sizeof(ei_classifier_inferencing_categories[0]));

if(continuous_mode == true) {
    inference_delay = 10000000;
    state = INFERENCE_DATA_READY;
}
else {
    inference_delay = 10000000;  // Đặt khoảng thời gian chụp ảnh và đếm lại ở đây
    state = INFERENCE_WAITING;
    ei_printf("Starting inferencing in %d seconds...\n", inference_delay / 1000);
}

if (use_max_uart_speed) {
    ei_printf("OK\r\n");
    ei_sleep(100);
    dev->set_max_data_output_baudrate();
    ei_sleep(100);
}

while(!ei_user_invoke_stop()) {
    // Thêm vòng lặp để đảm bảo luôn chạy quá trình chụp ảnh và đếm số lượng vật thể
    while (state != INFERENCE_STOPPED) {
        ei_run_impulse();
        ei_printf("Countsum: %d\n", countsum);  // In ra biến countsum sau mỗi lần chạy ei_run_impulse
        ei_sleep(1);
    }

    if (!continuous_mode) {
        state = INFERENCE_WAITING;
        ei_printf("Starting inferencing in %d seconds...\n", inference_delay / 1000);
        ei_sleep(inference_delay / 1000);
    }
}

ei_stop_impulse();

if (use_max_uart_speed) {
    ei_printf("\r\nOK\r\n");
    ei_sleep(100);
    dev->set_default_data_output_baudrate();
    ei_sleep(100);
}

}

void ei_stop_impulse(void)
{
state = INFERENCE_STOPPED;
}

bool is_inference_running(void)
{
return (state != INFERENCE_STOPPED);
}

#endif /* defined(EI_CLASSIFIER_SENSOR) && EI_CLASSIFIER_SENSOR == EI_CLASSIFIER_SENSOR_CAMERA */

main.cpp:
/* Edge Impulse ingestion SDK

  • Copyright (c) 2022 EdgeImpulse Inc.
  • Permission is hereby granted, free of charge, to any person obtaining a copy
  • of this software and associated documentation files (the “Software”), to deal
  • in the Software without restriction, including without limitation the rights
  • to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  • copies of the Software, and to permit persons to whom the Software is
  • furnished to do so, subject to the following conditions:
  • The above copyright notice and this permission notice shall be included in
  • all copies or substantial portions of the Software.
  • THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  • IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  • FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  • AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  • LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  • OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  • SOFTWARE.
    */

/* Include ----------------------------------------------------------------- */
#include “driver/gpio.h”
#include “sdkconfig.h”
#include <stdio.h>
#include “ei_device_espressif_esp32.h”
#include “ei_at_handlers.h”
#include “ei_classifier_porting.h”
#include “ei_run_impulse.h”
#include “freertos/FreeRTOS.h”
#include “freertos/task.h”

EiDeviceInfo *EiDevInfo = dynamic_cast<EiDeviceInfo *>(EiDeviceESP32::get_device());
static ATServer *at;

uint32_t elapsed_time = 0;

// Hàm để kiểm tra xem đã đủ 10 giây chưa
bool is_10_seconds_elapsed() {
if (elapsed_time >= 10000) { // 10 giây = 10000 ms
return true;
}
return false;
}

extern “C” void app_main() {
gpio_pad_select_gpio(GPIO_NUM_21);
gpio_reset_pin(GPIO_NUM_21);

gpio_pad_select_gpio(GPIO_NUM_22);
gpio_reset_pin(GPIO_NUM_22);    

gpio_set_direction(GPIO_NUM_21, GPIO_MODE_OUTPUT);
gpio_set_direction(GPIO_NUM_22, GPIO_MODE_OUTPUT);    

EiDeviceESP32* dev = static_cast<EiDeviceESP32*>(EiDeviceESP32::get_device());

ei_printf(
    "Hello DMM\r\n"
    "Compiled on %s %s\r\n",
    __DATE__,
    __TIME__);

at = ei_at_init(dev);
ei_printf("Type AT+HELP to see a list of commands.\r\n");
at->print_prompt();

dev->set_state(eiStateFinished);

while(1) {
    if (is_10_seconds_elapsed()) {
        break;
    }

    // Start Edge Impulse inference
    ei_start_impulse(false, false, false);

    // Check if inference is running and print count
    while (is_inference_running()) {
        ei_run_impulse();
    }

    elapsed_time += 1000; // Cộng thêm 1 giây (1000 ms)
    
    vTaskDelay(1000 / portTICK_PERIOD_MS); 
}

}

AND THIS IS MY OUTPUT:
Starting inferencing in 2 seconds…
Predictions (DSP: 324 ms., Classification: 4 ms., Anomaly: 0 ms.):

  • helloworld: 0.000000*
  • noise: 0.894531*
  • unknown: 0.105469*
    Starting inferencing in 2 seconds…
    Predictions (DSP: 324 ms., Classification: 4 ms., Anomaly: 0 ms.):
  • helloworld: 0.000000*
  • noise: 0.941406*
  • unknown: 0.058594*
    Starting inferencing in 2 seconds…

There are nothing about the taking image process and there are no result of counting objects system?

I want the system to do: taking image => process image and count the objects => print results only.
Please help me!

1 Like
#2

The code posted does count objects. Did you deploy a FOMO Impulse?