ERR: Unknown extract function, only MFCC, MFE and spectrogram supported

Project ID: 290646

Hi,
I have a 8x8 thermal sensor which I am using for gesture recognition. I wanted to change run_classifier() to run_classifier_continuous(), and I am receiving this error:

ERR: Unknown extract function, only MFCC, MFE and spectrogram supported

Does that mean the continuous classifier only works with audio? I need to work with raw data for my project.
I just replaced the run_classifier() function and did not change anything else.

Below is my code for the data extraction and classification if it matters.

#include <zephyr/kernel.h>
#include <zephyr/device.h>
#include <zephyr/drivers/sensor.h>
#include "edge-impulse-sdk/classifier/ei_run_classifier.h"
#include "edge-impulse-sdk/dsp/numpy.hpp"
#include <zephyr/sys/printk.h>
#include <stdio.h>

#define FREQUENCY_HZ        EI_CLASSIFIER_FREQUENCY
#define INTERVAL_MS         (1000 / (FREQUENCY_HZ + 1))

static unsigned long last_interval_ms = 0;
size_t feature_ix = 0;

static float features[EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE]={0};

int raw_feature_get_data(size_t offset, size_t length, float *out_ptr) {
memcpy(out_ptr, features + offset, length * sizeof(float));
return 0;
}

static struct sensor_value temp_value[64];
float sensor_array[64]={0};
int maxnow = 0;
int minnow = 25;
int cycle = 1;
int max = 0;
int min=0;
float ftemp=0;
int limlow = 24; //lower cutoff temperature
int body = 33; //max temperature to amplify

#ifdef CONFIG_AMG88XX_TRIGGER
K_SEM_DEFINE(sem, 0, 1);

static void trigger_handler(const struct device *dev,
			    const struct sensor_trigger *trigger)
{
	ARG_UNUSED(dev);
	ARG_UNUSED(trigger);

	k_sem_give(&sem);
}
#endif


void print_buffer(void *ptr, size_t l, float *arr)
{
	struct sensor_value *tv = (sensor_value*)ptr;
	maxnow = 0;
	minnow = 25;
	int temp1 = 0 ;
	int temp2 = 0 ;
	for (int i = 0; i < l; i++) {
temp1 = (tv[i].val1);
temp2 = (tv[i].val2 / 10000);

	// compare values to find maximum and minimum in one iteration
if (temp1 > maxnow && temp1 < body) {
	maxnow = temp1;
}
if (temp1 < minnow) {
	minnow = temp1;
}

	// cutoff values below defined lower limit
/*if (temp1 <= limlow) {
			temp1 = limlow;
			temp2 = 0;
		}
*/


	// set values out of range to 0
if (temp1 > (max + 1) || temp1 < (max - 2)) {
	temp1 = 0;
	temp2 = 0;
}
if (temp1 <= (min + 3)) {
	temp1 = 0;
	temp2 = 0;
}
else {
	//temp1 = (temp1 - (min + 3));
	temp1 = 55;
}

float ft1 = temp1;
float ft2 = temp2;
ftemp = ft1 + (ft2 / 100);

arr[i] = ftemp;

// set max and min temperature of last iteration for the next one
		max = (maxnow);
		min = (minnow);

	}
}

int main(void)
{	
	int ret;
	setvbuf(stdout, NULL, _IONBF, 0);
	const struct device *dev = device_get_binding(
				DT_LABEL(DT_INST(0, panasonic_amg88xx)));

	if (dev == NULL) {
		printk("Could not get AMG88XX device\n");
		return 1;
	}

	printk("device: %p, name: %s\n", dev, dev->name);

#ifdef CONFIG_AMG88XX_TRIGGER
	struct sensor_value attr = {
		.val1 = 27,
		.val2 = 0,
	};

	if (sensor_attr_set(dev, SENSOR_CHAN_AMBIENT_TEMP,
			    SENSOR_ATTR_UPPER_THRESH, &attr)) {
		printk("Could not set threshold\n");
		return;
	}

	struct sensor_trigger trig = {
		.type = SENSOR_TRIG_THRESHOLD,
		.chan = SENSOR_CHAN_AMBIENT_TEMP,
	};

	if (sensor_trigger_set(dev, &trig, trigger_handler)) {
		printk("Could not set trigger\n");
		return;
	}
#endif

	while (42) {
#ifdef CONFIG_AMG88XX_TRIGGER
		printk("Waiting for a threshold event\n");
		k_sem_take(&sem, K_FOREVER);
#endif

		ret = sensor_sample_fetch(dev);
		if (ret) {
			printk("Failed to fetch a sample, %d\n", ret);
			return 1;
		}

		ret = sensor_channel_get(dev, SENSOR_CHAN_AMBIENT_TEMP,
					 (struct sensor_value *)temp_value);
		if (ret) {
			printk("Failed to get sensor values, %d\n", ret);
			return 1;
		}

		/*
		print_buffer(temp_value, ARRAY_SIZE(temp_value), sensor_array);
		for (int i=0; i<64; i++) {
			printf("%f", sensor_array[i]);
			if (i<63) {
				printf(",");
			}
		}
		printf("\n");
		k_sleep(K_MSEC(45));
		*/

		if (sizeof(features) / sizeof(float) != EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE) {
        printk("The size of your 'features' array is not correct. Expected %d items, but had %u\n",
            EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE, sizeof(features) / sizeof(float));
    	}

		ei_impulse_result_t result = { 0 };

		print_buffer(temp_value, ARRAY_SIZE(temp_value), sensor_array);
		k_msleep(83);

		for (int i=0; i<64; i++) {
			features[feature_ix++] = sensor_array[i];
		}

		if (feature_ix == EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE) {

			signal_t signal;
			numpy::signal_from_buffer(features, EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE, &signal);

			 // Run classifier
			EI_IMPULSE_ERROR res = run_classifier_continuous(&signal, &result, false);
			//ei_printf("run_classifier returned: %d\n", res);
            if (res != 0) return 0;
			
			 // Print predictions
            /*ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
                result.timing.dsp, result.timing.classification, result.timing.anomaly);
			*/

			for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
                ei_printf("%s:\t%.5f\n", result.classification[ix].label, result.classification[ix].value);
            }
			// reset features frame
            feature_ix = 0;
		}
	}
	return 0;
}

Hi @rwahidi,

Continuous classifier can work with other types of data other than audio (remember that “data,” including audio data, is just a collection of numbers). However, only MFCC, MFE, and spectrogram processing blocks work with the run_classifier_continuous() function, as that function is optimized to cache previous slices of MFCC, MFE, and spectrogram data across time.

Note that Edge Impulse does not support video (multiple image frames in one sample) at this time. If you can do one frame at a time, then run_classifier() will work for you. If you must use video for gesture recognition, then your best bet is to probably train a model in TensorFlow, PyTorch, etc. and use our BYOM feature to optimize and deploy your model.

1 Like