863 lines
29 KiB
C
863 lines
29 KiB
C
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <assert.h>
|
|
#include <unistd.h>
|
|
#include "xmedia_cl.h"
|
|
#include "xmedia_mmz.h"
|
|
#include "xmedia_sys.h"
|
|
#include "xmedia_tde.h"
|
|
#include "xmedia_video_common.h"
|
|
#include "xmedia_svp.h"
|
|
#include "xmedia_vgs.h"
|
|
#include "xmedia_vb.h"
|
|
|
|
#include <sys/time.h>
|
|
#include <math.h>
|
|
#include <stdint.h>
|
|
|
|
#define ALIGN_BYTE 8
|
|
#define DEFAULT_INOUT_NUM 16
|
|
#define DFL_LEN 16
|
|
#define FEATURE_NUM 3
|
|
#define MAX_TARGET_NUM 30
|
|
#define AI_EPSILON 0.000001f
|
|
#define PHY_NUM 5
|
|
#define MAX_FILE_LEN 256
|
|
|
|
/* 模型相关信息 */
|
|
#define IMAGE_WIDTH 640
|
|
#define IMAGE_HEIGTH 640
|
|
#define MODEL_FILE_NAME "./model/neuron_network.xmm"
|
|
#define DETECT_THRE 0.7
|
|
#define DETECT_IOU_THRE 0.5
|
|
#define INPUT_IMAGE_FILE_NAME "./input/test_image1_640x640_NV21.yuv"
|
|
|
|
#define ALIGN_FUNC(A, ALIGN) \
|
|
((((A) % (ALIGN)) == 0) ? (A) : ((A) + (ALIGN) - ((A) % (ALIGN))))
|
|
#define STD_MAX(a, b) ((a) > (b) ? (a) : (b))
|
|
#define STD_MIN(a, b) ((a) > (b) ? (b) : (a))
|
|
|
|
#define CFREE(a) \
|
|
if (a) { \
|
|
free(a); \
|
|
a = NULL; \
|
|
}
|
|
|
|
static xmedia_cl_s32 feature_map_size[FEATURE_NUM] = {80, 40, 20};
|
|
static xmedia_cl_s32 stride_down[FEATURE_NUM] = {8, 16, 32};
|
|
static xmedia_video_rect target_rect[MAX_TARGET_NUM];
|
|
|
|
typedef struct {
|
|
xmedia_float x1;
|
|
xmedia_float y1;
|
|
xmedia_float x2;
|
|
xmedia_float y2;
|
|
} xmedia_ai_rect;
|
|
|
|
typedef struct {
|
|
xmedia_cl_s32 detect_type;
|
|
xmedia_cl_s32 class_type;
|
|
xmedia_float score;
|
|
xmedia_float classfier_score;
|
|
xmedia_float tracker_iou;
|
|
xmedia_ai_rect rect;
|
|
} xmedia_ai_detect_result;
|
|
|
|
static xmedia_cl_s32 sample_abs(int num)
|
|
{
|
|
return num < 0 ? -num : num;
|
|
}
|
|
|
|
inline static xmedia_s32 sample_clamp(xmedia_float val, int min, int max)
|
|
{
|
|
return val > min ? (val < max ? val : max) : min;
|
|
}
|
|
|
|
inline static xmedia_float sample_sigmoid(xmedia_float x) { return 1.0 / (1.0 + expf(-x)); }
|
|
|
|
inline static xmedia_float sample_unsigmoid(xmedia_float y) { return -1.0 * logf((1.0 / y) - 1.0); }
|
|
|
|
inline static xmedia_s32 sample_clip(xmedia_float val, xmedia_float min, xmedia_float max)
|
|
{
|
|
xmedia_float f = val <= min ? min : (val >= max ? max : val);
|
|
return f;
|
|
}
|
|
|
|
inline static xmedia_float sample_deqnt_affine_u8_to_f32(xmedia_u8 qnt, xmedia_s32 zp, xmedia_float scale)
|
|
{
|
|
return ((xmedia_float)qnt - (xmedia_float)zp) * scale;
|
|
}
|
|
|
|
static xmedia_u8 sample_qnt_f32_to_affine_u8(xmedia_float f32, xmedia_s32 zp, xmedia_float scale)
|
|
{
|
|
xmedia_float dst_val = (f32 / scale) + zp;
|
|
xmedia_u8 res = (xmedia_u8)sample_clip(dst_val, 0, 255);
|
|
return res;
|
|
}
|
|
|
|
static xmedia_cl_s32 sample_malloc_inout_tensor_mem(xmedia_cl_tensor_info_inout *inout)
|
|
{
|
|
inout->tensor = malloc(sizeof(xmedia_cl_tensor) * inout->num);
|
|
inout->current_batch = malloc(sizeof(xmedia_cl_u32) * inout->num);
|
|
inout->tensor_batch = malloc(sizeof(xmedia_cl_tensor_batch) * inout->num);
|
|
|
|
if (inout->tensor == NULL || inout->current_batch == NULL
|
|
|| inout->tensor_batch == NULL) {
|
|
return XMEDIA_CL_OUT_OF_HOST_MEMORY;
|
|
}
|
|
|
|
return XMEDIA_SUCCESS;
|
|
}
|
|
|
|
static xmedia_void sample_free_inout_tensor_mem(xmedia_cl_tensor_info_inout inout)
|
|
{
|
|
if (inout.tensor != NULL) {
|
|
free(inout.tensor);
|
|
}
|
|
if (inout.current_batch != NULL) {
|
|
free(inout.current_batch);
|
|
}
|
|
if (inout.tensor_batch != NULL) {
|
|
free(inout.tensor_batch);
|
|
}
|
|
}
|
|
|
|
static xmedia_void sample_compute_dfl(xmedia_float* tensor, xmedia_float* box)
|
|
{
|
|
for (xmedia_s32 b=0; b<4; b++){
|
|
xmedia_float exp_t[DFL_LEN];
|
|
xmedia_float exp_sum=0;
|
|
xmedia_float acc_sum=0;
|
|
for (xmedia_s32 i=0; i< DFL_LEN; i++){
|
|
exp_t[i] = exp(tensor[i+b*DFL_LEN]);
|
|
exp_sum += exp_t[i];
|
|
}
|
|
|
|
for (xmedia_s32 i=0; i< DFL_LEN; i++){
|
|
acc_sum += exp_t[i]/exp_sum *i;
|
|
}
|
|
box[b] = acc_sum;
|
|
}
|
|
}
|
|
|
|
static inline xmedia_s32 cmp_score_fall(const xmedia_void *lsh, const xmedia_void *rsh)
|
|
{
|
|
// 降序排列, 令score最大的box在队列末端
|
|
xmedia_ai_detect_result *s1 = (xmedia_ai_detect_result *)lsh;
|
|
xmedia_ai_detect_result *s2 = (xmedia_ai_detect_result *)rsh;
|
|
return s1->score > s2->score ? -1 : 1;
|
|
}
|
|
|
|
static inline xmedia_float sample_iou
|
|
(xmedia_ai_detect_result *box1, xmedia_ai_detect_result *box2, xmedia_bool total_ratio)
|
|
{
|
|
xmedia_float area1;
|
|
xmedia_float area2;
|
|
xmedia_float mix_area;
|
|
|
|
if (box1->class_type != box2->class_type) {
|
|
return 0.0f;
|
|
}
|
|
|
|
mix_area = STD_MAX(0.0f, STD_MIN(box1->rect.x2, box2->rect.x2) -
|
|
STD_MAX(box1->rect.x1, box2->rect.x1)) *
|
|
STD_MAX(0.0f, STD_MIN(box1->rect.y2, box2->rect.y2) -
|
|
STD_MAX(box1->rect.y1, box2->rect.y1));
|
|
if (mix_area == 0.0f) {
|
|
return 0.0f;
|
|
}
|
|
area1 = (box1->rect.x2 - box1->rect.x1) * (box1->rect.y2 - box1->rect.y1);
|
|
area2 = (box2->rect.x2 - box2->rect.x1) * (box2->rect.y2 - box2->rect.y1);
|
|
|
|
if (total_ratio) {
|
|
xmedia_float tmp = area1 + area2 - mix_area;
|
|
return (tmp == 0.0f) ? 0.0f : (mix_area / tmp);
|
|
} else {
|
|
return (area1 == 0.0f) ? 0.0f : (mix_area / area1);
|
|
}
|
|
}
|
|
|
|
static xmedia_s32 sample_det_nms(xmedia_ai_detect_result *vec_bbox,
|
|
xmedia_u32 vec_bbox_len, xmedia_float threshold, xmedia_u32 *picked_bbox_len)
|
|
{
|
|
xmedia_u32 i;
|
|
xmedia_u32 j;
|
|
xmedia_ai_detect_result *box_b;
|
|
xmedia_ai_detect_result *box_a;
|
|
xmedia_ai_detect_result *picked_bbox;
|
|
xmedia_u32 valid_count = 0;
|
|
|
|
qsort(vec_bbox, vec_bbox_len, sizeof(xmedia_ai_detect_result), cmp_score_fall);
|
|
for (i = 0; i < vec_bbox_len; i++) {
|
|
box_a = &vec_bbox[i];
|
|
if (box_a->score == 0.0) {
|
|
continue;
|
|
}
|
|
for (j = i + 1; j < vec_bbox_len; j++) {
|
|
box_b = &vec_bbox[j];
|
|
if (box_b->score == 0.0) {
|
|
continue;
|
|
}
|
|
xmedia_float box_iou = sample_iou(box_a, box_b, XMEDIA_TRUE);
|
|
if (box_iou >= threshold) {
|
|
box_b->score = 0.0;
|
|
}
|
|
}
|
|
valid_count++;
|
|
}
|
|
|
|
picked_bbox = (xmedia_ai_detect_result *)malloc(valid_count * sizeof(xmedia_ai_detect_result));
|
|
if (picked_bbox == XMEDIA_NULL) {
|
|
*picked_bbox_len = 0;
|
|
return XMEDIA_NULL;
|
|
}
|
|
for (i = 0, j = 0; i < vec_bbox_len; i++) {
|
|
if (vec_bbox[i].score == 0.0) {
|
|
continue;
|
|
}
|
|
if (j < valid_count) {
|
|
picked_bbox[j++] = vec_bbox[i];
|
|
}
|
|
}
|
|
*picked_bbox_len = valid_count;
|
|
|
|
memcpy(vec_bbox, picked_bbox, sizeof(xmedia_ai_detect_result) * valid_count);
|
|
|
|
CFREE(picked_bbox);
|
|
return XMEDIA_SUCCESS;
|
|
}
|
|
|
|
static xmedia_s32 sample_svp_draw(xmedia_video_rect target_rect[],
|
|
xmedia_video_frame_info *frame,
|
|
xmedia_svp_yolov5_output result)
|
|
{
|
|
xmedia_s32 ret;
|
|
xmedia_vgs_frame_info task = {0};
|
|
xmedia_vgs_cover_attr cover[XMEDIA_SVP_MAX_TARGET_NUM];
|
|
xmedia_s32 handle = -1;
|
|
xmedia_s32 i;
|
|
xmedia_u8 object_num = result.target_num;
|
|
if (0 == object_num)
|
|
{
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
|
|
object_num = object_num > XMEDIA_SVP_MAX_TARGET_NUM ? XMEDIA_SVP_MAX_TARGET_NUM : object_num;
|
|
memcpy(&task.img_in,frame,sizeof(xmedia_video_frame_info));
|
|
memcpy(&task.img_out,frame,sizeof(xmedia_video_frame_info));
|
|
ret = xmedia_vgs_init();
|
|
if(ret != XMEDIA_SUCCESS) {
|
|
printf("xmedia_vgs_init failed !\n");
|
|
return ret;
|
|
}
|
|
ret = xmedia_vgs_create_job(&handle);
|
|
if (XMEDIA_SUCCESS != ret)
|
|
{
|
|
xmedia_vgs_exit();
|
|
printf("xmedia_vgs_cancel_job failed !\n");
|
|
handle = -1;
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
for (i = 0; i < object_num; i++) {
|
|
if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_FIREWORKS_SMOKE ||
|
|
result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_TRICYCLER) {
|
|
cover[i].color = 0x0000FF;
|
|
} else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_BIKER) {
|
|
cover[i].color = 0x000FFF;
|
|
} else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_MOTOR) {
|
|
cover[i].color = 0x0F000F;
|
|
} else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_MOTORER) {
|
|
cover[i].color = 0xFF00FF;
|
|
} else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_TRICYCLE) {
|
|
cover[i].color = 0x00FF00;
|
|
} else {
|
|
cover[i].color = 0xFF0000;
|
|
}
|
|
cover[i].cover_type = XMEDIA_VGS_COVER_TYPE_QUAD_RANGLE;
|
|
cover[i].quadrangle.is_solid = XMEDIA_FALSE;
|
|
cover[i].quadrangle.thick = 2;
|
|
cover[i].quadrangle.points[0].x = target_rect[i].x;
|
|
cover[i].quadrangle.points[0].y = target_rect[i].y;
|
|
cover[i].quadrangle.points[1].x = target_rect[i].x + target_rect[i].width;
|
|
cover[i].quadrangle.points[1].y = target_rect[i].y;
|
|
cover[i].quadrangle.points[2].x = target_rect[i].x + target_rect[i].width;
|
|
cover[i].quadrangle.points[2].y = target_rect[i].y + target_rect[i].height;
|
|
cover[i].quadrangle.points[3].x = target_rect[i].x;
|
|
cover[i].quadrangle.points[3].y = target_rect[i].y + target_rect[i].height;
|
|
}
|
|
ret = xmedia_vgs_add_task_cover(handle, &task, &cover[0], object_num);
|
|
if (XMEDIA_SUCCESS != ret) {
|
|
xmedia_vgs_cancel_job(handle);
|
|
xmedia_vgs_exit();
|
|
printf("xmedia_vgs_add_task_cover failed ! ret [%#x] \n", ret);
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
ret = xmedia_vgs_submit_job(handle);
|
|
if (XMEDIA_SUCCESS != ret) {
|
|
xmedia_vgs_cancel_job(handle);
|
|
xmedia_vgs_exit();
|
|
printf("xmedia_vgs_submit_job failed ! ret [%d] \n", ret);
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
ret = xmedia_vgs_wait_job(handle, 2000);
|
|
if (ret != XMEDIA_SUCCESS) {
|
|
xmedia_vgs_exit();
|
|
printf("xmedia_vgs_wait_job failed ! ret [%d] \n", ret);
|
|
return ret;
|
|
}
|
|
|
|
return XMEDIA_SUCCESS;
|
|
}
|
|
|
|
static xmedia_s32 sample_mmzalloc_cached(xmedia_u64 *pu64PhyAddr,
|
|
xmedia_void **ppVirAddr,
|
|
const xmedia_char *strMmb,
|
|
const xmedia_char *strZone,
|
|
xmedia_u32 u32Len)
|
|
{
|
|
*pu64PhyAddr = xmedia_mmz_alloc(strZone, strMmb, u32Len);
|
|
if (*pu64PhyAddr == XMEDIA_NULL) {
|
|
return XMEDIA_CL_OUT_OF_HOST_MEMORY;
|
|
}
|
|
|
|
*ppVirAddr = xmedia_mmz_map(*pu64PhyAddr, u32Len, 1);
|
|
if (*ppVirAddr == XMEDIA_NULL) {
|
|
return XMEDIA_CL_OUT_OF_HOST_MEMORY;
|
|
}
|
|
|
|
return XMEDIA_SUCCESS;
|
|
}
|
|
|
|
static xmedia_s32 sample_mmzfree(xmedia_u64 u64PhyAddr, xmedia_void *pVirAddr)
|
|
{
|
|
xmedia_cl_s32 ret;
|
|
|
|
if (pVirAddr != XMEDIA_NULL) {
|
|
ret = xmedia_mmz_unmap(pVirAddr);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
}
|
|
|
|
return xmedia_mmz_free(u64PhyAddr);
|
|
}
|
|
|
|
static xmedia_s32 sample_get_input_one_data(xmedia_void* data_yuv, xmedia_u32 image_size)
|
|
{
|
|
FILE* fp_temp = fopen(INPUT_IMAGE_FILE_NAME, "rb");
|
|
if (fp_temp)
|
|
{
|
|
fread(data_yuv, image_size, 1, fp_temp);
|
|
}
|
|
else {
|
|
printf("fopen failed.\n");
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
fclose(fp_temp);
|
|
fp_temp = XMEDIA_NULL;
|
|
|
|
return XMEDIA_SUCCESS;
|
|
}
|
|
|
|
static xmedia_s32 sample_tde_y2r(xmedia_cl_tensor_info_inout input,
|
|
xmedia_u64 img_src_phy, xmedia_u64 img_src_rgb_phy)
|
|
{
|
|
xmedia_s32 handle = XMEDIA_FAILURE;
|
|
|
|
xmedia_s32 ret;
|
|
xmedia_tde_surface_info src = {0};
|
|
xmedia_tde_surface_info dst = {0};
|
|
|
|
ret = xmedia_tde_create_job(&handle);
|
|
if (XMEDIA_SUCCESS != ret)
|
|
{
|
|
printf("xmedia_tde_create_job failed %#x!\n",ret);
|
|
handle = XMEDIA_FAILURE;
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
|
|
src.surface.phys_addr[0] = img_src_phy;
|
|
src.surface.phys_addr[1] = img_src_phy + input.tensor[0].shape.dims[2] * input.tensor[0].shape.dims[1];
|
|
src.surface.stride[0] = input.tensor[0].shape.dims[2];
|
|
src.surface.stride[1] = input.tensor[0].shape.dims[2];
|
|
src.surface.width = input.tensor[0].shape.dims[2];
|
|
src.surface.height = input.tensor[0].shape.dims[1];
|
|
src.surface.pixel_format = XMEDIA_VIDEO_PIXEL_FMT_YVU_SEMIPLANAR_420;
|
|
src.surface.alpha0 = 0;
|
|
src.surface.alpha1 = 255;
|
|
src.surface.is_alpha_ext_1555 = 0;
|
|
src.surface.clut_reload = XMEDIA_FALSE;
|
|
|
|
src.rect.x = 0;
|
|
src.rect.y = 0;
|
|
src.rect.width = input.tensor[0].shape.dims[2];
|
|
src.rect.height = input.tensor[0].shape.dims[1];
|
|
|
|
dst.surface.phys_addr[0] = img_src_rgb_phy;
|
|
dst.surface.width = input.tensor[0].shape.dims[2];
|
|
dst.surface.height = input.tensor[0].shape.dims[1];
|
|
dst.surface.stride[0] = input.tensor[0].shape.dims[2] * 3;
|
|
dst.surface.pixel_format = XMEDIA_VIDEO_PIXEL_FMT_RGB_888;
|
|
dst.surface.is_alpha_ext_1555 = XMEDIA_FALSE;
|
|
dst.surface.alpha0 = 0;
|
|
dst.surface.alpha1 = 255;
|
|
dst.surface.clut_reload = XMEDIA_FALSE;
|
|
|
|
dst.rect.x = 0;
|
|
dst.rect.y = 0;
|
|
dst.rect.width = input.tensor[0].shape.dims[2];
|
|
dst.rect.height = input.tensor[0].shape.dims[1];
|
|
|
|
ret = xmedia_tde_add_task_single_blit(handle, &src, &dst, 0, XMEDIA_NULL);
|
|
if (XMEDIA_SUCCESS != ret)
|
|
{
|
|
printf("xmedia_tde_add_task_scale failed ret: %#x !\n", ret);
|
|
xmedia_tde_cancel_job(handle);
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
|
|
ret = xmedia_tde_submit_job(handle);
|
|
if (XMEDIA_SUCCESS != ret)
|
|
{
|
|
printf("xmedia_tde_submit_job failed ,ret: %#x!\n", ret);
|
|
xmedia_tde_cancel_job(handle);
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
|
|
ret = xmedia_tde_wait_job(handle, 2000);
|
|
if (XMEDIA_SUCCESS != ret)
|
|
{
|
|
printf("tde overtime %#x!\n",ret);
|
|
return XMEDIA_FAILURE;
|
|
}
|
|
|
|
return XMEDIA_SUCCESS;
|
|
}
|
|
|
|
int main()
|
|
{
|
|
void *buff[DEFAULT_INOUT_NUM] = { 0 };
|
|
xmedia_cl_s8 name[MAX_FILE_LEN] = { 0 };
|
|
xmedia_cl_u64 u64SrcPhyAddr[PHY_NUM] = { 0 };
|
|
xmedia_char *nocache_buffer_start[PHY_NUM] = { 0 };
|
|
xmedia_cl_s32 output_size[DEFAULT_INOUT_NUM] = { 0 };
|
|
xmedia_cl_s8 *model = MODEL_FILE_NAME;
|
|
xmedia_cl_s32 i = 0, j, size = 0, ret, err;
|
|
xmedia_cl_graph graph = XMEDIA_NULL;
|
|
xmedia_cl_u32 num_devices = 0, input_num = 0, output_num = 0;
|
|
xmedia_cl_tensor_info_inout input = { 0 }, output = { 0 };
|
|
xmedia_cl_s32 err_code = 0;
|
|
xmedia_cl_context context = XMEDIA_NULL;
|
|
xmedia_cl_device_id *devices = XMEDIA_NULL;
|
|
xmedia_cl_u32 worksize, weightsize, inputsize = 0, outputsize = 0;
|
|
xmedia_u32 image_size;
|
|
xmedia_s32 image_width, image_heigth;
|
|
|
|
// 公共模块初始化
|
|
ret = xmedia_sys_init(XMEDIA_NULL);
|
|
if (ret != XMEDIA_SUCCESS) {
|
|
printf("xmedia_sys_init err, errno %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = xmedia_tde_init();
|
|
if (XMEDIA_SUCCESS != ret)
|
|
{
|
|
printf("xmedia_tde_init failed %#x!\n",ret);
|
|
goto SYS_EXIT;
|
|
}
|
|
// 运行时初始化
|
|
ret = xmedia_cl_init();
|
|
if (ret != XMEDIA_SUCCESS) {
|
|
printf("xmedia_cl_init err, errno %d\n", ret);
|
|
goto TDE_EXIT;
|
|
}
|
|
// 第一次调用该接口获取设备数量
|
|
ret = xmedia_cl_get_device_ids(XMEDIA_CL_DEVICE_ALL, NULL, &num_devices);
|
|
if (ret != XMEDIA_SUCCESS) {
|
|
printf("xmedia_cl_get_device_ids err, errno %d\n", ret);
|
|
goto CL_EXIT;
|
|
}
|
|
devices = (xmedia_cl_device_id *)calloc(num_devices, sizeof(xmedia_cl_device_id));
|
|
if (devices == NULL) {
|
|
printf("calloc err\n");
|
|
ret = XMEDIA_CL_OUT_OF_HOST_MEMORY;
|
|
goto CL_EXIT;
|
|
}
|
|
// 第二次调用该接口传递获取到的设备数量申请设备资源
|
|
ret = xmedia_cl_get_device_ids(XMEDIA_CL_DEVICE_ALL, devices, &num_devices);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_get_device_ids err, errno %d\n", ret);
|
|
free(devices);
|
|
goto CL_EXIT;
|
|
}
|
|
// 创建资源管理的对象
|
|
context = xmedia_cl_create_context(num_devices, devices, &err_code);
|
|
if (err_code != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_create_context err, errno %d\n", ret);
|
|
goto DEVICE_EXIT;
|
|
}
|
|
|
|
// 从模型文件中查询work和weight的大小
|
|
ret = xmedia_cl_graph_querysize_from_file(model, &worksize, &weightsize);
|
|
if (XMEDIA_CL_SUCCESS != ret) {
|
|
printf("xmedia_cl_graph_querysize_from_file, errno %d\n", ret);
|
|
goto CONTEXT_EXIT;
|
|
}
|
|
if (worksize) {
|
|
// 根据查询到的大小申请work和weight的内存
|
|
ret = sample_mmzalloc_cached(&u64SrcPhyAddr[0], (void **)(&nocache_buffer_start[0]),
|
|
"npu_workspace", NULL, worksize);
|
|
if (XMEDIA_CL_SUCCESS != ret) {
|
|
goto CONTEXT_EXIT;
|
|
}
|
|
} else {
|
|
nocache_buffer_start[0] = NULL;
|
|
}
|
|
if (weightsize) {
|
|
ret = sample_mmzalloc_cached(&u64SrcPhyAddr[1], (void **)(&nocache_buffer_start[1]),
|
|
"npu_weight", NULL, weightsize);
|
|
if (XMEDIA_CL_SUCCESS != ret) {
|
|
goto ERROR;
|
|
}
|
|
} else {
|
|
nocache_buffer_start[1] = NULL;
|
|
}
|
|
// 加载解析模型文件，若是从文件中加载，则model为文件路径，若从内存中加载，则model为内存地址
|
|
ret = xmedia_cl_graph_loadmodel_from_file_withmem(
|
|
&context, model, nocache_buffer_start[0], worksize,
|
|
nocache_buffer_start[1], weightsize, &graph);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_loadmodel err, errno %d\n", ret);
|
|
goto ERROR;
|
|
}
|
|
|
|
// 第一次调用该接口获取输入数量
|
|
ret = xmedia_cl_graph_get_input(graph, input_num, &input);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_get_input, errno %d\n", ret);
|
|
goto ERROR;
|
|
}
|
|
if (input.num > DEFAULT_INOUT_NUM) {
|
|
printf("model input num is greater than default value!\n");
|
|
goto ERROR;
|
|
}
|
|
// 根据输入数量申请内存
|
|
sample_malloc_inout_tensor_mem(&input);
|
|
input_num = input.num;
|
|
// 第二次调用该接口获取输入信息
|
|
ret = xmedia_cl_graph_get_input(graph, input_num, &input);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_get_input, errno %d\n", ret);
|
|
goto ERROR;
|
|
}
|
|
|
|
// 第一次调用该接口获取输出数量
|
|
ret = xmedia_cl_graph_get_output(graph, output_num, &output);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_get_output, errno %d\n", ret);
|
|
goto ERROR;
|
|
}
|
|
if (output.num > DEFAULT_INOUT_NUM) {
|
|
printf("model output num is greater than default value!\n");
|
|
goto ERROR;
|
|
}
|
|
// 根据输出数量申请内存
|
|
sample_malloc_inout_tensor_mem(&output);
|
|
output_num = output.num;
|
|
|
|
// 第二次调用该接口获取输出信息
|
|
ret = xmedia_cl_graph_get_output(graph, output_num, &output);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_get_output, errno %d\n", ret);
|
|
goto ERROR;
|
|
}
|
|
|
|
// 计算输入的大小并申请内存
|
|
for (i = 0; i < input.num; i++) {
|
|
size = input.tensor[i].size;
|
|
inputsize += ALIGN_FUNC(size, ALIGN_BYTE);
|
|
}
|
|
|
|
ret = sample_mmzalloc_cached(&u64SrcPhyAddr[2], (void **)(&nocache_buffer_start[2]),
|
|
"npu_input", NULL, inputsize);
|
|
if (XMEDIA_CL_SUCCESS != ret) {
|
|
goto ERROR;
|
|
}
|
|
|
|
// 计算输出的大小并申请内存
|
|
for (i = 0; i < output.num; i++) {
|
|
output_size[i] = output.tensor[i].size;
|
|
outputsize += ALIGN_FUNC(output_size[i], ALIGN_BYTE);
|
|
}
|
|
ret = sample_mmzalloc_cached(&u64SrcPhyAddr[3], (void **)(&nocache_buffer_start[3]),
|
|
"npu_output", NULL, outputsize);
|
|
if (XMEDIA_CL_SUCCESS != ret) {
|
|
goto ERROR;
|
|
}
|
|
|
|
image_width = IMAGE_WIDTH;
|
|
image_heigth = IMAGE_HEIGTH;
|
|
image_size = image_width * image_heigth * 3 / 2;
|
|
|
|
// 申请yuv图像内存
|
|
sample_mmzalloc_cached(&u64SrcPhyAddr[4], (void **)(&nocache_buffer_start[4]),
|
|
"test_yuv_image", NULL, image_size);
|
|
|
|
// 读取yuv图像
|
|
ret = sample_get_input_one_data(nocache_buffer_start[4], image_size);
|
|
if (ret != XMEDIA_SUCCESS) {
|
|
printf("get input data failed!\n");
|
|
}
|
|
|
|
sample_tde_y2r(input, u64SrcPhyAddr[4], u64SrcPhyAddr[2]);
|
|
|
|
// 设置输入，把输入golden数据拷贝到输入地址,每个输入必须设置一次
|
|
for (i = 0; i < input.num; i++) {
|
|
if (i > 0) {
|
|
input.tensor[i].addr =
|
|
input.tensor[i - 1].addr + ALIGN_FUNC(size, ALIGN_BYTE);
|
|
} else {
|
|
input.tensor[i].addr = nocache_buffer_start[2];
|
|
}
|
|
size = input.tensor[i].size;
|
|
|
|
printf("input.tensor[%d].addr = %p input size = 0x%x\n", i,
|
|
input.tensor[i].addr, size);
|
|
}
|
|
|
|
// 把输出的golden数据拷贝到内存，用来与npu的输出做对比
|
|
for (i = 0; i < output.num; i++) {
|
|
if (i > 0) {
|
|
output.tensor[i].addr = output.tensor[i - 1].addr +
|
|
ALIGN_FUNC(output_size[i - 1], ALIGN_BYTE);
|
|
} else {
|
|
output.tensor[i].addr = nocache_buffer_start[3];
|
|
}
|
|
memset(name, 0, sizeof(name));
|
|
output_size[i] = output.tensor[i].size;
|
|
|
|
printf("output.tensor[%d].addr = %p output size = %d\n", i,
|
|
output.tensor[i].addr, output_size[i]);
|
|
}
|
|
|
|
// 在xmm2下，用户需要申请输入地址，输出地址可以动态改变，把输入与输出设置为推理的输入、输出
|
|
ret = xmedia_cl_graph_set_inout(graph, &input, &output);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_set_inout, errno %d\n", ret);
|
|
goto ERROR;
|
|
}
|
|
|
|
struct timeval t_start, t_end;
|
|
gettimeofday(&t_start, NULL);
|
|
ret = xmedia_cl_graph_process(graph);
|
|
gettimeofday(&t_end, NULL);
|
|
printf("xmedia_cl_graph_process start time = [%ld]\n", t_start.tv_usec);
|
|
printf("xmedia_cl_graph_process end time = [%ld]\n", t_end.tv_usec);
|
|
printf("xmedia_cl_graph_process timeval = [%ld] us\n",
|
|
((t_end.tv_sec - t_start.tv_sec) * 1000000) + t_end.tv_usec - t_start.tv_usec);
|
|
if (ret != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_process err, errno %d\n", ret);
|
|
}
|
|
|
|
// 解析输出tensor
|
|
xmedia_cl_s32 feature_map_num = FEATURE_NUM;
|
|
xmedia_cl_s32 index;
|
|
xmedia_cl_u32 idx = 0;
|
|
|
|
xmedia_cl_float detect_threshold = DETECT_THRE;
|
|
xmedia_cl_u32 target_num = MAX_TARGET_NUM;
|
|
xmedia_ai_detect_result *bbox;
|
|
bbox = (xmedia_ai_detect_result *)calloc(1, target_num * sizeof(xmedia_ai_detect_result));
|
|
|
|
for (index = 0; index < feature_map_num; index++) {
|
|
xmedia_cl_s32 grid_h = feature_map_size[index];
|
|
xmedia_cl_s32 grid_w = feature_map_size[index];
|
|
xmedia_cl_s32 stride = stride_down[index];
|
|
xmedia_cl_s32 box_index = index * 2;
|
|
xmedia_cl_s32 score_index = index * 2 + 1;
|
|
|
|
xmedia_cl_s32 box_zp = output.tensor[box_index].quant.zp;
|
|
xmedia_cl_float box_scale = output.tensor[box_index].quant.scale;
|
|
xmedia_cl_u8* box_tensor = output.tensor[box_index].addr;
|
|
|
|
xmedia_cl_s32 score_zp = output.tensor[score_index].quant.zp;
|
|
xmedia_cl_float score_scale = output.tensor[score_index].quant.scale;
|
|
xmedia_cl_u8* score_tensor = output.tensor[score_index].addr;
|
|
|
|
xmedia_cl_s32 grid_len = grid_w * grid_h;
|
|
xmedia_cl_u8 score_thres_u8 =
|
|
sample_qnt_f32_to_affine_u8(sample_unsigmoid(detect_threshold), score_zp, score_scale);
|
|
|
|
for (i = 0; i < grid_h; i++) {
|
|
for (j = 0; j < grid_w; j++) {
|
|
xmedia_cl_s32 score_offset = i * grid_w + j;
|
|
xmedia_cl_s32 max_class_id = -1;
|
|
xmedia_cl_s32 bbox_offset = i * grid_w + j;
|
|
xmedia_cl_u8 max_score = -score_zp;
|
|
xmedia_cl_float box[4];
|
|
xmedia_cl_float before_dfl[DFL_LEN * 4];
|
|
xmedia_cl_float x1, x2, y1, y2;
|
|
|
|
if (idx >= target_num) {
|
|
break;
|
|
}
|
|
|
|
for (xmedia_cl_s32 c = 0; c < 80; c++) {
|
|
if ((score_tensor[score_offset] > score_thres_u8) &&
|
|
(score_tensor[score_offset] > max_score)) {
|
|
max_score = score_tensor[score_offset];
|
|
max_class_id = c;
|
|
}
|
|
score_offset += grid_len;
|
|
}
|
|
|
|
if (max_score < score_thres_u8) {
|
|
continue;
|
|
}
|
|
|
|
for (xmedia_cl_s32 k = 0; k < DFL_LEN * 4; k++) {
|
|
before_dfl[k] = sample_deqnt_affine_u8_to_f32(box_tensor[bbox_offset], box_zp, box_scale);
|
|
bbox_offset += grid_len;
|
|
}
|
|
sample_compute_dfl(before_dfl, box);
|
|
|
|
x1 = (-box[0] + j + 0.5) * stride;
|
|
y1 = (-box[1] + i + 0.5) * stride;
|
|
x2 = (box[2] + j + 0.5) * stride;
|
|
y2 = (box[3] + i + 0.5) * stride;
|
|
|
|
bbox[idx].rect.x1 = sample_clamp(x1, 0, image_width);
|
|
bbox[idx].rect.y1 = sample_clamp(y1, 0, image_heigth);
|
|
bbox[idx].rect.x2 = sample_clamp(x2, 0, image_width);
|
|
bbox[idx].rect.y2 = sample_clamp(y2, 0, image_heigth);
|
|
|
|
bbox[idx].score = sample_sigmoid(sample_deqnt_affine_u8_to_f32(max_score, score_zp, score_scale));
|
|
bbox[idx].class_type = max_class_id;
|
|
idx++;
|
|
}
|
|
}
|
|
}
|
|
|
|
// NMS抑制
|
|
xmedia_cl_float iou_threshold = DETECT_IOU_THRE;
|
|
xmedia_u32 out_box_len = 0;
|
|
ret = sample_det_nms(bbox, idx, iou_threshold,&out_box_len);
|
|
|
|
// 构造框结构体
|
|
xmedia_u32 w,h;
|
|
xmedia_s32 x1, y1, x2, y2;
|
|
xmedia_svp_yolov5_output result;
|
|
result.target_num = out_box_len;
|
|
for (i = 0; i < out_box_len; i++) {
|
|
x1 = (xmedia_s32)roundf(bbox[i].rect.x1 / 2) * 2;
|
|
y1 = (xmedia_s32)roundf(bbox[i].rect.y1 / 2) * 2;
|
|
x2 = (xmedia_s32)roundf(bbox[i].rect.x2 / 2) * 2;
|
|
y2 = (xmedia_s32)roundf(bbox[i].rect.y2 / 2) * 2;
|
|
w = sample_abs(x2 - x1);
|
|
h = sample_abs(y2 - y1);
|
|
target_rect[i].x = x1;
|
|
target_rect[i].y = y1;
|
|
target_rect[i].width = w;
|
|
target_rect[i].height = h;
|
|
|
|
result.targets[i].class_type = bbox[i].class_type;
|
|
|
|
printf("yolov8 %d target tag:%d (x1:%.2f, y1:%.2f) (x1:%.2f, y1:%.2f) (w: %u, h: %u) !\n",
|
|
i+1, bbox[i].class_type,
|
|
bbox[i].rect.x1, bbox[i].rect.y1,
|
|
bbox[i].rect.x2, bbox[i].rect.y2,
|
|
target_rect[i].width, target_rect[i].height);
|
|
}
|
|
|
|
// 构造帧存结构体
|
|
xmedia_video_frame_info video_frame;
|
|
memset(&video_frame, 0, sizeof(video_frame));
|
|
video_frame.pool_id = VB_INVALID_POOLID;
|
|
video_frame.frame.width = ALIGN_FUNC(image_width, ALIGN_BYTE);
|
|
video_frame.frame.height = ALIGN_FUNC(image_heigth, ALIGN_BYTE);
|
|
video_frame.frame.pixel_fmt = XMEDIA_VIDEO_PIXEL_FMT_YVU_SEMIPLANAR_420;
|
|
video_frame.frame.bit_width = XMEDIA_VIDEO_DATA_WIDTH_8;
|
|
video_frame.frame.stride.y_stride = ALIGN_FUNC(image_width, ALIGN_BYTE);
|
|
video_frame.frame.stride.c_stride = ALIGN_FUNC(image_width, ALIGN_BYTE);
|
|
video_frame.frame.addr.y_phy_addr = u64SrcPhyAddr[4];
|
|
video_frame.frame.addr.c_phy_addr = u64SrcPhyAddr[4] + image_width * image_heigth;
|
|
// 绘制框
|
|
sample_svp_draw(&target_rect[0], &video_frame, result);
|
|
|
|
// 保存 YUV 图像
|
|
xmedia_char file_path[MAX_FILE_LEN];
|
|
snprintf(file_path, sizeof(file_path), "./output/yolov8_detect_result_%ux%u_NV21.yuv", image_width, image_heigth);
|
|
FILE *file = fopen(file_path, "wb");
|
|
if (file == NULL) {
|
|
perror("file open failed\n");
|
|
return -1;
|
|
}
|
|
fwrite(nocache_buffer_start[4], sizeof(uint8_t), image_width * image_heigth * 3 / 2, file);
|
|
fclose(file);
|
|
|
|
CFREE(bbox);
|
|
|
|
ERROR:
|
|
//释放资源
|
|
sample_free_inout_tensor_mem(input);
|
|
sample_free_inout_tensor_mem(output);
|
|
|
|
for (i = 0; i < PHY_NUM; i++) {
|
|
if (nocache_buffer_start[i] != NULL) {
|
|
sample_mmzfree(u64SrcPhyAddr[i], nocache_buffer_start[i]);
|
|
}
|
|
}
|
|
for (i = 0; i < output.num; i++) {
|
|
if (buff[i] != NULL) {
|
|
free(buff[i]);
|
|
}
|
|
}
|
|
if (graph != NULL) {
|
|
err = xmedia_cl_graph_unload(graph);
|
|
if (err != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_graph_unload err, errno %d\n", err);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
CONTEXT_EXIT:
|
|
if (context != NULL) {
|
|
err = xmedia_cl_release_context(context);
|
|
if (err != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_release_context err, errno %d\n", err);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
DEVICE_EXIT:
|
|
if (devices != NULL) {
|
|
err = xmedia_cl_release_device_ids(devices, &num_devices);
|
|
if (err != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_release_device_ids err, errno %d\n", err);
|
|
return err;
|
|
}
|
|
free(devices);
|
|
}
|
|
|
|
CL_EXIT:
|
|
err = xmedia_cl_uninit();
|
|
if (err != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_cl_uninit err, errno %d\n", err);
|
|
return err;
|
|
}
|
|
|
|
TDE_EXIT:
|
|
xmedia_tde_exit();
|
|
|
|
SYS_EXIT:
|
|
err = xmedia_sys_exit();
|
|
if (err != XMEDIA_CL_SUCCESS) {
|
|
printf("xmedia_sys_exit err, errno %d\n", err);
|
|
return err;
|
|
}
|
|
|
|
return ret;
|
|
}
|