#include #include #include #include #include #include "xmedia_cl.h" #include "xmedia_mmz.h" #include "xmedia_sys.h" #include "xmedia_tde.h" #include "xmedia_video_common.h" #include "xmedia_svp.h" #include "xmedia_vgs.h" #include "xmedia_vb.h" #include #include #include #define ALIGN_BYTE 8 #define DEFAULT_INOUT_NUM 16 #define DFL_LEN 16 #define FEATURE_NUM 3 #define MAX_TARGET_NUM 30 #define AI_EPSILON 0.000001f #define PHY_NUM 5 #define MAX_FILE_LEN 256 /* 模型相关信息 */ #define IMAGE_WIDTH 640 #define IMAGE_HEIGTH 640 #define MODEL_FILE_NAME "./model/neuron_network.xmm" #define DETECT_THRE 0.7 #define DETECT_IOU_THRE 0.5 #define INPUT_IMAGE_FILE_NAME "./input/test_image1_640x640_NV21.yuv" #define ALIGN_FUNC(A, ALIGN) \ ((((A) % (ALIGN)) == 0) ? (A) : ((A) + (ALIGN) - ((A) % (ALIGN)))) #define STD_MAX(a, b) ((a) > (b) ? (a) : (b)) #define STD_MIN(a, b) ((a) > (b) ? (b) : (a)) #define CFREE(a) \ if (a) { \ free(a); \ a = NULL; \ } static xmedia_cl_s32 feature_map_size[FEATURE_NUM] = {80, 40, 20}; static xmedia_cl_s32 stride_down[FEATURE_NUM] = {8, 16, 32}; static xmedia_video_rect target_rect[MAX_TARGET_NUM]; typedef struct { xmedia_float x1; xmedia_float y1; xmedia_float x2; xmedia_float y2; } xmedia_ai_rect; typedef struct { xmedia_cl_s32 detect_type; xmedia_cl_s32 class_type; xmedia_float score; xmedia_float classfier_score; xmedia_float tracker_iou; xmedia_ai_rect rect; } xmedia_ai_detect_result; static xmedia_cl_s32 sample_abs(int num) { return num < 0 ? -num : num; } inline static xmedia_s32 sample_clamp(xmedia_float val, int min, int max) { return val > min ? (val < max ? val : max) : min; } inline static xmedia_float sample_sigmoid(xmedia_float x) { return 1.0 / (1.0 + expf(-x)); } inline static xmedia_float sample_unsigmoid(xmedia_float y) { return -1.0 * logf((1.0 / y) - 1.0); } inline static xmedia_s32 sample_clip(xmedia_float val, xmedia_float min, xmedia_float max) { xmedia_float f = val <= min ? min : (val >= max ? max : val); return f; } inline static xmedia_float sample_deqnt_affine_u8_to_f32(xmedia_u8 qnt, xmedia_s32 zp, xmedia_float scale) { return ((xmedia_float)qnt - (xmedia_float)zp) * scale; } static xmedia_u8 sample_qnt_f32_to_affine_u8(xmedia_float f32, xmedia_s32 zp, xmedia_float scale) { xmedia_float dst_val = (f32 / scale) + zp; xmedia_u8 res = (xmedia_u8)sample_clip(dst_val, 0, 255); return res; } static xmedia_cl_s32 sample_malloc_inout_tensor_mem(xmedia_cl_tensor_info_inout *inout) { inout->tensor = malloc(sizeof(xmedia_cl_tensor) * inout->num); inout->current_batch = malloc(sizeof(xmedia_cl_u32) * inout->num); inout->tensor_batch = malloc(sizeof(xmedia_cl_tensor_batch) * inout->num); if (inout->tensor == NULL || inout->current_batch == NULL || inout->tensor_batch == NULL) { return XMEDIA_CL_OUT_OF_HOST_MEMORY; } return XMEDIA_SUCCESS; } static xmedia_void sample_free_inout_tensor_mem(xmedia_cl_tensor_info_inout inout) { if (inout.tensor != NULL) { free(inout.tensor); } if (inout.current_batch != NULL) { free(inout.current_batch); } if (inout.tensor_batch != NULL) { free(inout.tensor_batch); } } static xmedia_void sample_compute_dfl(xmedia_float* tensor, xmedia_float* box) { for (xmedia_s32 b=0; b<4; b++){ xmedia_float exp_t[DFL_LEN]; xmedia_float exp_sum=0; xmedia_float acc_sum=0; for (xmedia_s32 i=0; i< DFL_LEN; i++){ exp_t[i] = exp(tensor[i+b*DFL_LEN]); exp_sum += exp_t[i]; } for (xmedia_s32 i=0; i< DFL_LEN; i++){ acc_sum += exp_t[i]/exp_sum *i; } box[b] = acc_sum; } } static inline xmedia_s32 cmp_score_fall(const xmedia_void *lsh, const xmedia_void *rsh) { // 降序排列, 令score最大的box在队列末端 xmedia_ai_detect_result *s1 = (xmedia_ai_detect_result *)lsh; xmedia_ai_detect_result *s2 = (xmedia_ai_detect_result *)rsh; return s1->score > s2->score ? -1 : 1; } static inline xmedia_float sample_iou (xmedia_ai_detect_result *box1, xmedia_ai_detect_result *box2, xmedia_bool total_ratio) { xmedia_float area1; xmedia_float area2; xmedia_float mix_area; if (box1->class_type != box2->class_type) { return 0.0f; } mix_area = STD_MAX(0.0f, STD_MIN(box1->rect.x2, box2->rect.x2) - STD_MAX(box1->rect.x1, box2->rect.x1)) * STD_MAX(0.0f, STD_MIN(box1->rect.y2, box2->rect.y2) - STD_MAX(box1->rect.y1, box2->rect.y1)); if (mix_area == 0.0f) { return 0.0f; } area1 = (box1->rect.x2 - box1->rect.x1) * (box1->rect.y2 - box1->rect.y1); area2 = (box2->rect.x2 - box2->rect.x1) * (box2->rect.y2 - box2->rect.y1); if (total_ratio) { xmedia_float tmp = area1 + area2 - mix_area; return (tmp == 0.0f) ? 0.0f : (mix_area / tmp); } else { return (area1 == 0.0f) ? 0.0f : (mix_area / area1); } } static xmedia_s32 sample_det_nms(xmedia_ai_detect_result *vec_bbox, xmedia_u32 vec_bbox_len, xmedia_float threshold, xmedia_u32 *picked_bbox_len) { xmedia_u32 i; xmedia_u32 j; xmedia_ai_detect_result *box_b; xmedia_ai_detect_result *box_a; xmedia_ai_detect_result *picked_bbox; xmedia_u32 valid_count = 0; qsort(vec_bbox, vec_bbox_len, sizeof(xmedia_ai_detect_result), cmp_score_fall); for (i = 0; i < vec_bbox_len; i++) { box_a = &vec_bbox[i]; if (box_a->score == 0.0) { continue; } for (j = i + 1; j < vec_bbox_len; j++) { box_b = &vec_bbox[j]; if (box_b->score == 0.0) { continue; } xmedia_float box_iou = sample_iou(box_a, box_b, XMEDIA_TRUE); if (box_iou >= threshold) { box_b->score = 0.0; } } valid_count++; } picked_bbox = (xmedia_ai_detect_result *)malloc(valid_count * sizeof(xmedia_ai_detect_result)); if (picked_bbox == XMEDIA_NULL) { *picked_bbox_len = 0; return XMEDIA_NULL; } for (i = 0, j = 0; i < vec_bbox_len; i++) { if (vec_bbox[i].score == 0.0) { continue; } if (j < valid_count) { picked_bbox[j++] = vec_bbox[i]; } } *picked_bbox_len = valid_count; memcpy(vec_bbox, picked_bbox, sizeof(xmedia_ai_detect_result) * valid_count); CFREE(picked_bbox); return XMEDIA_SUCCESS; } static xmedia_s32 sample_svp_draw(xmedia_video_rect target_rect[], xmedia_video_frame_info *frame, xmedia_svp_yolov5_output result) { xmedia_s32 ret; xmedia_vgs_frame_info task = {0}; xmedia_vgs_cover_attr cover[XMEDIA_SVP_MAX_TARGET_NUM]; xmedia_s32 handle = -1; xmedia_s32 i; xmedia_u8 object_num = result.target_num; if (0 == object_num) { return XMEDIA_FAILURE; } object_num = object_num > XMEDIA_SVP_MAX_TARGET_NUM ? XMEDIA_SVP_MAX_TARGET_NUM : object_num; memcpy(&task.img_in,frame,sizeof(xmedia_video_frame_info)); memcpy(&task.img_out,frame,sizeof(xmedia_video_frame_info)); ret = xmedia_vgs_init(); if(ret != XMEDIA_SUCCESS) { printf("xmedia_vgs_init failed !\n"); return ret; } ret = xmedia_vgs_create_job(&handle); if (XMEDIA_SUCCESS != ret) { xmedia_vgs_exit(); printf("xmedia_vgs_cancel_job failed !\n"); handle = -1; return XMEDIA_FAILURE; } for (i = 0; i < object_num; i++) { if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_FIREWORKS_SMOKE || result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_TRICYCLER) { cover[i].color = 0x0000FF; } else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_BIKER) { cover[i].color = 0x000FFF; } else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_MOTOR) { cover[i].color = 0x0F000F; } else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_MOTORER) { cover[i].color = 0xFF00FF; } else if (result.targets[i].class_type == XMEDIA_SVP_CLASS_TYPE_TRICYCLE) { cover[i].color = 0x00FF00; } else { cover[i].color = 0xFF0000; } cover[i].cover_type = XMEDIA_VGS_COVER_TYPE_QUAD_RANGLE; cover[i].quadrangle.is_solid = XMEDIA_FALSE; cover[i].quadrangle.thick = 2; cover[i].quadrangle.points[0].x = target_rect[i].x; cover[i].quadrangle.points[0].y = target_rect[i].y; cover[i].quadrangle.points[1].x = target_rect[i].x + target_rect[i].width; cover[i].quadrangle.points[1].y = target_rect[i].y; cover[i].quadrangle.points[2].x = target_rect[i].x + target_rect[i].width; cover[i].quadrangle.points[2].y = target_rect[i].y + target_rect[i].height; cover[i].quadrangle.points[3].x = target_rect[i].x; cover[i].quadrangle.points[3].y = target_rect[i].y + target_rect[i].height; } ret = xmedia_vgs_add_task_cover(handle, &task, &cover[0], object_num); if (XMEDIA_SUCCESS != ret) { xmedia_vgs_cancel_job(handle); xmedia_vgs_exit(); printf("xmedia_vgs_add_task_cover failed ! ret [%#x] \n", ret); return XMEDIA_FAILURE; } ret = xmedia_vgs_submit_job(handle); if (XMEDIA_SUCCESS != ret) { xmedia_vgs_cancel_job(handle); xmedia_vgs_exit(); printf("xmedia_vgs_submit_job failed ! ret [%d] \n", ret); return XMEDIA_FAILURE; } ret = xmedia_vgs_wait_job(handle, 2000); if (ret != XMEDIA_SUCCESS) { xmedia_vgs_exit(); printf("xmedia_vgs_wait_job failed ! ret [%d] \n", ret); return ret; } return XMEDIA_SUCCESS; } static xmedia_s32 sample_mmzalloc_cached(xmedia_u64 *pu64PhyAddr, xmedia_void **ppVirAddr, const xmedia_char *strMmb, const xmedia_char *strZone, xmedia_u32 u32Len) { *pu64PhyAddr = xmedia_mmz_alloc(strZone, strMmb, u32Len); if (*pu64PhyAddr == XMEDIA_NULL) { return XMEDIA_CL_OUT_OF_HOST_MEMORY; } *ppVirAddr = xmedia_mmz_map(*pu64PhyAddr, u32Len, 1); if (*ppVirAddr == XMEDIA_NULL) { return XMEDIA_CL_OUT_OF_HOST_MEMORY; } return XMEDIA_SUCCESS; } static xmedia_s32 sample_mmzfree(xmedia_u64 u64PhyAddr, xmedia_void *pVirAddr) { xmedia_cl_s32 ret; if (pVirAddr != XMEDIA_NULL) { ret = xmedia_mmz_unmap(pVirAddr); if (ret != XMEDIA_CL_SUCCESS) { return XMEDIA_FAILURE; } } return xmedia_mmz_free(u64PhyAddr); } static xmedia_s32 sample_get_input_one_data(xmedia_void* data_yuv, xmedia_u32 image_size) { FILE* fp_temp = fopen(INPUT_IMAGE_FILE_NAME, "rb"); if (fp_temp) { fread(data_yuv, image_size, 1, fp_temp); } else { printf("fopen failed.\n"); return XMEDIA_FAILURE; } fclose(fp_temp); fp_temp = XMEDIA_NULL; return XMEDIA_SUCCESS; } static xmedia_s32 sample_tde_y2r(xmedia_cl_tensor_info_inout input, xmedia_u64 img_src_phy, xmedia_u64 img_src_rgb_phy) { xmedia_s32 handle = XMEDIA_FAILURE; xmedia_s32 ret; xmedia_tde_surface_info src = {0}; xmedia_tde_surface_info dst = {0}; ret = xmedia_tde_create_job(&handle); if (XMEDIA_SUCCESS != ret) { printf("xmedia_tde_create_job failed %#x!\n",ret); handle = XMEDIA_FAILURE; return XMEDIA_FAILURE; } src.surface.phys_addr[0] = img_src_phy; src.surface.phys_addr[1] = img_src_phy + input.tensor[0].shape.dims[2] * input.tensor[0].shape.dims[1]; src.surface.stride[0] = input.tensor[0].shape.dims[2]; src.surface.stride[1] = input.tensor[0].shape.dims[2]; src.surface.width = input.tensor[0].shape.dims[2]; src.surface.height = input.tensor[0].shape.dims[1]; src.surface.pixel_format = XMEDIA_VIDEO_PIXEL_FMT_YVU_SEMIPLANAR_420; src.surface.alpha0 = 0; src.surface.alpha1 = 255; src.surface.is_alpha_ext_1555 = 0; src.surface.clut_reload = XMEDIA_FALSE; src.rect.x = 0; src.rect.y = 0; src.rect.width = input.tensor[0].shape.dims[2]; src.rect.height = input.tensor[0].shape.dims[1]; dst.surface.phys_addr[0] = img_src_rgb_phy; dst.surface.width = input.tensor[0].shape.dims[2]; dst.surface.height = input.tensor[0].shape.dims[1]; dst.surface.stride[0] = input.tensor[0].shape.dims[2] * 3; dst.surface.pixel_format = XMEDIA_VIDEO_PIXEL_FMT_RGB_888; dst.surface.is_alpha_ext_1555 = XMEDIA_FALSE; dst.surface.alpha0 = 0; dst.surface.alpha1 = 255; dst.surface.clut_reload = XMEDIA_FALSE; dst.rect.x = 0; dst.rect.y = 0; dst.rect.width = input.tensor[0].shape.dims[2]; dst.rect.height = input.tensor[0].shape.dims[1]; ret = xmedia_tde_add_task_single_blit(handle, &src, &dst, 0, XMEDIA_NULL); if (XMEDIA_SUCCESS != ret) { printf("xmedia_tde_add_task_scale failed ret: %#x !\n", ret); xmedia_tde_cancel_job(handle); return XMEDIA_FAILURE; } ret = xmedia_tde_submit_job(handle); if (XMEDIA_SUCCESS != ret) { printf("xmedia_tde_submit_job failed ,ret: %#x!\n", ret); xmedia_tde_cancel_job(handle); return XMEDIA_FAILURE; } ret = xmedia_tde_wait_job(handle, 2000); if (XMEDIA_SUCCESS != ret) { printf("tde overtime %#x!\n",ret); return XMEDIA_FAILURE; } return XMEDIA_SUCCESS; } int main() { void *buff[DEFAULT_INOUT_NUM] = { 0 }; xmedia_cl_s8 name[MAX_FILE_LEN] = { 0 }; xmedia_cl_u64 u64SrcPhyAddr[PHY_NUM] = { 0 }; xmedia_char *nocache_buffer_start[PHY_NUM] = { 0 }; xmedia_cl_s32 output_size[DEFAULT_INOUT_NUM] = { 0 }; xmedia_cl_s8 *model = MODEL_FILE_NAME; xmedia_cl_s32 i = 0, j, size = 0, ret, err; xmedia_cl_graph graph = XMEDIA_NULL; xmedia_cl_u32 num_devices = 0, input_num = 0, output_num = 0; xmedia_cl_tensor_info_inout input = { 0 }, output = { 0 }; xmedia_cl_s32 err_code = 0; xmedia_cl_context context = XMEDIA_NULL; xmedia_cl_device_id *devices = XMEDIA_NULL; xmedia_cl_u32 worksize, weightsize, inputsize = 0, outputsize = 0; xmedia_u32 image_size; xmedia_s32 image_width, image_heigth; // 公共模块初始化 ret = xmedia_sys_init(XMEDIA_NULL); if (ret != XMEDIA_SUCCESS) { printf("xmedia_sys_init err, errno %d\n", ret); return ret; } ret = xmedia_tde_init(); if (XMEDIA_SUCCESS != ret) { printf("xmedia_tde_init failed %#x!\n",ret); goto SYS_EXIT; } // 运行时初始化 ret = xmedia_cl_init(); if (ret != XMEDIA_SUCCESS) { printf("xmedia_cl_init err, errno %d\n", ret); goto TDE_EXIT; } // 第一次调用该接口获取设备数量 ret = xmedia_cl_get_device_ids(XMEDIA_CL_DEVICE_ALL, NULL, &num_devices); if (ret != XMEDIA_SUCCESS) { printf("xmedia_cl_get_device_ids err, errno %d\n", ret); goto CL_EXIT; } devices = (xmedia_cl_device_id *)calloc(num_devices, sizeof(xmedia_cl_device_id)); if (devices == NULL) { printf("calloc err\n"); ret = XMEDIA_CL_OUT_OF_HOST_MEMORY; goto CL_EXIT; } // 第二次调用该接口传递获取到的设备数量申请设备资源 ret = xmedia_cl_get_device_ids(XMEDIA_CL_DEVICE_ALL, devices, &num_devices); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_get_device_ids err, errno %d\n", ret); free(devices); goto CL_EXIT; } // 创建资源管理的对象 context = xmedia_cl_create_context(num_devices, devices, &err_code); if (err_code != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_create_context err, errno %d\n", ret); goto DEVICE_EXIT; } // 从模型文件中查询work和weight的大小 ret = xmedia_cl_graph_querysize_from_file(model, &worksize, &weightsize); if (XMEDIA_CL_SUCCESS != ret) { printf("xmedia_cl_graph_querysize_from_file, errno %d\n", ret); goto CONTEXT_EXIT; } if (worksize) { // 根据查询到的大小申请work和weight的内存 ret = sample_mmzalloc_cached(&u64SrcPhyAddr[0], (void **)(&nocache_buffer_start[0]), "npu_workspace", NULL, worksize); if (XMEDIA_CL_SUCCESS != ret) { goto CONTEXT_EXIT; } } else { nocache_buffer_start[0] = NULL; } if (weightsize) { ret = sample_mmzalloc_cached(&u64SrcPhyAddr[1], (void **)(&nocache_buffer_start[1]), "npu_weight", NULL, weightsize); if (XMEDIA_CL_SUCCESS != ret) { goto ERROR; } } else { nocache_buffer_start[1] = NULL; } // 加载解析模型文件,若是从文件中加载,则model为文件路径,若从内存中加载,则model为内存地址 ret = xmedia_cl_graph_loadmodel_from_file_withmem( &context, model, nocache_buffer_start[0], worksize, nocache_buffer_start[1], weightsize, &graph); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_loadmodel err, errno %d\n", ret); goto ERROR; } // 第一次调用该接口获取输入数量 ret = xmedia_cl_graph_get_input(graph, input_num, &input); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_get_input, errno %d\n", ret); goto ERROR; } if (input.num > DEFAULT_INOUT_NUM) { printf("model input num is greater than default value!\n"); goto ERROR; } // 根据输入数量申请内存 sample_malloc_inout_tensor_mem(&input); input_num = input.num; // 第二次调用该接口获取输入信息 ret = xmedia_cl_graph_get_input(graph, input_num, &input); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_get_input, errno %d\n", ret); goto ERROR; } // 第一次调用该接口获取输出数量 ret = xmedia_cl_graph_get_output(graph, output_num, &output); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_get_output, errno %d\n", ret); goto ERROR; } if (output.num > DEFAULT_INOUT_NUM) { printf("model output num is greater than default value!\n"); goto ERROR; } // 根据输出数量申请内存 sample_malloc_inout_tensor_mem(&output); output_num = output.num; // 第二次调用该接口获取输出信息 ret = xmedia_cl_graph_get_output(graph, output_num, &output); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_get_output, errno %d\n", ret); goto ERROR; } // 计算输入的大小并申请内存 for (i = 0; i < input.num; i++) { size = input.tensor[i].size; inputsize += ALIGN_FUNC(size, ALIGN_BYTE); } ret = sample_mmzalloc_cached(&u64SrcPhyAddr[2], (void **)(&nocache_buffer_start[2]), "npu_input", NULL, inputsize); if (XMEDIA_CL_SUCCESS != ret) { goto ERROR; } // 计算输出的大小并申请内存 for (i = 0; i < output.num; i++) { output_size[i] = output.tensor[i].size; outputsize += ALIGN_FUNC(output_size[i], ALIGN_BYTE); } ret = sample_mmzalloc_cached(&u64SrcPhyAddr[3], (void **)(&nocache_buffer_start[3]), "npu_output", NULL, outputsize); if (XMEDIA_CL_SUCCESS != ret) { goto ERROR; } image_width = IMAGE_WIDTH; image_heigth = IMAGE_HEIGTH; image_size = image_width * image_heigth * 3 / 2; // 申请yuv图像内存 sample_mmzalloc_cached(&u64SrcPhyAddr[4], (void **)(&nocache_buffer_start[4]), "test_yuv_image", NULL, image_size); // 读取yuv图像 ret = sample_get_input_one_data(nocache_buffer_start[4], image_size); if (ret != XMEDIA_SUCCESS) { printf("get input data failed!\n"); } sample_tde_y2r(input, u64SrcPhyAddr[4], u64SrcPhyAddr[2]); // 设置输入,把输入golden数据拷贝到输入地址,每个输入必须设置一次 for (i = 0; i < input.num; i++) { if (i > 0) { input.tensor[i].addr = input.tensor[i - 1].addr + ALIGN_FUNC(size, ALIGN_BYTE); } else { input.tensor[i].addr = nocache_buffer_start[2]; } size = input.tensor[i].size; printf("input.tensor[%d].addr = %p input size = 0x%x\n", i, input.tensor[i].addr, size); } // 把输出的golden数据拷贝到内存,用来与npu的输出做对比 for (i = 0; i < output.num; i++) { if (i > 0) { output.tensor[i].addr = output.tensor[i - 1].addr + ALIGN_FUNC(output_size[i - 1], ALIGN_BYTE); } else { output.tensor[i].addr = nocache_buffer_start[3]; } memset(name, 0, sizeof(name)); output_size[i] = output.tensor[i].size; printf("output.tensor[%d].addr = %p output size = %d\n", i, output.tensor[i].addr, output_size[i]); } // 在xmm2下,用户需要申请输入地址,输出地址可以动态改变,把输入与输出设置为推理的输入、输出 ret = xmedia_cl_graph_set_inout(graph, &input, &output); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_set_inout, errno %d\n", ret); goto ERROR; } struct timeval t_start, t_end; gettimeofday(&t_start, NULL); ret = xmedia_cl_graph_process(graph); gettimeofday(&t_end, NULL); printf("xmedia_cl_graph_process start time = [%ld]\n", t_start.tv_usec); printf("xmedia_cl_graph_process end time = [%ld]\n", t_end.tv_usec); printf("xmedia_cl_graph_process timeval = [%ld] us\n", ((t_end.tv_sec - t_start.tv_sec) * 1000000) + t_end.tv_usec - t_start.tv_usec); if (ret != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_process err, errno %d\n", ret); } // 解析输出tensor xmedia_cl_s32 feature_map_num = FEATURE_NUM; xmedia_cl_s32 index; xmedia_cl_u32 idx = 0; xmedia_cl_float detect_threshold = DETECT_THRE; xmedia_cl_u32 target_num = MAX_TARGET_NUM; xmedia_ai_detect_result *bbox; bbox = (xmedia_ai_detect_result *)calloc(1, target_num * sizeof(xmedia_ai_detect_result)); for (index = 0; index < feature_map_num; index++) { xmedia_cl_s32 grid_h = feature_map_size[index]; xmedia_cl_s32 grid_w = feature_map_size[index]; xmedia_cl_s32 stride = stride_down[index]; xmedia_cl_s32 box_index = index * 2; xmedia_cl_s32 score_index = index * 2 + 1; xmedia_cl_s32 box_zp = output.tensor[box_index].quant.zp; xmedia_cl_float box_scale = output.tensor[box_index].quant.scale; xmedia_cl_u8* box_tensor = output.tensor[box_index].addr; xmedia_cl_s32 score_zp = output.tensor[score_index].quant.zp; xmedia_cl_float score_scale = output.tensor[score_index].quant.scale; xmedia_cl_u8* score_tensor = output.tensor[score_index].addr; xmedia_cl_s32 grid_len = grid_w * grid_h; xmedia_cl_u8 score_thres_u8 = sample_qnt_f32_to_affine_u8(sample_unsigmoid(detect_threshold), score_zp, score_scale); for (i = 0; i < grid_h; i++) { for (j = 0; j < grid_w; j++) { xmedia_cl_s32 score_offset = i * grid_w + j; xmedia_cl_s32 max_class_id = -1; xmedia_cl_s32 bbox_offset = i * grid_w + j; xmedia_cl_u8 max_score = -score_zp; xmedia_cl_float box[4]; xmedia_cl_float before_dfl[DFL_LEN * 4]; xmedia_cl_float x1, x2, y1, y2; if (idx >= target_num) { break; } for (xmedia_cl_s32 c = 0; c < 80; c++) { if ((score_tensor[score_offset] > score_thres_u8) && (score_tensor[score_offset] > max_score)) { max_score = score_tensor[score_offset]; max_class_id = c; } score_offset += grid_len; } if (max_score < score_thres_u8) { continue; } for (xmedia_cl_s32 k = 0; k < DFL_LEN * 4; k++) { before_dfl[k] = sample_deqnt_affine_u8_to_f32(box_tensor[bbox_offset], box_zp, box_scale); bbox_offset += grid_len; } sample_compute_dfl(before_dfl, box); x1 = (-box[0] + j + 0.5) * stride; y1 = (-box[1] + i + 0.5) * stride; x2 = (box[2] + j + 0.5) * stride; y2 = (box[3] + i + 0.5) * stride; bbox[idx].rect.x1 = sample_clamp(x1, 0, image_width); bbox[idx].rect.y1 = sample_clamp(y1, 0, image_heigth); bbox[idx].rect.x2 = sample_clamp(x2, 0, image_width); bbox[idx].rect.y2 = sample_clamp(y2, 0, image_heigth); bbox[idx].score = sample_sigmoid(sample_deqnt_affine_u8_to_f32(max_score, score_zp, score_scale)); bbox[idx].class_type = max_class_id; idx++; } } } // NMS抑制 xmedia_cl_float iou_threshold = DETECT_IOU_THRE; xmedia_u32 out_box_len = 0; ret = sample_det_nms(bbox, idx, iou_threshold,&out_box_len); // 构造框结构体 xmedia_u32 w,h; xmedia_s32 x1, y1, x2, y2; xmedia_svp_yolov5_output result; result.target_num = out_box_len; for (i = 0; i < out_box_len; i++) { x1 = (xmedia_s32)roundf(bbox[i].rect.x1 / 2) * 2; y1 = (xmedia_s32)roundf(bbox[i].rect.y1 / 2) * 2; x2 = (xmedia_s32)roundf(bbox[i].rect.x2 / 2) * 2; y2 = (xmedia_s32)roundf(bbox[i].rect.y2 / 2) * 2; w = sample_abs(x2 - x1); h = sample_abs(y2 - y1); target_rect[i].x = x1; target_rect[i].y = y1; target_rect[i].width = w; target_rect[i].height = h; result.targets[i].class_type = bbox[i].class_type; printf("yolov8 %d target tag:%d (x1:%.2f, y1:%.2f) (x1:%.2f, y1:%.2f) (w: %u, h: %u) !\n", i+1, bbox[i].class_type, bbox[i].rect.x1, bbox[i].rect.y1, bbox[i].rect.x2, bbox[i].rect.y2, target_rect[i].width, target_rect[i].height); } // 构造帧存结构体 xmedia_video_frame_info video_frame; memset(&video_frame, 0, sizeof(video_frame)); video_frame.pool_id = VB_INVALID_POOLID; video_frame.frame.width = ALIGN_FUNC(image_width, ALIGN_BYTE); video_frame.frame.height = ALIGN_FUNC(image_heigth, ALIGN_BYTE); video_frame.frame.pixel_fmt = XMEDIA_VIDEO_PIXEL_FMT_YVU_SEMIPLANAR_420; video_frame.frame.bit_width = XMEDIA_VIDEO_DATA_WIDTH_8; video_frame.frame.stride.y_stride = ALIGN_FUNC(image_width, ALIGN_BYTE); video_frame.frame.stride.c_stride = ALIGN_FUNC(image_width, ALIGN_BYTE); video_frame.frame.addr.y_phy_addr = u64SrcPhyAddr[4]; video_frame.frame.addr.c_phy_addr = u64SrcPhyAddr[4] + image_width * image_heigth; // 绘制框 sample_svp_draw(&target_rect[0], &video_frame, result); // 保存 YUV 图像 xmedia_char file_path[MAX_FILE_LEN]; snprintf(file_path, sizeof(file_path), "./output/yolov8_detect_result_%ux%u_NV21.yuv", image_width, image_heigth); FILE *file = fopen(file_path, "wb"); if (file == NULL) { perror("file open failed\n"); return -1; } fwrite(nocache_buffer_start[4], sizeof(uint8_t), image_width * image_heigth * 3 / 2, file); fclose(file); CFREE(bbox); ERROR: //释放资源 sample_free_inout_tensor_mem(input); sample_free_inout_tensor_mem(output); for (i = 0; i < PHY_NUM; i++) { if (nocache_buffer_start[i] != NULL) { sample_mmzfree(u64SrcPhyAddr[i], nocache_buffer_start[i]); } } for (i = 0; i < output.num; i++) { if (buff[i] != NULL) { free(buff[i]); } } if (graph != NULL) { err = xmedia_cl_graph_unload(graph); if (err != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_graph_unload err, errno %d\n", err); return err; } } CONTEXT_EXIT: if (context != NULL) { err = xmedia_cl_release_context(context); if (err != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_release_context err, errno %d\n", err); return err; } } DEVICE_EXIT: if (devices != NULL) { err = xmedia_cl_release_device_ids(devices, &num_devices); if (err != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_release_device_ids err, errno %d\n", err); return err; } free(devices); } CL_EXIT: err = xmedia_cl_uninit(); if (err != XMEDIA_CL_SUCCESS) { printf("xmedia_cl_uninit err, errno %d\n", err); return err; } TDE_EXIT: xmedia_tde_exit(); SYS_EXIT: err = xmedia_sys_exit(); if (err != XMEDIA_CL_SUCCESS) { printf("xmedia_sys_exit err, errno %d\n", err); return err; } return ret; }