add onboard_inference project

GAP8/ai_examples/onboard_inference/Makefile

@@ -0,0 +1,93 @@
+# Copyright (C) 2017 GreenWaves Technologies
+# All rights reserved.
+
+# This software may be modified and distributed under the terms
+# of the BSD license.  See the LICENSE file for details.
+
+ifndef GAP_SDK_HOME
+  $(error Source sourceme in gap_sdk first)
+endif
+
+MODEL_PREFIX=model
+AT_INPUT_WIDTH=28
+AT_INPUT_HEIGHT=28
+AT_INPUT_COLORS=3
+
+LOAD_QUANTIZATION= #-q #to load a tflite quantized model
+
+io=host
+
+
+RAM_FLASH_TYPE ?= HYPER
+#PMSIS_OS=freertos
+
+ifeq '$(RAM_FLASH_TYPE)' 'HYPER'
+APP_CFLAGS += -DUSE_HYPER
+MODEL_L3_EXEC=hram
+MODEL_L3_CONST=hflash
+else
+APP_CFLAGS += -DUSE_SPI
+CONFIG_SPIRAM = 1
+MODEL_L3_EXEC=qspiram
+MODEL_L3_CONST=qpsiflash
+endif
+
+
+QUANT_BITS?=8
+BUILD_DIR=BUILD
+ifeq ($(QUANT_BITS), 8)
+	MODEL_SQ8=1
+  ifeq '$(RAM_FLASH_TYPE)' 'HYPER'
+    NNTOOL_SCRIPT=model/nntool_script
+  else
+    NNTOOL_SCRIPT=model/nntool_script_spi
+  endif
+    MODEL_SUFFIX = _SQ8BIT
+else
+  ifeq ($(QUANT_BITS),16)
+    NNTOOL_SCRIPT=model/nntool_script16
+    MODEL_SUFFIX = _16BIT
+  else
+    $(error Don\'t know how to build with this bit width)
+  endif
+endif
+
+$(info Building GAP8 mode with $(QUANT_BITS) bit quantization)
+
+include model_decl.mk
+
+# Here we set the memory allocation for the generated kernels
+# REMEMBER THAT THE L1 MEMORY ALLOCATION MUST INCLUDE SPACE
+# FOR ALLOCATED STACKS!
+CLUSTER_STACK_SIZE=4028
+CLUSTER_SLAVE_STACK_SIZE=1024
+TOTAL_STACK_SIZE=$(shell expr $(CLUSTER_STACK_SIZE) \+ $(CLUSTER_SLAVE_STACK_SIZE) \* 7)
+MODEL_L1_MEMORY=$(shell expr 60000 \- $(TOTAL_STACK_SIZE))
+MODEL_L2_MEMORY=250000
+MODEL_L3_MEMORY=8388608
+
+pulpChip = GAP
+PULP_APP = model
+
+APP = model
+APP_SRCS += $(MODEL_PREFIX).c $(MODEL_GEN_C) $(MODEL_COMMON_SRCS) $(CNN_LIB)
+
+APP_CFLAGS += -g -O1 -mno-memcpy -fno-tree-loop-distribute-patterns
+APP_CFLAGS += -I. -I$(MODEL_COMMON_INC) -I$(TILER_EMU_INC) -I$(TILER_INC) $(CNN_LIB_INCLUDE) -I$(MODEL_BUILD)
+APP_CFLAGS += -DPERF -DAT_MODEL_PREFIX=$(MODEL_PREFIX) $(MODEL_SIZE_CFLAGS)
+APP_CFLAGS += -DSTACK_SIZE=$(CLUSTER_STACK_SIZE) -DSLAVE_STACK_SIZE=$(CLUSTER_SLAVE_STACK_SIZE)
+APP_CFLAGS += -DAT_IMAGE=$(IMAGE)
+
+READFS_FILES=$(abspath $(MODEL_TENSORS))
+
+# all depends on the model
+all:: model
+
+clean:: clean_model
+
+include train_model.mk
+include model_rules.mk
+$(info APP_SRCS... $(APP_SRCS))
+$(info APP_CFLAGS... $(APP_CFLAGS))
+include $(RULES_DIR)/pmsis_rules.mk
+

GAP8/ai_examples/onboard_inference/README.md

@@ -0,0 +1,64 @@
+# 构建数据集
+
+- 数据集组织方式：
+
+```
+\data
+   \train
+       \class1
+          \xxx.jpg
+          \xxx.jpg
+       \class2
+       ...
+       \classn
+   \test
+       \xxx.jpg
+       \xxx.jpg
+       ...
+       \xxx.jpg
+```
+
+- data_dir = "训练集根目录"
+
+- 构建方式
+
+```
+python dataloader.py
+```
+
+保存生成的train.npz、val.npz文件
+
+
+# 环境要求
+
+- GAP_SDK: release-v3.8.1  (docker or ubuntu)
+- Keras: 2.3.1
+- tensorflow: 1.15.2
+- numpy: 1.16.2
+
+
+# 运行实例
+
+- 在GAP8上运行
+
+```
+make clean all run'
+```
+
+- 在gvsoc虚拟平台运行
+
+```
+make clean all run platform=gvsoc
+```
+
+- 固化到芯片
+
+```
+make clean all image flash
+```
+
+# 清理编译产物
+
+```
+make clean clean_train clean_images clean_model
+```

GAP8/ai_examples/onboard_inference/bridge_stubs.h

@@ -0,0 +1,16 @@
+#ifndef __BRIDGE_STUBS_H__
+#define __BRIDGE_STUBS_H__
+
+#if defined(__PULP_OS__)
+
+#define BRIDGE_Init() {}
+#define BRIDGE_Connect(__wait_bridge, __event)          rt_bridge_connect(__wait_bridge, __event)
+#define BRIDGE_Open(__name, __flags, __mode, __event)   rt_bridge_open(__name, __flags, __mode, __event)
+#define BRIDGE_Read(__file, __buffer, __size, __event)  rt_bridge_read(__file, __buffer, __size, __event)
+#define BRIDGE_Write(__file, __buffer, __size, __event) rt_bridge_write(__file, __buffer, __size, __event)
+#define BRIDGE_Close(__file, __event)                   rt_bridge_close(__file, __event)
+#define BRIDGE_Disconnect(__event)                      rt_bridge_disconnect(__event)
+
+#endif  /* __PULP_OS__ */
+
+#endif  /* __BRIDGE_STUBS_H__ */

GAP8/ai_examples/onboard_inference/dataloader.py

@@ -0,0 +1,91 @@
+import os
+import numpy as np
+from PIL import Image
+import random
+
+
+def savelist(data_dir):
+	"""
+	从输入根目录获取图像路径、标签列表
+	"""
+	image_paths = []
+	image_labels = []
+
+	print(data_dir)
+
+	image_classes = os.listdir(data_dir)
+	print(image_classes)
+
+	for image_class in image_classes:
+	    sub_dir = os.path.join(data_dir + "\\" + image_class)
+	    # 进入子类目录
+	    image_names = os.listdir(sub_dir)
+	    for image_name in image_names:
+	        sub_path = os.path.join(sub_dir + "\\" + image_name)
+	        # 操作每类图片
+	        image_paths.append(sub_path)
+	        image_labels.append(image_classes.index(sub_dir[len(data_dir)+1:]))
+
+
+	# 随机打乱
+	random.seed(123)
+	random.shuffle(image_paths)
+	random.seed(123)
+	random.shuffle(image_labels)
+
+	image_count = len(image_paths)
+	split = 0.2
+
+	val_image_paths = image_paths[:int(split*image_count)]
+	train_image_paths = image_paths[int(split*image_count):]
+
+	val_image_labels = image_labels[:int(split*image_count)]
+	train_image_labels = image_labels[int(split*image_count):]
+
+	return train_image_paths, train_image_labels, val_image_paths, val_image_labels
+
+
+
+
+def save_as_numpy(image_paths, image_labels,str):
+	d = len(image_paths)
+	print(d)
+	data = np.empty((d, 28, 28, 3),dtype=np.uint8)
+	print(data)
+	while d>0:
+	    img = Image.open(image_paths[d-1])
+	    img = img.resize((28,28),Image.ANTIALIAS)
+	    img_ndarray = np.array(img)
+	    print(img_ndarray.shape)
+	    data[d - 1] = img_ndarray
+	    d = d - 1
+
+	print(str + "转换完成")
+	print(data)
+
+	d = len(image_labels)
+	label = np.zeros((d),dtype=np.uint8)
+	print(label)
+	while d>0:
+	    label[d - 1] = image_labels[d-1]
+	    d = d - 1
+	print(str + "转换完成")
+	print(label)
+
+	np.save(str + '_images.npy',data)
+	np.save(str + '_labels.npy',label)
+	np.savez(str + '.npz',image=data,label=label)
+
+
+
+
+if __name__=='__main__':
+
+	data_dir = ""
+
+	train_images, train_labels, val_images, val_labels = savelist(data_dir)
+
+
+	save_as_numpy(train_images, train_labels, str='train')
+	save_as_numpy(val_images, val_labels, str='val')
+

GAP8/ai_examples/onboard_inference/img_l3.ppm

@@ -0,0 +1,4 @@
+P6
+28 28
+255
+ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�ﾭ�

GAP8/ai_examples/onboard_inference/img_proc.c

@@ -0,0 +1,89 @@
+#include "img_proc.h"
+
+void demosaicking(char *input, char* output, int width, int height, bool grayscale)
+{
+    int idx = 0;
+    int idxr[8];
+    char red, blue, green;
+
+    for (int y = 0; y < height ; y++)
+    {
+        for (int x = 0; x < width ; x++)
+        {
+            int idx = y * width + x;
+
+            if (x == 0 || y == 0 || x == width-1 || y == height-1)
+            {
+                if(grayscale)
+                {
+                    output[idx] = 0;
+                }
+                else
+                {
+                    output[idx * 3] = 0;
+                    output[idx * 3 + 1] = 0;
+                    output[idx * 3 + 2] = 0;
+                }
+            }
+            else
+            {
+
+                idxr[0] = (y - 1) * width + (x - 1);
+                idxr[1] = (y)*width + (x - 1);
+                idxr[2] = (y + 1) * width + (x - 1);
+                idxr[3] = (y + 1) * width + (x);
+                idxr[4] = (y + 1) * width + (x + 1);
+                idxr[5] = (y)*width + (x + 1);
+                idxr[6] = (y - 1) * width + (x + 1);
+                idxr[7] = (y - 1) * width + (x);
+
+                int x_shift = 0;
+                int y_shift = 0;
+
+                if ((x + x_shift) % 2 == 0 && (y + y_shift) % 2 == 0) //R
+                {
+                    red = input[idx];
+                    blue = (input[idxr[0]] + input[idxr[2]] + input[idxr[4]] + input[idxr[6]]) / 4;
+                    green = (input[idxr[1]] + input[idxr[3]] + input[idxr[5]] + input[idxr[7]]) / 4;
+                }
+                else if ((x + x_shift) % 2 == 1 && (y + y_shift) % 2 == 0) //G2
+                {
+                    red = (input[idxr[1]] + input[idxr[5]]) / 2;
+                    blue = (input[idxr[3]] + input[idxr[7]]) / 2;
+                    green = input[idx];
+                }
+                else if ((x + x_shift) % 2 == 0 && (y + y_shift) % 2 == 1) //G1
+                {
+                    red = (input[idxr[3]] + input[idxr[7]]) / 2;
+                    blue = (input[idxr[1]] + input[idxr[5]]) / 2;
+                    green = input[idx];
+                }
+                else if ((x + x_shift) % 2 == 1 && (y + y_shift) % 2 == 1) //B
+                {
+                    red = (input[idxr[0]] + input[idxr[2]] + input[idxr[4]] + input[idxr[6]]) / 4;
+                    blue = input[idx];
+                    green = (input[idxr[1]] + input[idxr[3]] + input[idxr[5]] + input[idxr[7]]) / 4;
+                }
+                else
+                {
+                    red = 0;
+                    green = 0;
+                    blue = 0;
+                }
+
+                if(grayscale)
+                {
+                    output[idx] = 0.33*red + 0.33*green + 0.33*blue;
+
+                }else
+                {
+                    output[idx * 3] = red;
+                    output[idx * 3 + 1] = green;
+                    output[idx * 3 + 2] = blue;
+                }
+
+
+            }
+        }
+    }
+}