Added demo for ONNX. And fix accuracy-checker for ONNX

itlab-vision · Sep 25, 2024 · a845190 · a845190
1 parent d584de9
commit a845190
Show file tree

Hide file tree

Showing 7 changed files with 144 additions and 14 deletions.
diff --git a/demo/accuracy_checker_configs/ONNXRuntime.xml b/demo/accuracy_checker_configs/ONNXRuntime.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Tests>
+    <Test>
+        <Model>
+            <Task>classification</Task>
+            <Name>SampleNet</Name>
+            <Precision>FP32</Precision>
+            <SourceFramework>ONNXRuntime</SourceFramework>
+            <Directory>/media/models</Directory>
+        </Model>
+        <Parameters>
+            <InferenceFramework>ONNXRuntime</InferenceFramework>
+            <Device>CPU</Device>
+            <Config>{CONFIG_PATH}</Config>
+        </Parameters>
+    </Test>
+</Tests>
diff --git a/demo/accuracy_checker_configs/ONNXRuntime.yml b/demo/accuracy_checker_configs/ONNXRuntime.yml
@@ -0,0 +1,34 @@
+models:
+  - name: SampLeNet_example
+    launchers:
+      - framework: onnx_runtime
+        device: CPU
+        model:   samplenet.onnx
+        adapter: classification
+        batch: 1
+        inputs:
+          - type: INPUT
+            name: data
+            layout: NCHW
+
+    datasets:
+      - name: sample_dataset
+        data_source: sample_dataset/test
+        annotation_conversion:
+          converter: cifar
+          data_batch_file: cifar-10-batches-py/test_batch
+          convert_images: True
+          converted_images_dir: sample_dataset/test
+          num_classes: 10
+
+        preprocessing:
+          - type: resize
+            size: 32
+          - type: bgr_to_rgb
+          - type: normalization
+            mean: (125.307, 122.961, 113.8575)
+            std: (51.5865, 50.847, 51.255)
+
+        metrics:
+          - type: accuracy
+            top_k: 1
diff --git a/demo/benchmark_configs/ONNXRuntime.xml b/demo/benchmark_configs/ONNXRuntime.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Tests>
+    <Test>
+        <Model>
+            <Task>classification</Task>
+            <Name>SampleNet</Name>
+            <Precision>FP32</Precision>
+            <SourceFramework>ONNXRuntime</SourceFramework>
+            <ModelPath>/media/models/samplenet.onnx</ModelPath>
+            <WeightsPath>/media/models/samplenet.onnx</WeightsPath>
+        </Model>
+        <Dataset>
+            <Name>ImageNET</Name>
+            <Path>/tmp/{DLI_DATASET_REPO_NAME}/Datasets/ImageNET/</Path>
+        </Dataset>
+        <FrameworkIndependent>
+            <InferenceFramework>ONNX Runtime Python</InferenceFramework>
+            <BatchSize>1</BatchSize>
+            <Device>CPU</Device>
+            <IterationCount>10</IterationCount>
+            <TestTimeLimit>0</TestTimeLimit>
+        </FrameworkIndependent>
+        <FrameworkDependent>
+            <ChannelSwap></ChannelSwap>
+            <Mean></Mean>
+            <InputScale></InputScale>
+            <InputShape></InputShape>
+            <InputName></InputName>
+            <Layout></Layout>
+            <ExecutionProviders></ExecutionProviders>
+            <ThreadCount></ThreadCount>
+            <InterThreadCount></InterThreadCount>
+            <ExecutionMode></ExecutionMode>
+        </FrameworkDependent>
+    </Test>
+</Tests>
diff --git a/demo/linux.sh b/demo/linux.sh
@@ -1,14 +1,51 @@
 #!/bin/bash
 
-while getopts l:p:d: flag
+supported_frameworks="OpenVINO_DLDT ONNXRuntime"
+
+usage() {
+    echo "Usage: $0 [-l LOGIN] [-p PASSWORD] [-f FRAMEWORK] [-d GIT_LINK_TO_DATASET]"
+    echo "Options:"
+    echo "  -l          Login of the current user."
+    echo "  -p          Password of the current user."
+    echo "  -f          Framework (supported: $supported_frameworks)."
+    echo "  -d          The address to the GitHub repository, which contains datasets for benchmarking."
+    echo "              It is required that the Databases/ImageNet/ directory be created"
+    echo "              in the repository, which stores at least one image."
+}
+
+exit_abnormal() {
+    usage
+    exit 1
+}
+
+
+while getopts :l:p:d:f: flag;
 do
     case "${flag}" in
-        l) login=${OPTARG};;
-        p) password=${OPTARG};;
-        d) benchmark_datasets=${OPTARG};;
+        l)  login=${OPTARG}
+            ;;
+        p)  password=${OPTARG}
+            ;;
+        d)  benchmark_datasets=${OPTARG}
+            ;;
+        f)  framework=${OPTARG}
+            ;;
+        :)  echo "Error: -${OPTARG} requires an argument."
+            exit_abnormal
+      ;;
     esac
 done
 
+if [[(-z $login) || (-z $password) || (-z $benchmark_datasets) || (-z $framework)]]; then
+    echo "One or more of required parameters is not specified."
+    exit_abnormal
+fi
+
+if [[ ! " $supported_frameworks " =~ " $framework " ]]; then
+    echo "Framework '$framework' is not supported."
+    exit_abnormal
+fi
+
 
 echo "[ INFO ] Demo application has been started"
 demo_folder="$PWD"
@@ -84,15 +121,19 @@ echo "[ INFO ] The name of repository with datasets is $dli_dataset_repo_name"
 echo "[ INFO ] Build a base image has been started"
 docker build -t ubuntu_for_dli --build-arg DATASET_DOWNLOAD_LINK=$benchmark_datasets .
 echo "[ INFO ] Build a base image has been completed"
-
-cd ./OpenVINO_DLDT
-image_name="openvino_${openvino_version}"
+cd ./$framework
+if [ "$framework" = "OpenVINO_DLDT" ]; then
+    docker_name="openvino_${openvino_version}"
+else
+    docker_name=${framework,,}
+fi
+image_name=${docker_name}
 echo "[ INFO ] Build a $image_name image has been started"
 docker build -t $image_name .
 echo "[ INFO ] Build a $image_name image has been completed"
 
 echo "[ INFO ] Creation of archive with Docker image"
-archive_name="openvino_${openvino_version}.tar"
+archive_name="$docker_name.tar"
 docker save $image_name -o $archive_name
 archive_path="$PWD/$archive_name"
 echo "[ INFO ] Archive ${archive_path} has been created"
@@ -125,18 +166,17 @@ cd $dlb_server/src/deployment
 $PYTHON deploy.py -s $ip_address -l $login -p itmm \
                      -i $archive_path \
                      -d $server_folder \
-                     -n OpenVINO_DLDT \
+                     -n $framework \
                      --machine_list $deployment_config \
                      --project_folder $dlb_client
 echo "[ INFO ] Deployment of DLI Benchmark system has been completed"
 
-
 echo "[ INFO ] Preparing configuration for benchmarking"
 cd $demo_folder
 benchmark_config="benchmark_config.xml"
 benchmark_config_path="${PWD}/${benchmark_config}"
 [ -f $benchmark_config_path ] && rm -rf $benchmark_config_path
-template_benchmark_config="benchmark_configs/OpenVINO_DLDT.xml"
+template_benchmark_config="benchmark_configs/${framework}.xml"
 echo "[ INFO ] Using template config file ${template_benchmark_config}"
 sed "s@{DLI_DATASET_REPO_NAME}@$dli_dataset_repo_name@g" $template_benchmark_config > $benchmark_config_path
 echo "[ INFO ] Copying of benchmark configuration file ${benchmark_config_path} to server"
@@ -150,8 +190,8 @@ echo "[ INFO ] Preparing configuration for accuracy checker"
 accuracy_checker_config="accuracy_checker_config.xml"
 accuracy_checker_config_path="${PWD}/${accuracy_checker_config}"
 [ -f $accuracy_checker_config_path ] && rm -rf $accuracy_checker_config_path
-config_path="${PWD}/accuracy_checker_configs/OpenVINO_DLDT.yml"
-template_accuracy_checker_config="accuracy_checker_configs/OpenVINO_DLDT.xml"
+config_path="${PWD}/accuracy_checker_configs/${framework}.yml"
+template_accuracy_checker_config="accuracy_checker_configs/${framework}.xml"
 echo "[ INFO ] Using template config file ${template_accuracy_checker_config}"
 sed "s@{CONFIG_PATH}@$config_path@g" $template_accuracy_checker_config > $accuracy_checker_config_path
 echo "[ INFO ] Copying of accuracy checker configuration ${accuracy_checker_config_path} file to server"

diff --git a/docker/ONNXRuntime/Dockerfile b/docker/ONNXRuntime/Dockerfile
@@ -48,6 +48,7 @@ RUN pip install onnxruntime==${ONNX_VERSION} && pip install onnx==${ONNX_VERSION
 
 # accuracy-check
 WORKDIR /tmp/open_model_zoo/tools/accuracy_checker
+RUN python3 setup.py install_core
 RUN accuracy_check -c /tmp/dl-benchmark/docker/ONNXRuntime/config.yml -m data/test_models -s sample
 
 WORKDIR /tmp/
diff --git a/src/accuracy_checker/process.py b/src/accuracy_checker/process.py
@@ -11,7 +11,7 @@ def __init__(self, log, executor, test):
         self._status = None
         self.supported_frameworks = {'OpenVINO DLDT': 'dlsdk', 'Caffe': 'caffe', 'TensorFlow': 'tf',
                                      'TensorFlow_Lite': 'tf_lite', 'MXNet': 'mxnet', 'OpenCV': 'opencv',
-                                     'PyTorch': 'pytorch', 'TVM': 'tvm'}
+                                     'PyTorch': 'pytorch', 'TVM': 'tvm', 'ONNXRuntime': 'onnx_runtime'}
         self.csv_name = executor.get_csv_file()
 
     @staticmethod

diff --git a/src/benchmark/executors.py b/src/benchmark/executors.py
@@ -24,6 +24,8 @@ def get_executor(executor_type, log):
         raise ValueError('Executor type must be from list: host_machine, docker_container')
 
     def set_target_framework(self, target_framework):
+        if target_framework == 'ONNX Runtime Python' or target_framework == 'ONNX Runtime CPP':
+            target_framework = 'ONNXRuntime'
         self.target_framework = target_framework.replace(' ', '_')
 
     @abc.abstractmethod