Test acutal model (openvinotoolkit#22841)

Ticket 25942

.github/workflows/job_samples_tests.yml

@@ -124,7 +124,7 @@ jobs:
 
           source ${INSTALL_DIR}/setupvars.sh
 
-          PYTHONCOERCECLOCALE=warn python3 -bb -W error -X dev -X warn_default_encoding -m pytest $INSTALL_TEST_DIR/smoke_tests \
+          PYTHONCOERCECLOCALE=warn python3 -bb -W error -X dev -m pytest $INSTALL_TEST_DIR/smoke_tests \
             --junitxml=$INSTALL_TEST_DIR/TEST-SamplesSmokeTests.xml
 
       - name: Upload Test Results

samples/cpp/classification_sample_async/main.cpp

@@ -30,16 +30,9 @@
 #include "classification_sample_async.h"
 // clang-format on
 
-constexpr auto N_TOP_RESULTS = 10;
-
 using namespace ov::preprocess;
 
-/**
- * @brief Checks input args
- * @param argc number of args
- * @param argv list of input arguments
- * @return bool status true(Success) or false(Fail)
- */
+namespace {
 bool parse_and_check_command_line(int argc, char* argv[]) {
     gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
     if (FLAGS_h) {
@@ -61,6 +54,7 @@ bool parse_and_check_command_line(int argc, char* argv[]) {
 
     return true;
 }
+}  // namespace
 
 int main(int argc, char* argv[]) {
     try {
@@ -100,7 +94,7 @@ int main(int argc, char* argv[]) {
         // - precision of tensor is supposed to be 'u8'
         // - layout of data is 'NHWC'
         input_info.tensor().set_element_type(ov::element::u8).set_layout(tensor_layout);
-        // 3) Here we suppose model has 'NCHW' layout for input
+        // 3) Suppose model has 'NCHW' layout for input
         input_info.model().set_layout("NCHW");
         // 4) output() with no args assumes a model has a single result
         // - output() with no args assumes a model has a single result
@@ -222,6 +216,7 @@ int main(int argc, char* argv[]) {
         }
 
         // Prints formatted classification results
+        constexpr size_t N_TOP_RESULTS = 10;
         ClassificationResult classificationResult(output, valid_image_names, batchSize, N_TOP_RESULTS, labels);
         classificationResult.show();
     } catch (const std::exception& ex) {

samples/cpp/hello_classification/main.cpp

@@ -82,7 +82,7 @@ int tmain(int argc, tchar* argv[]) {
         // - convert layout to 'NCHW' (from 'NHWC' specified above at tensor layout)
         // - apply linear resize from tensor spatial dims to model spatial dims
         ppp.input().preprocess().resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR);
-        // 4) Here we suppose model has 'NCHW' layout for input
+        // 4) Suppose model has 'NCHW' layout for input
         ppp.input().model().set_layout("NCHW");
         // 5) Set output tensor information:
         // - precision of tensor is supposed to be 'f32'

samples/python/classification_sample_async/classification_sample_async.py

@@ -75,18 +75,7 @@ def main() -> int:
         log.error('Sample supports only single output topologies')
         return -1
 
-# --------------------------- Step 3. Set up input --------------------------------------------------------------------
-    # Read input images
-    images = [cv2.imread(image_path) for image_path in args.input]
-
-    # Resize images to model input dims
-    _, _, h, w = model.input().shape
-    resized_images = [cv2.resize(image, (w, h)) for image in images]
-
-    # Add N dimension
-    input_tensors = [np.expand_dims(image, 0) for image in resized_images]
-
-# --------------------------- Step 4. Apply preprocessing -------------------------------------------------------------
+# --------------------------- Step 3. Apply preprocessing -------------------------------------------------------------
     ppp = ov.preprocess.PrePostProcessor(model)
 
     # 1) Set input tensor information:
@@ -97,7 +86,7 @@ def main() -> int:
         .set_element_type(ov.Type.u8) \
         .set_layout(ov.Layout('NHWC'))  # noqa: N400
 
-    # 2) Here we suppose model has 'NCHW' layout for input
+    # 2) Suppose model has 'NCHW' layout for input
     ppp.input().model().set_layout(ov.Layout('NCHW'))
 
     # 3) Set output tensor information:
@@ -107,6 +96,17 @@ def main() -> int:
     # 4) Apply preprocessing modifing the original 'model'
     model = ppp.build()
 
+    # --------------------------- Step 4. Set up input --------------------------------------------------------------------
+    # Read input images
+    images = (cv2.imread(image_path) for image_path in args.input)
+
+    # Resize images to model input dims
+    _, h, w, _ = model.input().shape
+    resized_images = (cv2.resize(image, (w, h)) for image in images)
+
+    # Add N dimension
+    input_tensors = (np.expand_dims(image, 0) for image in resized_images)
+
 # --------------------------- Step 5. Loading model to the device -----------------------------------------------------
     log.info('Loading the model to the plugin')
     compiled_model = core.compile_model(model, args.device)

tests/samples_tests/smoke_tests/common/common_utils.py

@@ -27,7 +27,7 @@ def shell(cmd, env=None, cwd=None, out_format="plain"):
     else:
         cmd = " ".join(cmd)
 
-    sys.stdout.write("Running command:\n" + "".join(cmd) + "\n")
+    sys.stdout.write("Running command:\n" + " ".join(cmd) + "\n")
     p = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (stdout, stderr) = p.communicate()
     stdout = str(stdout.decode('utf-8'))

tests/samples_tests/smoke_tests/common/samples_common_test_class.py

@@ -10,6 +10,7 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 """
+import cv2
 import contextlib
 import io
 import os
@@ -55,35 +56,34 @@ def download(test_data_dir, file_path):
         with contextlib.suppress(FileExistsError, PermissionError):
             with lock_path.open('bx'):
                 if not file_path.exists():
-                    response = requests.get("https://storage.openvinotoolkit.org/repositories/openvino/ci_dependencies/test/2021.4/samples_smoke_tests_data_2021.4.zip")
-                    with zipfile.ZipFile(io.BytesIO(response.content)) as zfile:
-                        zfile.extractall(test_data_dir)
+                    if test_data_dir / 'bvlcalexnet-12.onnx' == file_path:
+                        response = requests.get("https://github.com/onnx/models/raw/main/validated/vision/classification/alexnet/model/bvlcalexnet-12.onnx?download=")
+                        with file_path.open('wb') as nfnet:
+                            nfnet.write(response.content)
+                    elif test_data_dir / 'efficientnet-lite4-11-qdq.onnx' == file_path:
+                        response = requests.get("https://github.com/onnx/models/raw/main/validated/vision/classification/efficientnet-lite4/model/efficientnet-lite4-11-qdq.onnx?download=")
+                        with file_path.open('wb') as nfnet:
+                            nfnet.write(response.content)
+                    else:
+                        response = requests.get("https://storage.openvinotoolkit.org/repositories/openvino/ci_dependencies/test/2021.4/samples_smoke_tests_data_2021.4.zip")
+                        with zipfile.ZipFile(io.BytesIO(response.content)) as zfile:
+                            zfile.extractall(test_data_dir)
+                        cv2.imwrite(str(test_data_dir / 'dog-224x224.bmp'), cv2.resize(cv2.imread(str(test_data_dir / 'samples_smoke_tests_data_2021.4/validation_set/227x227/dog.bmp')), (224, 224)))
             lock_path.unlink(missing_ok=True)
             assert file_path.exists()
             return file_path
         time.sleep(1.0)
 
 
 def prepend(cache, inp='', model=''):
-    test_data_dir = cache.mkdir('test_data_dir')
-    unpacked = test_data_dir / 'samples_smoke_tests_data_2021.4'
+    test_data_dir = cache.mkdir('test_data')
     if inp:
-        inp = '-i', download(test_data_dir, unpacked / 'validation_set' / inp)
+        inp = '-i', download(test_data_dir, test_data_dir / inp)
     if model:
-        model = '-m', download(test_data_dir, unpacked / 'models' / 'public' / model)
+        model = '-m', download(test_data_dir, test_data_dir / model)
     return *inp, *model
 
 
-class Environment:
-    """
-    Environment used by tests.
-
-    :attr env:  environment dictionary. populated dynamically from environment
-                configuration file.
-    """
-    env = {}
-
-
 def get_tests(cmd_params, use_device=True, use_batch=False):
     # Several keys:
     #         use_device
@@ -171,7 +171,7 @@ def made_executable_path(cls, path1, path2, sample_type):
 
     @staticmethod
     def join_env_path(param, cache, executable_path, complete_path=True):
-        test_data_dir = cache.mkdir('test_data_dir')
+        test_data_dir = cache.mkdir('test_data')
         unpacked = test_data_dir / 'samples_smoke_tests_data_2021.4'
         if 'i' in param:
             # If batch > 1, then concatenate images
@@ -181,10 +181,10 @@ def join_env_path(param, cache, executable_path, complete_path=True):
                 param['i'] = list([param['i']])
         for k in param.keys():
             if ('i' == k) and complete_path:
-                param['i'] = [str(download(test_data_dir, unpacked / 'validation_set' / e)) for e in param['i']]
+                param['i'] = [str(download(test_data_dir, test_data_dir / e)) for e in param['i']]
                 param['i'] = ' '.join(map(str, param['i']))
             elif 'm' == k and not param['m'].endswith('/samples/cpp/model_creation_sample/lenet.bin"'):
-                param['m'] = download(test_data_dir, unpacked / 'models' / 'public' / param['m'])
+                param['m'] = download(test_data_dir, test_data_dir / param['m'])
 
     @staticmethod
     def get_cmd_line(param, use_preffix=True, long_hyphen=None):

tests/samples_tests/smoke_tests/test_benchmark_app.py

@@ -26,10 +26,10 @@ def get_executable(sample_language):
     return 'benchmark_app'
 
 
-def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape=None, nstreams=None, layout=None, pin=None, cache=None, tmp_path=None):
+def verify(sample_language, device, api=None, nireq=None, shape=None, data_shape=None, nstreams=None, layout=None, pin=None, cache=None, tmp_path=None, model='bvlcalexnet-12.onnx'):
     output = get_cmd_output(
         get_executable(sample_language),
-        *prepend(cache, '227x227/dog.bmp', 'squeezenet1.1/FP32/squeezenet1.1.xml'),
+        *prepend(cache, 'dog-224x224.bmp', model),
         *('-nstreams', nstreams) if nstreams else '',
         *('-layout', layout) if layout else '',
         *('-nireq', nireq) if nireq else '',
@@ -92,10 +92,10 @@ def test_api(sample_language, api, device, cache, tmp_path):
 @pytest.mark.parametrize('sample_language', ['C++', 'Python'])
 @pytest.mark.parametrize('device', get_devices())
 def test_reshape(sample_language, device, cache):
-    verify(sample_language, device, shape='data[2,3,227,227]', cache=cache)
+    verify(sample_language, device, shape='data_0[2,3,224,224]', cache=cache)
 
 
 @pytest.mark.parametrize('sample_language', ['C++', 'Python'])
 @pytest.mark.parametrize('device', get_devices())
 def test_dynamic_shape(sample_language, device, cache):
-    verify(sample_language, device, shape='[?,3,?,?]', data_shape='[1,3,227,227][1,3,227,227]', layout='[NCHW]', cache=cache)
+    verify(sample_language, device, model='efficientnet-lite4-11-qdq.onnx', shape='[?,224,224,3]', data_shape='[1,224,224,3][2,224,224,3]', layout='[NHWC]', cache=cache)

tests/samples_tests/smoke_tests/test_classification_sample_async.py

@@ -10,7 +10,6 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 """
-import os
 import pytest
 import re
 import sys
@@ -20,19 +19,17 @@
 
 log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
 
-test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
-                                       'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
-                                       'sample_type': ['C++','Python'],
-                                       'batch': [1, 2, 4],
-                                      },
-                           )
+test_data_fp32 = get_tests({
+    'i': ['dog-224x224.bmp'],
+    'm': ['bvlcalexnet-12.onnx'],  # Remove the model forom .md and .rst if removed from here
+    'sample_type': ['C++','Python'],
+})
 
-test_data_fp16 = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
-                                       'm': [os.path.join('squeezenet1.1', 'FP16', 'squeezenet1.1.xml')],
-                                       'sample_type': ['C++','Python'],
-                                       'batch': [1, 2, 4],
-                                       },
-                           )
+test_data_fp16 = get_tests({
+    'i': ['dog-224x224.bmp'],
+    'm': ['bvlcalexnet-12.onnx'],
+    'sample_type': ['C++','Python'],
+})
 
 
 class TestClassification(SamplesCommonTestClass):

tests/samples_tests/smoke_tests/test_hello_classification.py

@@ -11,30 +11,41 @@
  limitations under the License.
 """
 import os
+import pathlib
 import pytest
 import re
 import sys
 import logging as log
-from common.samples_common_test_class import get_cmd_output, get_tests
+from common.samples_common_test_class import get_cmd_output, get_tests, download
 from common.samples_common_test_class import SamplesCommonTestClass
 from common.common_utils import shell
 from pathlib import Path
 import shutil
 
-log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
 
-test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
-                                       'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
-                                       'sample_type': ['C++', 'C']},
-                          )
+def get_executable(sample_language):
+    return pathlib.Path(os.environ['IE_APP_PATH'], 'hello_classification' + ('' if sample_language == 'C++' else '_c')).with_suffix('.exe' if os.name == 'nt' else '')
 
-test_data_fp32_unicode = get_tests(cmd_params={'i': [os.path.join('227x227', 'dog.bmp')],
-                                               'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
-                                               'sample_type': ['C++']},
-                                  )
 
+def prepend(cache, model, inp):
+    test_data_dir = cache.mkdir('test_data')
+    return download(test_data_dir, test_data_dir / model), download(test_data_dir, test_data_dir / inp)
 
-class TestHello(SamplesCommonTestClass):
+
+test_data_fp32 = get_tests({
+    'i': ['dog-224x224.bmp'],
+    'm': ['bvlcalexnet-12.onnx'],  # Remove the model forom .md and .rst if removed from here
+    'sample_type': ['C++', 'C'],
+})
+
+test_data_fp32_unicode = get_tests({
+    'i': ['dog-224x224.bmp'],
+    'm': ['bvlcalexnet-12.onnx'],
+    'sample_type': ['C++'],
+})
+
+
+class Test_hello_classification(SamplesCommonTestClass):
     sample_name = 'hello_classification'
 
     @pytest.mark.parametrize("param", test_data_fp32)
@@ -46,20 +57,18 @@ def test_hello_classification_fp32(self, param, cache):
         """
 
         # Run _test function, that returns stdout or 0.
-        stdout = self._test(param, cache, use_preffix=False, get_cmd_func=self.get_hello_cmd_line)
-        if not stdout:
-            return 0
+        output = get_cmd_output(get_executable(param['sample_type']), *prepend(cache, param['m'], param['i']), param['d'])
 
-        stdout = stdout.split('\n')
+        output = output.split('\n')
 
         is_ok = True
-        for line in range(len(stdout)):
-            if re.match('\\d+ +\\d+.\\d+$', stdout[line].replace('[ INFO ]', '').strip()) is not None:
-                top1 = stdout[line].replace('[ INFO ]', '').strip().split(' ')[0]
+        for line in range(len(output)):
+            if re.match('\\d+ +\\d+.\\d+$', output[line].replace('[ INFO ]', '').strip()) is not None:
+                top1 = output[line].replace('[ INFO ]', '').strip().split(' ')[0]
                 top1 = re.sub('\\D', '', top1)
                 if '215' not in top1:
                     is_ok = False
-                    log.error('Expected class 215, Detected class {}'.format(top1))
+                    log.error('Expected class 262, Detected class {}'.format(top1))
                 break
         assert is_ok, 'Wrong top1 class'
         log.info('Accuracy passed')
@@ -79,11 +88,10 @@ def test_hello_classification_check_unicode_path_support(self, param, cache, tmp
         tmp_image_dir.mkdir()
         tmp_model_dir.mkdir()
 
-        test_data_dir = cache.makedir('test_data_dir') / 'samples_smoke_tests_data_2021.4'
+        test_data_dir = cache.makedir('test_data')
         # Copy files
-        shutil.copy(test_data_dir / 'validation_set' / param['i'], tmp_image_dir)
-        shutil.copy(test_data_dir / 'models' / 'public' / param['m'], tmp_model_dir)
-        shutil.copy(test_data_dir / 'models' / 'public' / Path(param['m'].replace('.xml', '.bin')), tmp_model_dir)
+        shutil.copy(test_data_dir / param['i'], tmp_image_dir)
+        shutil.copy(test_data_dir / param['m'], tmp_model_dir)
 
         image_path = tmp_image_dir / Path(param['i']).name
         original_image_name = image_path.name.split(sep='.')[0]
@@ -130,7 +138,6 @@ def test_hello_classification_check_unicode_path_support(self, param, cache, tmp
 
                 new_model_path = tmp_model_dir / (original_model_name + f"_{model_name.decode('utf-8')}.xml")
                 model_path.rename(new_model_path)
-                Path(str(model_path).replace('.xml', '.bin')).rename(Path(str(new_model_path).replace('.xml', '.bin')))
                 stdout = get_cmd_output(executable_path, new_model_path, image_path, param['d'])
                 probs = []
                 for line in stdout.split(sep='\n'):

tests/samples_tests/smoke_tests/test_hello_nv12_input_classification.py

@@ -10,7 +10,6 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 """
-import os
 import pytest
 import re
 import sys
@@ -20,12 +19,12 @@
 
 log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
 
-test_data_fp32 = get_tests(cmd_params={'i': [os.path.join('224x224', 'dog6.yuv')],
-                                       'm': [os.path.join('squeezenet1.1', 'FP32', 'squeezenet1.1.xml')],
-                                       'size': ['224x224'],
-				       'sample_type': ['C++', 'C'],
-                                       },
-                           )
+test_data_fp32 = get_tests({
+    'i': ['samples_smoke_tests_data_2021.4/validation_set/224x224/dog6.yuv'],
+    'm': ['bvlcalexnet-12.onnx'],  # Remove the model forom .md and .rst if removed from here
+    'size': ['224x224'],
+    'sample_type': ['C++', 'C'],
+})
 
 class TestHelloNV12Input(SamplesCommonTestClass):
     sample_name = 'hello_nv12_input_classification'