Add Regression Test

openvinotoolkit · Oct 24, 2023 · e855193 · e855193
1 parent fa5360d
commit e855193
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml
@@ -170,6 +170,7 @@ jobs:
       source .env3/bin/activate
       python -m pip install --upgrade pip
       python -m pip install -r $(REPO_DIR)/modules/custom_operations/tests/requirements.txt
+      python -m pip install -r $(REPO_DIR)/modules/custom_operations/user_ie_extensions/tokenizer/python/.[all]
       cd ${OPENVINO_REPO_DIR}/tools && python -m pip install mo/
     workingDirectory: $(WORK_DIR)
     displayName: 'Create user custom operations env'
@@ -181,3 +182,10 @@ jobs:
       python -m pytest -k "not sparse_conv" tests/run_tests.py
     workingDirectory: $(REPO_DIR)/modules/custom_operations
     displayName: 'Custom user operation tests'
+
+  - script: |
+      . $(SETUPVARS)
+      source  $(WORK_DIR)/.env3/bin/activate
+      python -m pytest user_ie_extensions/tokenizer/python/tests/tokenizers_test.py
+    workingDirectory: $(WORK_DIR)
+    displayName: 'Tokenizers extension regression test'
diff --git a/modules/custom_operations/user_ie_extensions/tokenizer/python/tests/conftest.py b/modules/custom_operations/user_ie_extensions/tokenizer/python/tests/conftest.py
@@ -1,6 +1,7 @@
 import json
 import os
 from math import isclose
+from pathlib import Path
 
 import pytest
 
@@ -16,7 +17,7 @@ def prebuild_extenson_path():
 
 
 os.environ["OV_TOKENIZER_PREBUILD_EXTENSION_PATH"] = prebuild_extenson_path()
-PASS_RATES_FILE = "pass_rates.json"
+PASS_RATES_FILE = Path(__file__).parent / "pass_rates.json"
 
 
 @pytest.hookimpl(trylast=True)
@@ -36,6 +37,7 @@ def pytest_sessionfinish(session, exitstatus) -> None:
     previous = previous_rates.get(parent, 0)
 
     if isclose(pass_rate, previous):
+        session.exitstatus = pytest.ExitCode.OK
         return
 
     if pass_rate > previous:

diff --git a/modules/custom_operations/user_ie_extensions/tokenizer/python/tests/pass_rates.json b/modules/custom_operations/user_ie_extensions/tokenizer/python/tests/pass_rates.json
@@ -5,5 +5,6 @@
     "tokenizers_test.py::test_sentencepiece_model_detokenizer": 0.5458333333333334,
     "tokenizers_test.py::test_hf_bpe_tokenizers_outputs": 0.846875,
     "tokenizers_test.py::test_bpe_detokenizer": 0.93125,
-    "tokenizers_test.py::test_": 0.7512332628611699
+    "tokenizers_test.py::test_": 0.7512332628611699,
+    "user_ie_extensions/tokenizer/python/tests/tokenizers_test.py::test_": 0.7512332628611699
 }