From fcb8c138805e18c88110010ad2f2651b72a539cc Mon Sep 17 00:00:00 2001
From: Mikhail Ryzhov <mikhail.ryzhov@intel.com>
Date: Thu, 21 Dec 2023 15:15:42 +0100
Subject: [PATCH 1/2] added build tokenizers for wheel

---
 .ci/azure/linux.yml                                       | 3 +++
 .../user_ie_extensions/tokenizer/CMakeLists.txt           | 8 +++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml
index f9718eab2..02b02e290 100644
--- a/.ci/azure/linux.yml
+++ b/.ci/azure/linux.yml
@@ -186,6 +186,9 @@ jobs:
       source $(WORK_DIR)/.env3/bin/activate && source $(SETUPVARS)
       python -m pip install build
       python -m build --wheel --outdir $(BUILD_WHEEL_DIR) $(REPO_DIR)/modules/custom_operations
+    env:
+      CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=ON',
+      CMAKE_GENERATOR: 'Unix Makefiles'
     workingDirectory: $(WORK_DIR)
     displayName: 'Build tokenizers wheel'
 
diff --git a/modules/custom_operations/user_ie_extensions/tokenizer/CMakeLists.txt b/modules/custom_operations/user_ie_extensions/tokenizer/CMakeLists.txt
index 8689c7a8a..9ecf80ea7 100644
--- a/modules/custom_operations/user_ie_extensions/tokenizer/CMakeLists.txt
+++ b/modules/custom_operations/user_ie_extensions/tokenizer/CMakeLists.txt
@@ -8,8 +8,6 @@ endif()
 
 option(BUILD_FAST_TOKENIZERS OFF)
 
-# to build only direct dependencies
-set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON)
 
 #
 # Compile flags
@@ -168,8 +166,12 @@ set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_OPTIONS "${extra_flags}"
 # Post build steps to copy core_tokenizers dependencies
 #
 
+set_property(DIRECTORY ${sentencepiece_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON)
+
 if(BUILD_FAST_TOKENIZERS)
-  # TODO
+  install(TARGETS core_tokenizers
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 else()
   if(WIN32 AND X86_64)
     set(extra_libs "${fast_tokenizer_SOURCE_DIR}/lib/core_tokenizers.dll"

From 339b7716ae0ce70436999a8b205101e17f94ad43 Mon Sep 17 00:00:00 2001
From: Mikhail Ryzhov <mikhail.ryzhov@intel.com>
Date: Thu, 21 Dec 2023 16:21:02 +0100
Subject: [PATCH 2/2] fixed azure pipeline

---
 .ci/azure/linux.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml
index 02b02e290..1654304b9 100644
--- a/.ci/azure/linux.yml
+++ b/.ci/azure/linux.yml
@@ -187,7 +187,7 @@ jobs:
       python -m pip install build
       python -m build --wheel --outdir $(BUILD_WHEEL_DIR) $(REPO_DIR)/modules/custom_operations
     env:
-      CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=ON',
+      CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=ON'
       CMAKE_GENERATOR: 'Unix Makefiles'
     workingDirectory: $(WORK_DIR)
     displayName: 'Build tokenizers wheel'