[Tokenizers] Added support for conda-forge, archive installations (#782)

* Supported different installation methods like archives, conda-forge

* Update CMakeLists.txt

renamed component core => tokenizers

modules/custom_operations/pyproject.toml

@@ -52,9 +52,10 @@ lines-after-imports = 2
 [tool.scikit-build]
 cmake.minimum-version = "3.15"
 cmake.build-type = "Release"
-cmake.args = ["-DCUSTOM_OPERATIONS:STRING=tokenizer", "-DBUILD_FAST_TOKENIZERS=OFF"]
+cmake.args = ["-DCUSTOM_OPERATIONS=tokenizer", "-DCMAKE_INSTALL_BINDIR=lib"]
+cmake.targets = ["user_ov_extensions"]
 wheel.packages = ["user_ie_extensions/tokenizer/python/openvino_tokenizers"]
-wheel.install-dir = "openvino_tokenizers/libs"
+wheel.install-dir = "openvino_tokenizers"
 wheel.py-api = "py3"
 wheel.license-files = ['../../LICENSE', '../../third-party-programs.txt', '../../SECURITY.md']
 sdist.cmake = true

modules/custom_operations/user_ie_extensions/CMakeLists.txt

@@ -109,23 +109,29 @@ target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_A
 # TODO: remove
 target_include_directories(${TARGET_NAME} PUBLIC ./include/)
 
-# Wheel packaging using skbuild
-if(DEFINED SKBUILD)
-  # Installing the extension module to the root of the package
-  if(LINUX)
-    set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN")
-    install(TARGETS ${TARGET_NAME} LIBRARY DESTINATION .)
-  elseif(APPLE)
-    set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "@loader_path")
-    install(TARGETS ${TARGET_NAME} LIBRARY DESTINATION .)
-  elseif(WIN32 AND X86_64)
-    install(TARGETS ${TARGET_NAME} RUNTIME DESTINATION .)
-  else()
-    message(FATAL_ERROR "Unsupported build platform")
-  endif()
+#
+# Installation rules
+#
 
-  if(extra_libs)
-    install(FILES ${extra_libs} DESTINATION .)
-  endif()
+include(GNUInstallDirs)
 
+# setting RPATH / LC_RPATH depending on platform
+if(LINUX)
+  set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN")
+elseif(APPLE)
+  set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "@loader_path")
+endif()
+
+# Installing the extension module to the root of the package
+install(TARGETS ${TARGET_NAME}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT tokenizers
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT tokenizers)
+
+if(extra_libs)
+  if(WIN32)
+    set(extra_libs_location ${CMAKE_INSTALL_BINDIR})
+  else()
+    set(extra_libs_location ${CMAKE_INSTALL_LIBDIR})
+  endif()
+  install(FILES ${extra_libs} DESTINATION ${extra_libs_location} COMPONENT tokenizers)
 endif()

modules/custom_operations/user_ie_extensions/tokenizer/CMakeLists.txt

@@ -120,8 +120,13 @@ else()
     message(FATAL_ERROR "Platform ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} does not have prebuilt Fast Tokenizer"
                         "Please, use -DBUILD_FAST_TOKENIZERS=ON cmake option to enable build from soures")
   endif()
+
   FetchContent_MakeAvailable(fast_tokenizer)
+  # to allow find_library to work with conda-forge env
+  set(_old_CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ${CMAKE_FIND_ROOT_PATH_MODE_LIBRARY})
+  set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER)
   include("${fast_tokenizer_SOURCE_DIR}/FastTokenizer.cmake")
+  set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ${_old_CMAKE_FIND_ROOT_PATH_MODE_LIBRARY})
 
   set(fast_tokenizer_SOURCE_DIR "${fast_tokenizer_SOURCE_DIR}" PARENT_SCOPE)
 
@@ -168,8 +173,8 @@ if(BUILD_FAST_TOKENIZERS)
 else()
   if(WIN32 AND X86_64)
     set(extra_libs "${fast_tokenizer_SOURCE_DIR}/lib/core_tokenizers.dll"
-                  "${fast_tokenizer_SOURCE_DIR}/third_party/lib/icudt70.dll"
-                  "${fast_tokenizer_SOURCE_DIR}/third_party/lib/icuuc70.dll" PARENT_SCOPE)
+                   "${fast_tokenizer_SOURCE_DIR}/third_party/lib/icudt70.dll"
+                   "${fast_tokenizer_SOURCE_DIR}/third_party/lib/icuuc70.dll" PARENT_SCOPE)
   elseif(LINUX)
     set(extra_libs "${fast_tokenizer_SOURCE_DIR}/lib/libcore_tokenizers.so" PARENT_SCOPE)
   elseif(APPLE)

modules/custom_operations/user_ie_extensions/tokenizer/python/openvino_tokenizers/__init__.py

@@ -16,38 +16,42 @@
 
 
 _extension_path = os.environ.get("OV_TOKENIZER_PREBUILD_EXTENSION_PATH")
-_ext_name = "user_ov_extensions"
 if _extension_path:
-    # when the path to extension set manually
+    # when the path to the extension set manually
     _ext_libs_path = Path(_extension_path).parent
 else:
     # python installation case
-    _ext_libs_path = Path(sysconfig.get_paths()["purelib"]) / __name__ / "libs"
+    _ext_libs_path = Path(sysconfig.get_paths()["purelib"]) / __name__ / "lib"
 
+_ext_name = "user_ov_extensions"
 if sys.platform == "win32":
-    _ext_path = _ext_libs_path / f"{_ext_name}.dll"
-    if _ext_libs_path.is_dir():
-        # On Windows, with Python >= 3.8, DLLs are no longer imported from the PATH.
-        os.add_dll_directory(str(_ext_libs_path.absolute()))
-    else:
-        sys.exit(f"Error: extention libriary path {_ext_libs_path} not found")
+    _ext_name = f"{_ext_name}.dll"
 elif sys.platform == "darwin":
-    _ext_path = _ext_libs_path / f"lib{_ext_name}.dylib"
+    _ext_name = f"lib{_ext_name}.dylib"
 elif sys.platform == "linux":
-    _ext_path = _ext_libs_path / f"lib{_ext_name}.so"
+    _ext_name = f"lib{_ext_name}.so"
+else:
+    sys.exit(f"Error: extension does not support the platform {sys.platform}")
+
+# conda-forge case
+conda_forge_lib_path = Path(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+if os.path.exists(conda_forge_lib_path / _ext_name):
+    _ext_path = conda_forge_lib_path / _ext_name
 else:
-    sys.exit(f"Error: extension does not support platform {sys.platform}")
+    _ext_path = _ext_libs_path / _ext_name
+
+del _ext_name
+del _ext_libs_path
+del conda_forge_lib_path
 
 # patching openvino
 old_core_init = openvino.runtime.Core.__init__
 
-
 @functools.wraps(old_core_init)
 def new_core_init(self, *args, **kwargs):
     old_core_init(self, *args, **kwargs)
     self.add_extension(str(_ext_path))  # Core.add_extension doesn't support Path object
 
-
 openvino.runtime.Core.__init__ = new_core_init
 
 _factory = NodeFactory()