This repository has been archived by the owner on Aug 2, 2022. It is now read-only.
Adding support for FAISS JNI #285
+1,591
−297
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,6 @@ | ||
| [submodule "jni/external/nmslib"] | ||
| path = jni/external/nmslib | ||
| url = https://github.com/nmslib/nmslib.git | ||
| [submodule "jniFaiss/external/faiss"] | ||
| path = jniFaiss/external/faiss | ||
| url = https://github.com/facebookresearch/faiss |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| cmake_minimum_required(VERSION 2.8) | ||
|
There was a problem hiding this comment. Design question: Should we have separate JNI libraries for FAISS and nmslib, or should they be contained in one? There was a problem hiding this comment. i am not sure which is better. There was a problem hiding this comment. I feel having seperate JNI would be more cleaner and easy to abstract out the underlying business logic to dedicated files. I like the current approach. There was a problem hiding this comment. I think having 2 separate libraries is okay. |
||
|
|
||
| project(KNNIndexVFaiss) | ||
|
|
||
| # Corner case. For CMake 2.8, there is no option to specify set(CMAKE_CXX_STANDARD 11). Instead, the flag manually needs | ||
| # to be set. | ||
| if (CMAKE_VERSION VERSION_LESS "3.1") | ||
| set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") | ||
| else() | ||
| set(CMAKE_CXX_STANDARD 11) | ||
| set(CMAKE_CXX_STANDARD_REQUIRED True) | ||
| endif() | ||
|
|
||
| # Target Library to be built | ||
| set(KNN_INDEX KNNIndexVFaiss) | ||
| set(KNN_PACKAGE_NAME opendistro-knnlib) | ||
|
|
||
| # --- NMSLIB BEGIN --- | ||
| # Check if similarity search exists | ||
| #find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) | ||
|
|
||
| # If not, pull the updated submodule | ||
| #if (NOT EXISTS ${NMS_REPO_DIR}) | ||
| # message(STATUS "Could not find nmslib. Pulling updated submodule.") | ||
| # execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) | ||
| #endif () | ||
|
|
||
| # Add the subdirectory so it is possible to use its targets | ||
| #add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search EXCLUDE_FROM_ALL) | ||
| # --- NMSLIB END --- | ||
|
|
||
| # --- FAISS BEGIN --- | ||
| # Check if faiss search exists | ||
| find_path(NMS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) | ||
|
|
||
| # If not, pull the updated submodule | ||
| if (NOT EXISTS ${NMS_REPO_DIR}) | ||
| message(STATUS "Could not find faiss. Pulling updated submodule.") | ||
| execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) | ||
| endif () | ||
|
|
||
| include_directories(${CMAKE_CURRENT_SOURCE_DIR}/external/faiss/) | ||
| set(FIASSLIB ${CMAKE_CURRENT_SOURCE_DIR}/external/libfaiss.a) | ||
|
|
||
| set(CMAKE_CXX_STANDARD_REQUIRED ON) | ||
| set(CMAKE_C_COMPILER "/usr/bin/gcc") | ||
| set(CMAKE_CXX_COMPILER "/usr/bin/g++") | ||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_RELEASE} -fpermissive -std=c++11 -Wall -pthread -O3 -g") | ||
| find_package(ZLIB REQUIRED) | ||
| find_package(BLAS REQUIRED) | ||
| find_package(LAPACK REQUIRED) | ||
|
|
||
| # ---- OPENMP BEGIN ---- | ||
| OPTION (USE_OpenMP "Use OpenMP to enamble <omp.h>" ON) | ||
|
|
||
| # Find OpenMP | ||
| if(APPLE AND USE_OpenMP) | ||
| if(CMAKE_C_COMPILER_ID MATCHES "Clang") | ||
| set(OpenMP_C "${CMAKE_C_COMPILER}") | ||
| set(OpenMP_C_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument") | ||
| set(OpenMP_C_LIB_NAMES "libomp" "libgomp" "libiomp5") | ||
| set(OpenMP_libomp_LIBRARY ${OpenMP_C_LIB_NAMES}) | ||
| set(OpenMP_libgomp_LIBRARY ${OpenMP_C_LIB_NAMES}) | ||
| set(OpenMP_libiomp5_LIBRARY ${OpenMP_C_LIB_NAMES}) | ||
| endif() | ||
| if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") | ||
| set(OpenMP_CXX "${CMAKE_CXX_COMPILER}") | ||
| set(OpenMP_CXX_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument") | ||
| set(OpenMP_CXX_LIB_NAMES "libomp" "libgomp" "libiomp5") | ||
| set(OpenMP_libomp_LIBRARY ${OpenMP_CXX_LIB_NAMES}) | ||
| set(OpenMP_libgomp_LIBRARY ${OpenMP_CXX_LIB_NAMES}) | ||
| set(OpenMP_libiomp5_LIBRARY ${OpenMP_CXX_LIB_NAMES}) | ||
| endif() | ||
| endif() | ||
|
|
||
| if(USE_OpenMP) | ||
| find_package(OpenMP REQUIRED) | ||
| endif(USE_OpenMP) | ||
|
|
||
| if (OPENMP_FOUND) | ||
| #include_directories("${OPENMP_INCLUDES}") | ||
| link_directories("${OPENMP_LIBRARIES}") | ||
| set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") | ||
| set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") | ||
| # set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") | ||
| endif(OPENMP_FOUND) | ||
| # ---- OPENMP END ---- | ||
| # --- FAISS END | ||
|
|
||
| # Set OS specific variables | ||
| if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) | ||
| set(CMAKE_MACOSX_RPATH 1) | ||
| set(JVM_OS_TYPE darwin) | ||
| set(LIB_EXT .jnilib) | ||
| elseif(${CMAKE_SYSTEM_NAME} STREQUAL Linux) | ||
| set(JVM_OS_TYPE linux) | ||
| set(LIB_EXT .so) | ||
| else() | ||
| message( FATAL_ERROR "Unable to run on system: ${CMAKE_SYSTEM_NAME}") | ||
| endif() | ||
|
|
||
| # Compile the library | ||
| add_library(${KNN_INDEX} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/com_amazon_opendistroforelasticsearch_knn_index_faiss_KNNFIndex.cpp) | ||
| target_link_libraries(${KNN_INDEX} ${FIASSLIB} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) | ||
| target_include_directories(${KNN_INDEX} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external ) | ||
|
|
||
| set_target_properties(${KNN_INDEX} PROPERTIES SUFFIX ${LIB_EXT}) | ||
| set_target_properties(${KNN_INDEX} PROPERTIES POSITION_INDEPENDENT_CODE ON) | ||
| set_target_properties(${KNN_INDEX} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) | ||
|
|
||
| # Installation rules for shared library | ||
| install(TARGETS ${KNN_INDEX} | ||
| LIBRARY DESTINATION lib | ||
| COMPONENT library) | ||
|
|
||
| # CPack section to build artifacts | ||
| set(KNN_MAINTAINER "OpenDistro for Elasticsearch Team <[email protected]>") | ||
| set(ODFE_DOWNLOAD_URL "https://opendistro.github.io/elasticsearch/downloads") | ||
| set(CPACK_PACKAGE_NAME ${KNN_PACKAGE_NAME}) | ||
| set(CPACK_PACKAGE_VERSION 1.12.0.0) | ||
| set(CMAKE_INSTALL_PREFIX /usr) | ||
| set(CPACK_GENERATOR "RPM;DEB") | ||
| SET(CPACK_OUTPUT_FILE_PREFIX packages) | ||
| set(CPACK_PACKAGE_RELEASE 1) | ||
| set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "KNN JNI library built off of NMSLIB for OpenDistro for Elasticsearch. Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/.") | ||
| set(CPACK_PACKAGE_VENDOR "Amazon") | ||
| set(CPACK_PACKAGE_CONTACT "Maintainer: ${KNN_MAINTAINER}") | ||
| set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) | ||
| set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}_${JVM_OS_TYPE}.${CMAKE_SYSTEM_PROCESSOR}") | ||
|
|
||
| # RPM Specific variables | ||
| set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE}) | ||
| set(CPACK_RPM_PACKAGE_URL ${ODFE_DOWNLOAD_URL}) | ||
| set(CPACK_RPM_PACKAGE_DESCRIPTION "Open Distro for Elasticsearch KNN JNI Library") | ||
| set(CPACK_RPM_PACKAGE_LICENSE "ASL-2.0") | ||
|
|
||
| # DEB Specific variables | ||
| set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${ODFE_DOWNLOAD_URL}) | ||
| set(CPACK_DEBIAN_PACKAGE_MAINTAINER ${KNN_MAINTAINER}) | ||
| set(CPACK_DEBIAN_PACKAGE_SOURCE ${CPACK_PACKAGE_NAME}) | ||
| set(CPACK_DEBIAN_PACKAGE_VERSION ${CPACK_PACKAGE_VERSION}) | ||
| set(CPACK_DEBIAN_PACKAGE_SECTION "libs") | ||
|
|
||
| include(CPack) | ||
Binary file not shown.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You could remove
dependsOn cmakeJniLiband just rely ondependsOn buildJniNmsLibdependsOn buildJniFaissLibas they internally depends oncmakeJniNmsLibandcmakeJniFaissLibThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks, I fixed it