From 01da8742b5a565b07f7523d538d2d2c6d3348b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 14 Feb 2024 12:46:35 +0100 Subject: [PATCH 1/9] tests: add support for llama-index 0.10.x --- pyproject.toml | 2 +- .../e2e_tests/llama_index/test_astra.py | 55 +++++++++++----- .../llama_index/test_compatibility_rag.py | 66 ++++++++++++------- .../e2e_tests/llama_index/test_llama_parse.py | 27 ++++++-- .../astradb_vector_store_handler.py | 8 ++- .../cassandra_vector_store_handler.py | 21 ++++-- ragstack-e2e-tests/pyproject.langchain.toml | 2 +- ragstack-e2e-tests/pyproject.llamaindex.toml | 19 +++++- ragstack-e2e-tests/pyproject.ragstack-ai.toml | 2 +- ragstack-e2e-tests/pyproject.toml | 2 +- 10 files changed, 147 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9d5634a1e..2bf8b00e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ documentation = "https://docs.datastax.com/en/ragstack" packages = [{ include = "ragstack" }] [tool.poetry.dependencies] -python = ">=3.9,<4.0" +python = ">=3.9,<3.12" astrapy = "~0.7.0" cassio = "~0.1.3" unstructured = "^0.10" diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py index 046526416..b99fda79a 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py @@ -8,21 +8,46 @@ get_required_env, is_astra, ) -from llama_index import ( - ServiceContext, - StorageContext, - VectorStoreIndex, - Document, -) -from llama_index.embeddings import BaseEmbedding -from llama_index.llms import OpenAI, LLM -from llama_index.node_parser import SimpleNodeParser -from llama_index.schema import NodeWithScore -from llama_index.vector_stores import ( - AstraDBVectorStore, - MetadataFilters, - ExactMatchFilter, -) +try: + # llamaindex 0.9.x + from llama_index import ( + ServiceContext, + StorageContext, + VectorStoreIndex, + Document, + ) + from llama_index.embeddings import BaseEmbedding + from llama_index.llms import OpenAI, LLM + from llama_index.node_parser import SimpleNodeParser + from llama_index.schema import NodeWithScore + from llama_index.vector_stores import ( + AstraDBVectorStore, + MetadataFilters, + ExactMatchFilter, + ) +except ImportError: + # llamaindex 0.10.x + from llama_index.core import ( + ServiceContext, + StorageContext, + VectorStoreIndex, + Document, + ) + from llama_index.core.embeddings import BaseEmbedding + from llama_index.core.llms import LLM + from llama_index.core.node_parser import SimpleNodeParser + from llama_index.core.schema import NodeWithScore + from llama_index.core.vector_stores import ( + + MetadataFilters, + ExactMatchFilter, + ) + from llama_index.vector_stores.astra import AstraDBVectorStore + from llama_index.llms.openai import OpenAI + + + + from e2e_tests.test_utils import skip_test_due_to_implementation_not_supported from e2e_tests.test_utils.astradb_vector_store_handler import AstraDBVectorStoreHandler diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py index 1b3b6c354..fb0de0bb5 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py @@ -2,28 +2,50 @@ import pytest from langchain.embeddings import VertexAIEmbeddings, HuggingFaceInferenceAPIEmbeddings -from llama_index import ( - VectorStoreIndex, - StorageContext, - ServiceContext, - Document, -) -from llama_index.embeddings import ( - OpenAIEmbedding, - AzureOpenAIEmbedding, - BedrockEmbedding, -) -from llama_index.llms import ( - OpenAI, - AzureOpenAI, - Vertex, - Bedrock, - HuggingFaceInferenceAPI, - ChatMessage, - Gemini, -) -from llama_index.multi_modal_llms import GeminiMultiModal -from llama_index.schema import ImageNode +try: + # llamaindex 0.9.x + from llama_index import ( + ServiceContext, + StorageContext, + VectorStoreIndex, + Document + ) + from llama_index.embeddings import ( + OpenAIEmbedding, + AzureOpenAIEmbedding, + BedrockEmbedding, + ) + from llama_index.llms import ( + OpenAI, + AzureOpenAI, + Vertex, + Bedrock, + HuggingFaceInferenceAPI, + ChatMessage, + Gemini, + ) + from llama_index.multi_modal_llms import GeminiMultiModal + from llama_index.schema import ImageNode +except ImportError: + # llamaindex 0.10.x + from llama_index.core import ( + ServiceContext, + StorageContext, + VectorStoreIndex, + Document, + ) + from llama_index.core.base.llms.types import ChatMessage + from llama_index.core.schema import ImageNode + from llama_index.llms.openai import OpenAI + from llama_index.llms.azure_openai import AzureOpenAI + from llama_index.embeddings.openai import OpenAIEmbedding + from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding + from llama_index.embeddings.bedrock import BedrockEmbedding + from llama_index.llms.vertex import Vertex + from llama_index.llms.bedrock import Bedrock + from llama_index.llms.huggingface import HuggingFaceInferenceAPI + from llama_index.llms.gemini import Gemini + from e2e_tests.conftest import ( set_current_test_info, diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py index 0265bb55f..6b08bffd4 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py @@ -5,14 +5,27 @@ except ImportError: pytest.skip("llama_parse is not supported, skipping tests", allow_module_level=True) -from llama_index import ( - VectorStoreIndex, - StorageContext, - ServiceContext, -) -from llama_index.embeddings import OpenAIEmbedding -from llama_index.llms import OpenAI +try: + # llamaindex 0.9.x + from llama_index import ( + ServiceContext, + StorageContext, + VectorStoreIndex + ) + from llama_index.embeddings import OpenAIEmbedding + from llama_index.llms import OpenAI +except ImportError: + # llamaindex 0.10.x + from llama_index.core import ( + ServiceContext, + StorageContext, + VectorStoreIndex + ) + from llama_index.llms.openai import OpenAI + from llama_index.embeddings.openai import OpenAIEmbedding + + from e2e_tests.conftest import ( set_current_test_info, diff --git a/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py b/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py index a75936b41..d9f57c57f 100644 --- a/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py +++ b/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py @@ -10,7 +10,13 @@ from langchain_community.chat_message_histories import AstraDBChatMessageHistory from langchain_community.vectorstores.astradb import AstraDB from langchain_core.chat_history import BaseChatMessageHistory -from llama_index.vector_stores import AstraDBVectorStore +try: + from llama_index.vector_stores import AstraDBVectorStore +except ImportError: + from llama_index.core.schema import NodeRelationship, RelatedNodeInfo, TextNode + from llama_index.core.vector_stores.types import VectorStoreQuery + from llama_index.vector_stores.astra import AstraDBVectorStore + from llama_index.vector_stores.astra import AstraDBVectorStore from e2e_tests.test_utils import get_required_env, random_string from e2e_tests.test_utils.vector_store_handler import ( diff --git a/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py b/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py index 7bfa36081..24ea8aec4 100644 --- a/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py +++ b/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py @@ -11,11 +11,22 @@ ) from langchain_community.vectorstores.cassandra import Cassandra from langchain_core.chat_history import BaseChatMessageHistory -from llama_index.schema import TextNode -from llama_index.vector_stores import CassandraVectorStore -from llama_index.vector_stores.types import ( - VectorStoreQuery, -) +try: + # llamaindex 0.9.x + from llama_index.schema import TextNode + from llama_index.vector_stores import CassandraVectorStore + from llama_index.vector_stores.types import ( + VectorStoreQuery, + ) +except ImportError: + # llamaindex 0.10.x + from llama_index.core.schema import TextNode + from llama_index.vector_stores.cassandra import CassandraVectorStore + from llama_index.core.vector_stores.types import ( + VectorStoreQuery, + ) + + from e2e_tests.test_utils import ( random_string, diff --git a/ragstack-e2e-tests/pyproject.langchain.toml b/ragstack-e2e-tests/pyproject.langchain.toml index f8ce85dfc..72327ab82 100644 --- a/ragstack-e2e-tests/pyproject.langchain.toml +++ b/ragstack-e2e-tests/pyproject.langchain.toml @@ -14,7 +14,7 @@ black = "*" ruff = "*" google-cloud-aiplatform = "^1.36.4" boto3 = "^1.29.6" -huggingface-hub = "^0.19.4" +huggingface-hub = "^0.20.3" azure-storage-blob = "^12.19.0" pillow = "^10.2.0" testcontainers = "^3.7.1" diff --git a/ragstack-e2e-tests/pyproject.llamaindex.toml b/ragstack-e2e-tests/pyproject.llamaindex.toml index e35a26fa8..7cb4e1cd4 100644 --- a/ragstack-e2e-tests/pyproject.llamaindex.toml +++ b/ragstack-e2e-tests/pyproject.llamaindex.toml @@ -14,7 +14,7 @@ black = "*" ruff = "*" google-cloud-aiplatform = "^1.36.4" boto3 = "^1.29.6" -huggingface-hub = "^0.19.4" +huggingface-hub = "^0.20.3" azure-storage-blob = "^12.19.0" pillow = "^10.2.0" testcontainers = "^3.7.1" @@ -24,8 +24,21 @@ trulens-eval = "^0.21.0" # From LangChain optional deps, needed by WebBaseLoader beautifulsoup4 = "^4" -llama-index = { git = "https://github.com/run-llama/llama_index.git", branch = "main" } -llama-parse = { version = "0.2.0" } +llama-index-core = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-core" } +llama-index-vector-stores-astra = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/vector_stores/llama-index-vector-stores-astra" } +llama-index-vector-stores-cassandra = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/vector_stores/llama-index-vector-stores-cassandra" } +llama-index-llms-bedrock = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-bedrock" } +llama-index-llms-azure-openai = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-azure-openai" } +llama-index-llms-gemini = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-gemini" } +llama-index-llms-huggingface = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-huggingface" } +llama-index-llms-vertex = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-vertex" } +llama-index-embeddings-bedrock = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/embeddings/llama-index-embeddings-bedrock" } +llama-index-embeddings-azure-openai = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/embeddings/llama-index-embeddings-azure-openai" } +llama-index-embeddings-gemini = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/embeddings/llama-index-embeddings-gemini" } +llama-index-embeddings-huggingface = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/embeddings/llama-index-embeddings-huggingface" } +llama-index-multi-modal-llms-gemini = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/multi_modal_llms/llama-index-multi-modal-llms-gemini" } + +llama-parse = { git = "https://github.com/run-llama/llama_parse.git", branch = "main" } langchain = { version = "0.1.2" } langchain-core = "0.1.15" diff --git a/ragstack-e2e-tests/pyproject.ragstack-ai.toml b/ragstack-e2e-tests/pyproject.ragstack-ai.toml index 636d0a954..769eaf37e 100644 --- a/ragstack-e2e-tests/pyproject.ragstack-ai.toml +++ b/ragstack-e2e-tests/pyproject.ragstack-ai.toml @@ -16,7 +16,7 @@ google-cloud-aiplatform = "^1.36.4" langchain-google-genai = "^0.0.4" langchain-nvidia-ai-endpoints = "^0.0.1" boto3 = "^1.29.6" -huggingface-hub = "^0.19.4" +huggingface-hub = "^0.20.3" azure-storage-blob = "^12.19.0" pillow = "^10.2.0" testcontainers = "^3.7.1" diff --git a/ragstack-e2e-tests/pyproject.toml b/ragstack-e2e-tests/pyproject.toml index 729863b08..be6bf4f1a 100644 --- a/ragstack-e2e-tests/pyproject.toml +++ b/ragstack-e2e-tests/pyproject.toml @@ -14,7 +14,7 @@ pytest = "*" black = "*" ruff = "*" boto3 = "^1.34.21" -huggingface-hub = "^0.19.4" +huggingface-hub = "^0.20.3" azure-storage-blob = "^12.19.0" pillow = "^10.2.0" testcontainers = "^3.7.1" From 8a19564ed4f722a0a7e3b6db43dde5d954a310df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 14 Feb 2024 13:01:22 +0100 Subject: [PATCH 2/9] lit --- .../e2e_tests/llama_index/test_astra.py | 5 +---- .../e2e_tests/llama_index/test_compatibility_rag.py | 8 ++------ .../e2e_tests/llama_index/test_llama_parse.py | 13 ++----------- .../test_utils/astradb_vector_store_handler.py | 4 +--- .../test_utils/cassandra_vector_store_handler.py | 2 +- 5 files changed, 7 insertions(+), 25 deletions(-) diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py index b99fda79a..3cbe1c39f 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py @@ -8,6 +8,7 @@ get_required_env, is_astra, ) + try: # llamaindex 0.9.x from llama_index import ( @@ -38,7 +39,6 @@ from llama_index.core.node_parser import SimpleNodeParser from llama_index.core.schema import NodeWithScore from llama_index.core.vector_stores import ( - MetadataFilters, ExactMatchFilter, ) @@ -46,9 +46,6 @@ from llama_index.llms.openai import OpenAI - - - from e2e_tests.test_utils import skip_test_due_to_implementation_not_supported from e2e_tests.test_utils.astradb_vector_store_handler import AstraDBVectorStoreHandler from e2e_tests.test_utils.vector_store_handler import VectorStoreImplementation diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py index fb0de0bb5..3529aab15 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py @@ -2,14 +2,10 @@ import pytest from langchain.embeddings import VertexAIEmbeddings, HuggingFaceInferenceAPIEmbeddings + try: # llamaindex 0.9.x - from llama_index import ( - ServiceContext, - StorageContext, - VectorStoreIndex, - Document - ) + from llama_index import ServiceContext, StorageContext, VectorStoreIndex, Document from llama_index.embeddings import ( OpenAIEmbedding, AzureOpenAIEmbedding, diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py index 6b08bffd4..333b63577 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_llama_parse.py @@ -8,25 +8,16 @@ try: # llamaindex 0.9.x - from llama_index import ( - ServiceContext, - StorageContext, - VectorStoreIndex - ) + from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.embeddings import OpenAIEmbedding from llama_index.llms import OpenAI except ImportError: # llamaindex 0.10.x - from llama_index.core import ( - ServiceContext, - StorageContext, - VectorStoreIndex - ) + from llama_index.core import ServiceContext, StorageContext, VectorStoreIndex from llama_index.llms.openai import OpenAI from llama_index.embeddings.openai import OpenAIEmbedding - from e2e_tests.conftest import ( set_current_test_info, get_required_env, diff --git a/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py b/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py index d9f57c57f..f90f94326 100644 --- a/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py +++ b/ragstack-e2e-tests/e2e_tests/test_utils/astradb_vector_store_handler.py @@ -10,12 +10,10 @@ from langchain_community.chat_message_histories import AstraDBChatMessageHistory from langchain_community.vectorstores.astradb import AstraDB from langchain_core.chat_history import BaseChatMessageHistory + try: from llama_index.vector_stores import AstraDBVectorStore except ImportError: - from llama_index.core.schema import NodeRelationship, RelatedNodeInfo, TextNode - from llama_index.core.vector_stores.types import VectorStoreQuery - from llama_index.vector_stores.astra import AstraDBVectorStore from llama_index.vector_stores.astra import AstraDBVectorStore from e2e_tests.test_utils import get_required_env, random_string diff --git a/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py b/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py index 24ea8aec4..0f7f01794 100644 --- a/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py +++ b/ragstack-e2e-tests/e2e_tests/test_utils/cassandra_vector_store_handler.py @@ -11,6 +11,7 @@ ) from langchain_community.vectorstores.cassandra import Cassandra from langchain_core.chat_history import BaseChatMessageHistory + try: # llamaindex 0.9.x from llama_index.schema import TextNode @@ -27,7 +28,6 @@ ) - from e2e_tests.test_utils import ( random_string, ) From 91f3a56786b8b20e3685af2cac67a5a67290150b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 14 Feb 2024 15:49:01 +0100 Subject: [PATCH 3/9] fix --- ragstack-e2e-tests/pyproject.llamaindex.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ragstack-e2e-tests/pyproject.llamaindex.toml b/ragstack-e2e-tests/pyproject.llamaindex.toml index 7cb4e1cd4..0c096eaf1 100644 --- a/ragstack-e2e-tests/pyproject.llamaindex.toml +++ b/ragstack-e2e-tests/pyproject.llamaindex.toml @@ -24,7 +24,7 @@ trulens-eval = "^0.21.0" # From LangChain optional deps, needed by WebBaseLoader beautifulsoup4 = "^4" -llama-index-core = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-core" } +llama-index = { git = "https://github.com/run-llama/llama_index.git", branch = "main" } llama-index-vector-stores-astra = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/vector_stores/llama-index-vector-stores-astra" } llama-index-vector-stores-cassandra = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/vector_stores/llama-index-vector-stores-cassandra" } llama-index-llms-bedrock = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-bedrock" } From 12e45a6fde7e17d06938a4fe8d254f754da32b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 15 Feb 2024 08:55:23 +0100 Subject: [PATCH 4/9] fix Document --- ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py index 3cbe1c39f..62270ac28 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py @@ -31,13 +31,12 @@ from llama_index.core import ( ServiceContext, StorageContext, - VectorStoreIndex, - Document, + VectorStoreIndex ) from llama_index.core.embeddings import BaseEmbedding from llama_index.core.llms import LLM from llama_index.core.node_parser import SimpleNodeParser - from llama_index.core.schema import NodeWithScore + from llama_index.core.schema import NodeWithScore, Document from llama_index.core.vector_stores import ( MetadataFilters, ExactMatchFilter, From f2fe731d66bc6d7e59b1372d99268e24277256bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 15 Feb 2024 08:59:45 +0100 Subject: [PATCH 5/9] lint --- ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py index 62270ac28..e33e00d67 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_astra.py @@ -28,11 +28,7 @@ ) except ImportError: # llamaindex 0.10.x - from llama_index.core import ( - ServiceContext, - StorageContext, - VectorStoreIndex - ) + from llama_index.core import ServiceContext, StorageContext, VectorStoreIndex from llama_index.core.embeddings import BaseEmbedding from llama_index.core.llms import LLM from llama_index.core.node_parser import SimpleNodeParser From fc0f3c73e7aeb12e15d2b02e5edcd3b6f2866add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 15 Feb 2024 09:53:45 +0100 Subject: [PATCH 6/9] fix --- .../langchain_llamaindex/test_astra.py | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py b/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py index 6063a6bcc..585f2f42c 100644 --- a/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py +++ b/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py @@ -7,19 +7,37 @@ from langchain.chat_models import ChatOpenAI from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores.astradb import AstraDB -from llama_index import ( - Document, - OpenAIEmbedding, - ServiceContext, - StorageContext, - VectorStoreIndex, -) -from llama_index.llms import OpenAI -from llama_index.vector_stores import ( - AstraDBVectorStore, - MetadataFilters, - ExactMatchFilter, -) + +try: + # llamaindex 0.9.x + from llama_index import ( + Document, + OpenAIEmbedding, + ServiceContext, + StorageContext, + VectorStoreIndex, + ) + from llama_index.llms import OpenAI + from llama_index.vector_stores import ( + AstraDBVectorStore, + MetadataFilters, + ExactMatchFilter, + ) +except ImportError: + # llamaindex 0.10.x + from llama_index.core import ServiceContext, StorageContext, VectorStoreIndex + from llama_index.core.schema import Document + from llama_index.core.vector_stores import ( + MetadataFilters, + ExactMatchFilter, + ) + from llama_index.vector_stores.astra import AstraDBVectorStore + from llama_index.embeddings.openai import OpenAIEmbedding + from llama_index.llms.openai import OpenAI + + + + from e2e_tests.test_utils import skip_test_due_to_implementation_not_supported from e2e_tests.test_utils.astradb_vector_store_handler import ( From 929f97b8a8f6f5fc6288a7882f7c576d5e572d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 15 Feb 2024 09:59:26 +0100 Subject: [PATCH 7/9] f link --- .../e2e_tests/langchain_llamaindex/test_astra.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py b/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py index 585f2f42c..d7e58d2d8 100644 --- a/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py +++ b/ragstack-e2e-tests/e2e_tests/langchain_llamaindex/test_astra.py @@ -36,9 +36,6 @@ from llama_index.llms.openai import OpenAI - - - from e2e_tests.test_utils import skip_test_due_to_implementation_not_supported from e2e_tests.test_utils.astradb_vector_store_handler import ( AstraDBVectorStoreHandler, From 2ea855a63816b55e8caff329e857041cd330a00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 15 Feb 2024 10:40:55 +0100 Subject: [PATCH 8/9] fix --- .../e2e_tests/llama_index/test_compatibility_rag.py | 1 + ragstack-e2e-tests/pyproject.llamaindex.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py b/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py index 3529aab15..91821f2ab 100644 --- a/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py +++ b/ragstack-e2e-tests/e2e_tests/llama_index/test_compatibility_rag.py @@ -41,6 +41,7 @@ from llama_index.llms.bedrock import Bedrock from llama_index.llms.huggingface import HuggingFaceInferenceAPI from llama_index.llms.gemini import Gemini + from llama_index.multi_modal_llms.gemini import GeminiMultiModal from e2e_tests.conftest import ( diff --git a/ragstack-e2e-tests/pyproject.llamaindex.toml b/ragstack-e2e-tests/pyproject.llamaindex.toml index 0c096eaf1..52262ac93 100644 --- a/ragstack-e2e-tests/pyproject.llamaindex.toml +++ b/ragstack-e2e-tests/pyproject.llamaindex.toml @@ -25,6 +25,7 @@ trulens-eval = "^0.21.0" beautifulsoup4 = "^4" llama-index = { git = "https://github.com/run-llama/llama_index.git", branch = "main" } +llama-index-embeddings-langchain = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/embeddings/llama-index-embeddings-langchain" } llama-index-vector-stores-astra = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/vector_stores/llama-index-vector-stores-astra" } llama-index-vector-stores-cassandra = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/vector_stores/llama-index-vector-stores-cassandra" } llama-index-llms-bedrock = { git = "https://github.com/run-llama/llama_index.git", branch = "main", subdirectory = "llama-index-integrations/llms/llama-index-llms-bedrock" } From 20baf3f91b0fa453df6bf4183bfa51dfb958787a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 15 Feb 2024 11:15:01 +0100 Subject: [PATCH 9/9] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2bf8b00e7..9d5634a1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ documentation = "https://docs.datastax.com/en/ragstack" packages = [{ include = "ragstack" }] [tool.poetry.dependencies] -python = ">=3.9,<3.12" +python = ">=3.9,<4.0" astrapy = "~0.7.0" cassio = "~0.1.3" unstructured = "^0.10"