added an Unstructured.io test

.github/workflows/_run_e2e_tests.yml

@@ -90,6 +90,7 @@ jobs:
           LLAMA_CLOUD_API_KEY: "${{ secrets.E2E_TESTS_LLAMA_CLOUD_API_KEY }}"
           GCLOUD_ACCOUNT_KEY_JSON: "${{ secrets.E2E_TESTS_GCLOUD_ACCOUNT_KEY_JSON }}"
           NVIDIA_API_KEY: "${{ secrets.E2E_TESTS_NVIDIA_API_KEY }}"
+          UNSTRUCTURED_API_KEY: "${{ secrets.E2E_TESTS_UNSTRUCTURED_API_KEY }}"
         run: |
           source scripts/ci-common-env.sh
           tox -e notebooks
@@ -116,6 +117,7 @@ jobs:
           NVIDIA_API_KEY: "${{ secrets.E2E_TESTS_NVIDIA_API_KEY }}"
           LANGCHAIN_API_KEY: "${{ secrets.E2E_TESTS_LANGCHAIN_API_KEY }}"
           LLAMA_CLOUD_API_KEY: "${{ secrets.E2E_TESTS_LLAMA_CLOUD_API_KEY }}"
+          UNSTRUCTURED_API_KEY: "${{ secrets.E2E_TESTS_UNSTRUCTURED_API_KEY }}"
         run: |
           source scripts/ci-common-env.sh
           if [ "${{ inputs.suite-name == 'ragstack' }}" == "true" ]; then

pyproject.toml

@@ -13,7 +13,7 @@ packages = [{ include = "ragstack" }]
 python = ">=3.9,<4.0"
 astrapy = "~0.7.0"
 cassio = "~0.1.3"
-unstructured = "^0.10"
+unstructured = { version = "0.12.4" }
 llama-index = { version = "0.9.48", extras = ["langchain"] }
 llama-parse = { version = "0.1.4" }
 langchain = { version = "0.1.4" }

ragstack-e2e-tests/.env.template

@@ -33,3 +33,6 @@ VECTOR_DATABASE_TYPE=astradb
 
 # LlamaIndex
 # LLAMA_CLOUD_API_KEY=
+
+# Unstructured.io
+# UNSTRUCTURED_API_KEY=

ragstack-e2e-tests/e2e_tests/langchain/test_unstructured.py

@@ -0,0 +1,79 @@
+import pytest
+
+from langchain_community.document_loaders import UnstructuredAPIFileLoader
+from langchain.text_splitter import TokenTextSplitter
+from langchain.schema.output_parser import StrOutputParser
+from langchain.prompts import PromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+
+from e2e_tests.langchain.rag_application import BASIC_QA_PROMPT
+from e2e_tests.test_utils import get_local_resource_path
+
+from e2e_tests.conftest import (
+    set_current_test_info,
+    get_required_env,
+    get_vector_store_handler,
+)
+
+from e2e_tests.test_utils.vector_store_handler import (
+    VectorStoreImplementation,
+    VectorStoreTestContext,
+)
+
+
+@pytest.fixture
+def astra_db():
+    handler = get_vector_store_handler(VectorStoreImplementation.ASTRADB)
+    context = handler.before_test()
+    yield context
+    handler.after_test()
+
+
+@pytest.fixture
+def cassandra():
+    handler = get_vector_store_handler(VectorStoreImplementation.CASSANDRA)
+    context = handler.before_test()
+    yield context
+    handler.after_test()
+
+
+@pytest.mark.parametrize("vector_store", ["cassandra", "astra_db"])
+@pytest.mark.parametrize("unstructured_mode", ["single", "elements"])
+def test_unstructured_api(vector_store, unstructured_mode, request):
+    set_current_test_info(
+        "langchain::unstructured-api",
+        f"{unstructured_mode},{vector_store}",
+    )
+
+    vector_store_context: VectorStoreTestContext = request.getfixturevalue(vector_store)
+    embedding = OpenAIEmbeddings(openai_api_key=get_required_env("OPEN_AI_KEY"))
+    vector_store = vector_store_context.new_langchain_vector_store(embedding=embedding)
+
+    loader = UnstructuredAPIFileLoader(
+        file_path=get_local_resource_path("tree.pdf"),
+        mode=unstructured_mode,
+        strategy="auto",
+        api_key=get_required_env("UNSTRUCTURED_API_KEY"),
+    )
+
+    splitter = TokenTextSplitter(chunk_size=512, chunk_overlap=0)
+    vector_store.add_documents(splitter.split_documents(loader.load()))
+
+    prompt = PromptTemplate.from_template(BASIC_QA_PROMPT)
+    llm = ChatOpenAI(
+        openai_api_key=get_required_env("OPEN_AI_KEY"),
+        model="gpt-3.5-turbo-16k",
+        streaming=False,
+        temperature=0,
+    )
+
+    chain = (
+        {"context": vector_store.as_retriever(), "question": RunnablePassthrough()}
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+    response = chain.invoke("What was Eldenroot?")
+    assert len(response) > 0

ragstack-e2e-tests/pyproject.langchain.toml

@@ -38,7 +38,7 @@ llama-parse = { version = "0.1.4" }
 astrapy = "~0.7.0"
 # we need this specific feature from cassio: https://github.com/CassioML/cassio/pull/128
 cassio = "~0.1.4"
-unstructured = "^0.10"
+unstructured = { git = "https://github.com/Unstructured-IO/unstructured.git", branch = "main" }
 
 [build-system]
 requires = ["poetry-core"]

ragstack-e2e-tests/pyproject.llamaindex.toml

@@ -51,7 +51,7 @@ langchain-nvidia-ai-endpoints = { version = "0.0.1" }
 astrapy = "~0.7.0"
 # we need this specific feature from cassio: https://github.com/CassioML/cassio/pull/128
 cassio = "~0.1.4"
-unstructured = "^0.10"
+unstructured = { version = "0.12.4" }
 
 [build-system]
 requires = ["poetry-core"]

ragstack-e2e-tests/tox.ini

@@ -27,6 +27,7 @@ pass_env =
     LANGCHAIN_ENDPOINT
     LANGCHAIN_PROJECT
     LLAMA_CLOUD_API_KEY
+    UNSTRUCTURED_API_KEY
 deps =
     poetry
 commands =

tox.ini

@@ -21,6 +21,7 @@ pass_env =
     LANGCHAIN_PROJECT
     LLAMA_CLOUD_API_KEY
     NVIDIA_API_KEY
+    UNSTRUCTURED_API_KEY
 deps =
     pytest
     nbmake