test_retriever1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e7af139a-40c8-4686-acf3-2942f901c2a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "from langchain_core.documents.base import Document\n",
    "from langchain_community.document_loaders import UnstructuredPDFLoader\n",
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "from langchain_community.llms import OCIGenAI\n",
    "from langchain_community.vectorstores import FAISS\n",
    "\n",
    "from oci_utils import OCIGenAIEmbeddingsWithBatch\n",
    "\n",
    "from config import EMBED_MODEL, GENAI_MODEL, ENDPOINT\n",
    "from config_private import COMPARTMENT_OCID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "92627b4d-c181-44f9-a738-b05ef7ae4a7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "TOP_K = 4\n",
    "F_NAME = \"database-concepts.pdf\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8286a2d2-9806-40d9-9284-98cd9ac388cc",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'open_filename' from 'pdfminer.utils' (/Users/lsaetta/miniforge3/envs/langchain2024/lib/python3.11/site-packages/pdfminer/utils.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 5\u001b[0m\n\u001b[1;32m      2\u001b[0m loader \u001b[38;5;241m=\u001b[39m UnstructuredPDFLoader(F_NAME)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# document are pages\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m docs \u001b[38;5;241m=\u001b[39m \u001b[43mloader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_and_split\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m texts \u001b[38;5;241m=\u001b[39m [doc\u001b[38;5;241m.\u001b[39mpage_content \u001b[38;5;28;01mfor\u001b[39;00m doc \u001b[38;5;129;01min\u001b[39;00m docs]\n",
      "File \u001b[0;32m~/miniforge3/envs/langchain2024/lib/python3.11/site-packages/langchain_core/document_loaders/base.py:63\u001b[0m, in \u001b[0;36mBaseLoader.load_and_split\u001b[0;34m(self, text_splitter)\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     62\u001b[0m     _text_splitter \u001b[38;5;241m=\u001b[39m text_splitter\n\u001b[0;32m---> 63\u001b[0m docs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _text_splitter\u001b[38;5;241m.\u001b[39msplit_documents(docs)\n",
      "File \u001b[0;32m~/miniforge3/envs/langchain2024/lib/python3.11/site-packages/langchain_core/document_loaders/base.py:29\u001b[0m, in \u001b[0;36mBaseLoader.load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[Document]:\n\u001b[1;32m     28\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Load data into Document objects.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 29\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlazy_load\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/langchain2024/lib/python3.11/site-packages/langchain_community/document_loaders/unstructured.py:88\u001b[0m, in \u001b[0;36mUnstructuredBaseLoader.lazy_load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlazy_load\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[Document]:\n\u001b[1;32m     87\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Load file.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 88\u001b[0m     elements \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_elements\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     89\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_post_process_elements(elements)\n\u001b[1;32m     90\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124melements\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[0;32m~/miniforge3/envs/langchain2024/lib/python3.11/site-packages/langchain_community/document_loaders/pdf.py:71\u001b[0m, in \u001b[0;36mUnstructuredPDFLoader._get_elements\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_get_elements\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List:\n\u001b[0;32m---> 71\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01munstructured\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpartition\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpdf\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m partition_pdf\n\u001b[1;32m     73\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m partition_pdf(filename\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_path, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39munstructured_kwargs)\n",
      "File \u001b[0;32m~/miniforge3/envs/langchain2024/lib/python3.11/site-packages/unstructured/partition/pdf.py:36\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpdfminer\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlayout\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     29\u001b[0m     LTChar,\n\u001b[1;32m     30\u001b[0m     LTContainer,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     33\u001b[0m     LTTextBox,\n\u001b[1;32m     34\u001b[0m )\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpdfminer\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpdftypes\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PDFObjRef\n\u001b[0;32m---> 36\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpdfminer\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m open_filename\n\u001b[1;32m     37\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mPIL\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Image \u001b[38;5;28;01mas\u001b[39;00m PILImage\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpillow_heif\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m register_heif_opener\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'open_filename' from 'pdfminer.utils' (/Users/lsaetta/miniforge3/envs/langchain2024/lib/python3.11/site-packages/pdfminer/utils.py)"
     ]
    }
   ],
   "source": [
    "# loader = PyPDFLoader(F_NAME)\n",
    "loader = UnstructuredPDFLoader(F_NAME)\n",
    "\n",
    "# document are pages\n",
    "docs = loader.load_and_split()\n",
    "\n",
    "texts = [doc.page_content for doc in docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff169c4a-9a7b-4e34-8b4e-46b14cfe415e",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_model = OCIGenAIEmbeddingsWithBatch(\n",
    "    auth_type=\"API_KEY\",\n",
    "    model_id=EMBED_MODEL,\n",
    "    service_endpoint=ENDPOINT,\n",
    "    compartment_id=COMPARTMENT_OCID,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "718956fa-6d17-4ef5-b55f-acddf1083329",
   "metadata": {},
   "outputs": [],
   "source": [
    "db = FAISS.from_documents(docs, embed_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7bd7756-6dcc-4613-8aed-b11cd247826d",
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = db.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bec0815a-9f56-4edd-96f7-97b7e72980fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"What is Data Guard?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac29e984-8038-48f7-b4df-39833911cdfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = retriever.get_relevant_documents(query, search_kwargs={\"k\": TOP_K})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b4e079a-473b-4113-84c3-3ce5317ba379",
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in docs:\n",
    "    print(doc.page_content)\n",
    "    print(\"\")\n",
    "    print(\"-----------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93aab96d-fdf3-44f4-b493-5ddef9b75e98",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}