diff --git a/docs/docs/integrations/document_loaders/pymupdf.ipynb b/docs/docs/integrations/document_loaders/pymupdf.ipynb index 65c92cb6ef0f8..ffc82cdd99698 100644 --- a/docs/docs/integrations/document_loaders/pymupdf.ipynb +++ b/docs/docs/integrations/document_loaders/pymupdf.ipynb @@ -4,9 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# PyMuPDF\n", + "# PyMuPDFLoader\n", "\n", - "`PyMuPDF` is optimized for speed, and contains detailed metadata about the PDF and its pages. It returns one document per page.\n", + "This notebook provides a quick overview for getting started with `PyMuPDF` [document loader](https://python.langchain.com/docs/concepts/document_loaders). For detailed documentation of all __ModuleName__Loader features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyMuPDFLoader.html).\n", + "\n", + " \n", "\n", "## Overview\n", "### Integration details\n", @@ -14,16 +16,22 @@ "| Class | Package | Local | Serializable | JS support|\n", "| :--- | :--- | :---: | :---: | :---: |\n", "| [PyMuPDFLoader](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyMuPDFLoader.html) | [langchain_community](https://python.langchain.com/api_reference/community/index.html) | ✅ | ❌ | ❌ | \n", + "\n", + "--------- \n", + "\n", "### Loader features\n", - "| Source | Document Lazy Loading | Native Async Support\n", - "| :---: | :---: | :---: | \n", - "| PyMuPDFLoader | ✅ | ❌ | \n", + "\n", + "| Source | Document Lazy Loading | Native Async Support | Extract Images | Extract Tables |\n", + "| :---: | :---: | :---: | :---: |:---: |\n", + "| PyMuPDFLoader | ✅ | ❌ | ✅ | ✅ |\n", + "\n", + " \n", "\n", "## Setup\n", "\n", "### Credentials\n", "\n", - "No credentials are needed to use the `PyMuPDFLoader`." + "No credentials are required to use PyMuPDFLoader" ] }, { @@ -35,8 +43,13 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T13:24:30.653579Z", + "start_time": "2025-01-17T13:24:30.650990Z" + } + }, "outputs": [], "source": [ "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n", @@ -54,11 +67,24 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T13:24:33.695776Z", + "start_time": "2025-01-17T13:24:31.737888Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ - "%pip install -qU langchain-community pymupdf" + "%pip install -qU langchain_community pymupdf" ] }, { @@ -67,38 +93,47 @@ "source": [ "## Initialization\n", "\n", - "Now we can initialize our loader and start loading documents. " + "Now we can instantiate our model object and load documents:" ] }, { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:44.403523Z", + "start_time": "2025-01-17T11:06:43.736030Z" + } + }, "outputs": [], "source": [ "from langchain_community.document_loaders import PyMuPDFLoader\n", "\n", - "loader = PyMuPDFLoader(\"./example_data/layout-parser-paper.pdf\")" + "file_path = \"./example_data/layout-parser-paper.pdf\"\n", + "loader = PyMuPDFLoader(file_path)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Load\n", - "\n", - "You can pass along any of the options from the [PyMuPDF documentation](https://pymupdf.readthedocs.io/en/latest/app1.html#plain-text/) as keyword arguments in the `load` call, and it will be pass along to the `get_text()` call." + "## Load" ] }, { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:46.138267Z", + "start_time": "2025-01-17T11:06:46.001187Z" + } + }, "outputs": [ { "data": { "text/plain": [ - "Document(metadata={'source': './example_data/layout-parser-paper.pdf', 'file_path': './example_data/layout-parser-paper.pdf', 'page': 0, 'total_pages': 16, 'format': 'PDF 1.5', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'creator': 'LaTeX with hyperref', 'producer': 'pdfTeX-1.40.21', 'creationDate': 'D:20210622012710Z', 'modDate': 'D:20210622012710Z', 'trapped': ''}, page_content='LayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1 (\\x00), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\nKeywords: Document Image Analysis · Deep Learning · Layout Analysis\\n· Character Recognition · Open Source library · Toolkit.\\n1\\nIntroduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [11,\\narXiv:2103.15348v2 [cs.CV] 21 Jun 2021\\n')" + "Document(metadata={'producer': 'pdfTeX-1.40.21', 'creator': 'LaTeX with hyperref', 'creationdate': '2021-06-22T01:27:10+00:00', 'source': './example_data/layout-parser-paper.pdf', 'file_path': './example_data/layout-parser-paper.pdf', 'total_pages': 16, 'format': 'PDF 1.5', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'trapped': '', 'page': 0}, page_content='LayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1 (\\x00), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\nKeywords: Document Image Analysis · Deep Learning · Layout Analysis\\n· Character Recognition · Open Source library · Toolkit.\\n1\\nIntroduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [11,\\narXiv:2103.15348v2 [cs.CV] 21 Jun 2021')" ] }, "execution_count": 4, @@ -114,41 +149,1126 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:46.646335Z", + "start_time": "2025-01-17T11:06:46.642667Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'source': './example_data/layout-parser-paper.pdf', 'file_path': './example_data/layout-parser-paper.pdf', 'page': 0, 'total_pages': 16, 'format': 'PDF 1.5', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'creator': 'LaTeX with hyperref', 'producer': 'pdfTeX-1.40.21', 'creationDate': 'D:20210622012710Z', 'modDate': 'D:20210622012710Z', 'trapped': ''}\n" + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'file_path': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'format': 'PDF 1.5',\n", + " 'title': '',\n", + " 'author': '',\n", + " 'subject': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'trapped': '',\n", + " 'page': 0}\n" ] } ], "source": [ - "print(docs[0].metadata)" + "import pprint\n", + "\n", + "pprint.pp(docs[0].metadata)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Lazy Load" + "## Lazy Load\n" ] }, { "cell_type": "code", "execution_count": 6, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:48.147692Z", + "start_time": "2025-01-17T11:06:48.094257Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "page = []\n", + "pages = []\n", "for doc in loader.lazy_load():\n", - " page.append(doc)\n", - " if len(page) >= 10:\n", + " pages.append(doc)\n", + " if len(pages) >= 10:\n", " # do some paged operation, e.g.\n", " # index.upsert(page)\n", "\n", - " page = []" + " pages = []\n", + "len(pages)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:50.003790Z", + "start_time": "2025-01-17T11:06:50.000060Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LayoutParser: A Unified Toolkit for DL-Based DIA\n", + "11\n", + "focuses on precision, efficiency, and robustness. T\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'file_path': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'format': 'PDF 1.5',\n", + " 'title': '',\n", + " 'author': '',\n", + " 'subject': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'trapped': '',\n", + " 'page': 10}\n" + ] + } + ], + "source": [ + "print(pages[0].page_content[:100])\n", + "pprint.pp(pages[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The metadata attribute contains at least the following keys:\n", + "- source\n", + "- page (if in mode *page*)\n", + "- total_page\n", + "- creationdate\n", + "- creator\n", + "- producer\n", + "\n", + "Additional metadata are specific to each parser.\n", + "These pieces of information can be helpful (to categorize your PDFs for example)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Splitting mode & custom pages delimiter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When loading the PDF file you can split it in two different ways:\n", + "- By page\n", + "- As a single text flow\n", + "\n", + "By default PDFPlumberLoader will split the PDF by page." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extract the PDF by page. Each page is extracted as a langchain Document object:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:53.613494Z", + "start_time": "2025-01-17T11:06:53.563930Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'file_path': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'format': 'PDF 1.5',\n", + " 'title': '',\n", + " 'author': '',\n", + " 'subject': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'trapped': '',\n", + " 'page': 0}\n" + ] + } + ], + "source": [ + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + ")\n", + "docs = loader.load()\n", + "print(len(docs))\n", + "pprint.pp(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this mode the pdf is split by pages and the resulting Documents metadata contains the page number. But in some cases we could want to process the pdf as a single text flow (so we don't cut some paragraphs in half). In this case you can use the *single* mode :" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extract the whole PDF as a single langchain Document object:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:06:55.955935Z", + "start_time": "2025-01-17T11:06:55.903604Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'file_path': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'format': 'PDF 1.5',\n", + " 'title': '',\n", + " 'author': '',\n", + " 'subject': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'trapped': ''}\n" + ] + } + ], + "source": [ + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"single\",\n", + ")\n", + "docs = loader.load()\n", + "print(len(docs))\n", + "pprint.pp(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Logically, in this mode, the ‘page_number’ metadata disappears. Here's how to clearly identify where pages end in the text flow :" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add a custom *pages_delimiter* to identify where are ends of pages in *single* mode:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:07:31.932597Z", + "start_time": "2025-01-17T11:07:31.885499Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LayoutParser: A Unified Toolkit for Deep\n", + "Learning Based Document Image Analysis\n", + "Zejiang Shen1 (\u0000), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\n", + "Lee4, Jacob Carlson3, and Weining Li5\n", + "1 Allen Institute for AI\n", + "shannons@allenai.org\n", + "2 Brown University\n", + "ruochen zhang@brown.edu\n", + "3 Harvard University\n", + "{melissadell,jacob carlson}@fas.harvard.edu\n", + "4 University of Washington\n", + "bcgl@cs.washington.edu\n", + "5 University of Waterloo\n", + "w422li@uwaterloo.ca\n", + "Abstract. Recent advances in document image analysis (DIA) have been\n", + "primarily driven by the application of neural networks. Ideally, research\n", + "outcomes could be easily deployed in production and extended for further\n", + "investigation. However, various factors like loosely organized codebases\n", + "and sophisticated model configurations complicate the easy reuse of im-\n", + "portant innovations by a wide audience. Though there have been on-going\n", + "efforts to improve reusability and simplify deep learning (DL) model\n", + "development in disciplines like natural language processing and computer\n", + "vision, none of them are optimized for challenges in the domain of DIA.\n", + "This represents a major gap in the existing toolkit, as DIA is central to\n", + "academic research across a wide range of disciplines in the social sciences\n", + "and humanities. This paper introduces LayoutParser, an open-source\n", + "library for streamlining the usage of DL in DIA research and applica-\n", + "tions. The core LayoutParser library comes with a set of simple and\n", + "intuitive interfaces for applying and customizing DL models for layout de-\n", + "tection, character recognition, and many other document processing tasks.\n", + "To promote extensibility, LayoutParser also incorporates a community\n", + "platform for sharing both pre-trained models and full document digiti-\n", + "zation pipelines. We demonstrate that LayoutParser is helpful for both\n", + "lightweight and large-scale digitization pipelines in real-word use cases.\n", + "The library is publicly available at https://layout-parser.github.io.\n", + "Keywords: Document Image Analysis · Deep Learning · Layout Analysis\n", + "· Character Recognition · Open Source library · Toolkit.\n", + "1\n", + "Introduction\n", + "Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of\n", + "document image analysis (DIA) tasks including document image classification [11,\n", + "arXiv:2103.15348v2 [cs.CV] 21 Jun 2021\n", + "-------THIS IS A CUSTOM END OF PAGE-------\n", + "2\n", + "Z. Shen et al.\n", + "37], layout detection [38, 22], table detection [26], and scene text detection [4].\n", + "A generalized learning-based framework dramatically reduces the need for the\n", + "manual specification of complicated rules, which is the status quo with traditional\n", + "methods. DL has the potential to transform DIA pipelines and benefit a broad\n", + "spectrum of large-scale document digitization projects.\n", + "However, there are several practical difficulties for taking advantages of re-\n", + "cent advances in DL-based methods: 1) DL models are notoriously convoluted\n", + "for reuse and extension. Existing models are developed using distinct frame-\n", + "works like TensorFlow [1] or PyTorch [24], and the high-level parameters can\n", + "be obfuscated by implementation details [8]. It can be a time-consuming and\n", + "frustrating experience to debug, reproduce, and adapt existing models for DIA,\n", + "and many researchers who would benefit the most from using these methods lack\n", + "the technical background to implement them from scratch. 2) Document images\n", + "contain diverse and disparate patterns across domains, and customized training\n", + "is often required to achieve a desirable detection accuracy. Currently there is no\n", + "full-fledged infrastructure for easily curating the target document image datasets\n", + "and fine-tuning or re-training the models. 3) DIA usually requires a sequence of\n", + "models and other processing to obtain the final outputs. Often research teams use\n", + "DL models and then perform further document analyses in separate processes,\n", + "and these pipelines are not documented in any central location (and often not\n", + "documented at all). This makes it difficult for research teams to learn about how\n", + "full pipelines are implemented and leads them to invest significant resources in\n", + "reinventing the DIA wheel.\n", + "LayoutParser provides a unified toolkit to support DL-based document image\n", + "analysis and processing. To address the aforementioned challenges, LayoutParser\n", + "is built with the following components:\n", + "1. An off-the-shelf toolkit for applying DL models for layout detection, character\n", + "recognition, and other DIA tasks (Section 3)\n", + "2. A rich repository of pre-trained neural network models (Model Zoo) that\n", + "underlies the off-the-shelf usage\n", + "3. Comprehensive tools for efficient document image data annotation and model\n", + "tuning to support different levels of customization\n", + "4. A DL model hub and community platform for the easy sharing, distribu-\n", + "tion, and discussion of DIA models and pipelines, to promote reusability,\n", + "reproducibility, and extensibility (Section 4)\n", + "The library implements simple and intuitive Python APIs without sacrificing\n", + "generalizability and versatility, and can be easily installed via pip. Its convenient\n", + "functions for handling document image data can be seamlessly integrated with\n", + "existing DIA pipelines. With detailed documentations and carefully curated\n", + "tutorials, we hope this tool will benefit a variety of end-users, and will lead to\n", + "advances in applications in both industry and academic research.\n", + "LayoutParser is well aligned with recent efforts for improving DL model\n", + "reusability in other disciplines like natural language processing [8, 34] and com-\n", + "puter vision [35], but with a focus on unique challenges in DIA. We show\n", + "LayoutParser can be applied in sophisticated and large-scale digitization projects\n", + "-------THIS IS A CUSTOM END OF PAGE-------\n", + "LayoutParser: A Unified Toolkit for DL-Based DIA\n", + "3\n", + "that require precision, efficiency, and robustness, as well as simple and light-\n", + "weigh\n" + ] + } + ], + "source": [ + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"single\",\n", + " pages_delimiter=\"\\n-------THIS IS A CUSTOM END OF PAGE-------\\n\",\n", + ")\n", + "docs = loader.load()\n", + "print(docs[0].page_content[:5780])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This could simply be \\n, or \\f to clearly indicate a page change, or \\ for seamless injection in a Markdown viewer without a visual effect." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Extract images from the PDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can extract images from your PDFs with a choice of three different solutions:\n", + "- rapidOCR (lightweight Optical Character Recognition tool)\n", + "- Tesseract (OCR tool with high precision)\n", + "- Multimodal language model\n", + "\n", + "You can tune these functions to choose the output format of the extracted images among *html*, *markdown* or *text*\n", + "\n", + "The result is inserted between the last and the second-to-last paragraphs of text of the page." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extract images from the PDF with rapidOCR:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:07:39.281686Z", + "start_time": "2025-01-17T11:07:37.500638Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU rapidocr-onnxruntime" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:08:46.036783Z", + "start_time": "2025-01-17T11:08:22.713011Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6\n", + "Z. Shen et al.\n", + "Fig. 2: The relationship between the three types of layout data structures.\n", + "Coordinate supports three kinds of variation; TextBlock consists of the co-\n", + "ordinate information and extra features like block text, types, and reading orders;\n", + "a Layout object is a list of all possible layout elements, including other Layout\n", + "objects. They all support the same set of transformation and operation APIs for\n", + "maximum flexibility.\n", + "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n", + "on 5 different datasets. Description of the training dataset is provided alongside\n", + "with the trained models such that users can quickly identify the most suitable\n", + "models for their tasks. Additionally, when such a model is not readily available,\n", + "LayoutParser also supports training customized layout models and community\n", + "sharing of the models (detailed in Section 3.5).\n", + "3.2\n", + "Layout Data Structures\n", + "A critical feature of LayoutParser is the implementation of a series of data\n", + "structures and operations that can be used to efficiently process and manipulate\n", + "the layout elements. In document image analysis pipelines, various post-processing\n", + "on the layout analysis model outputs is usually required to obtain the final\n", + "outputs. Traditionally, this requires exporting DL model outputs and then loading\n", + "the results into other pipelines. All model outputs from LayoutParser will be\n", + "stored in carefully engineered data types optimized for further processing, which\n", + "makes it possible to build an end-to-end document digitization pipeline within\n", + "LayoutParser. There are three key components in the data structure, namely\n", + "the Coordinate system, the TextBlock, and the Layout. They provide different\n", + "levels of abstraction for the layout data, and a set of APIs are supported for\n", + "transformations or operations on these classes.\n", + "\n", + "\n", + "\n", + "\n", + "![Coordinate\n", + "(x1, y1)\n", + "(X1, y1)\n", + "(x2,y2)\n", + "APIS\n", + "x-interval\n", + "tart\n", + "end\n", + "Quadrilateral\n", + "operation\n", + "Rectangle\n", + "y-interval\n", + "ena\n", + "(x2, y2)\n", + "(x4, y4)\n", + "(x3, y3)\n", + "and\n", + "textblock\n", + "Coordinate\n", + "transformation\n", + "+\n", + "Block\n", + "Block\n", + "Reading\n", + "Extra features\n", + "Text\n", + "Type\n", + "Order\n", + "coordinatel\n", + "textblockl\n", + "layout\n", + " same\n", + "textblock2\n", + "layoutl\n", + "The\n", + "A list of the layout elements](#)\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders.parsers import RapidOCRBlobParser\n", + "\n", + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " images_inner_format=\"markdown-img\",\n", + " images_parser=RapidOCRBlobParser(),\n", + ")\n", + "docs = loader.load()\n", + "\n", + "print(docs[5].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Be careful, RapidOCR is designed to work with Chinese and English, not other languages." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extract images from the PDF with Tesseract:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:08:53.698734Z", + "start_time": "2025-01-17T11:08:52.248547Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU pytesseract" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:09:03.699153Z", + "start_time": "2025-01-17T11:08:55.660127Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6\n", + "Z. Shen et al.\n", + "Fig. 2: The relationship between the three types of layout data structures.\n", + "Coordinate supports three kinds of variation; TextBlock consists of the co-\n", + "ordinate information and extra features like block text, types, and reading orders;\n", + "a Layout object is a list of all possible layout elements, including other Layout\n", + "objects. They all support the same set of transformation and operation APIs for\n", + "maximum flexibility.\n", + "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n", + "on 5 different datasets. Description of the training dataset is provided alongside\n", + "with the trained models such that users can quickly identify the most suitable\n", + "models for their tasks. Additionally, when such a model is not readily available,\n", + "LayoutParser also supports training customized layout models and community\n", + "sharing of the models (detailed in Section 3.5).\n", + "3.2\n", + "Layout Data Structures\n", + "A critical feature of LayoutParser is the implementation of a series of data\n", + "structures and operations that can be used to efficiently process and manipulate\n", + "the layout elements. In document image analysis pipelines, various post-processing\n", + "on the layout analysis model outputs is usually required to obtain the final\n", + "outputs. Traditionally, this requires exporting DL model outputs and then loading\n", + "the results into other pipelines. All model outputs from LayoutParser will be\n", + "stored in carefully engineered data types optimized for further processing, which\n", + "makes it possible to build an end-to-end document digitization pipeline within\n", + "LayoutParser. There are three key components in the data structure, namely\n", + "the Coordinate system, the TextBlock, and the Layout. They provide different\n", + "levels of abstraction for the layout data, and a set of APIs are supported for\n", + "transformations or operations on these classes.\n", + "\n", + "\n", + "\n", + "\n", + "\"Coordinate\n",\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders.parsers import TesseractBlobParser\n", + "\n", + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " images_inner_format=\"html-img\",\n", + " images_parser=TesseractBlobParser(),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[5].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extract images from the PDF with multimodal model:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:09:08.637429Z", + "start_time": "2025-01-17T11:09:07.177157Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain_openai" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:09:09.670266Z", + "start_time": "2025-01-17T11:09:09.634422Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T11:09:11.652399Z", + "start_time": "2025-01-17T11:09:11.649497Z" + } + }, + "outputs": [], + "source": [ + "from getpass import getpass\n", + "\n", + "if not os.environ.get(\"OPENAI_API_KEY\"):\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T12:46:33.398682Z", + "start_time": "2025-01-17T11:09:14.102369Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6\n", + "Z. Shen et al.\n", + "Fig. 2: The relationship between the three types of layout data structures.\n", + "Coordinate supports three kinds of variation; TextBlock consists of the co-\n", + "ordinate information and extra features like block text, types, and reading orders;\n", + "a Layout object is a list of all possible layout elements, including other Layout\n", + "objects. They all support the same set of transformation and operation APIs for\n", + "maximum flexibility.\n", + "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n", + "on 5 different datasets. Description of the training dataset is provided alongside\n", + "with the trained models such that users can quickly identify the most suitable\n", + "models for their tasks. Additionally, when such a model is not readily available,\n", + "LayoutParser also supports training customized layout models and community\n", + "sharing of the models (detailed in Section 3.5).\n", + "3.2\n", + "Layout Data Structures\n", + "A critical feature of LayoutParser is the implementation of a series of data\n", + "structures and operations that can be used to efficiently process and manipulate\n", + "the layout elements. In document image analysis pipelines, various post-processing\n", + "on the layout analysis model outputs is usually required to obtain the final\n", + "outputs. Traditionally, this requires exporting DL model outputs and then loading\n", + "the results into other pipelines. All model outputs from LayoutParser will be\n", + "stored in carefully engineered data types optimized for further processing, which\n", + "makes it possible to build an end-to-end document digitization pipeline within\n", + "LayoutParser. There are three key components in the data structure, namely\n", + "the Coordinate system, the TextBlock, and the Layout. They provide different\n", + "levels of abstraction for the layout data, and a set of APIs are supported for\n", + "transformations or operations on these classes.\n", + "\n", + "\n", + "\n", + "\n", + "![**Image Summary:** Diagram illustrating coordinate systems and textblock features in layout processing. Includes intervals, rectangles, quadrilaterals, and extra features. Textblock elements feature block text, type, and reading order, all transformed by the same APIs.\n", + "\n", + "**Extracted Text:**\n", + "\n", + "Coordinate\n", + "Coordinate\n", + "\n", + "start\n", + "\n", + "start\n", + "\n", + "x-interval\n", + "\n", + "end\n", + "\n", + "y-interval\n", + "\n", + "end\n", + "\n", + "(x1, y1)\n", + "\n", + "Rectangle\n", + "\n", + "(x2, y2)\n", + "\n", + "(x1, y1)\n", + "\n", + "Quadrilateral\n", + "\n", + "(x2, y2)\n", + "\n", + "(x4, y4)\n", + "\n", + "(x3, y3)\n", + "\n", + "→\n", + "\n", + "The same transformation and operation APIs\n", + "\n", + "textblock\n", + "\n", + "Coordinate\n", + "\n", + "+\n", + "\n", + "Extra features\n", + "\n", + "Block Text\n", + "\n", + "Block Type\n", + "\n", + "Reading Order\n", + "\n", + "...\n", + "\n", + "→\n", + "\n", + "layout\n", + "\n", + "[ coordinate1, textblock1, ...\n", + "\n", + "..., textblock2, layout1 \\\\]\n", + "\n", + "A list of the layout elements](#)\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders.parsers import LLMImageBlobParser\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " images_inner_format=\"markdown-img\",\n", + " images_parser=LLMImageBlobParser(model=ChatOpenAI(model=\"gpt-4o\", max_tokens=1024)),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[5].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Extract tables from the PDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With PyMUPDF you can extract tables from your PDFs in *html*, *markdown* or *csv* format :" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T13:49:21.098861Z", + "start_time": "2025-01-17T13:49:19.786166Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LayoutParser: A Unified Toolkit for DL-Based DIA\n", + "5\n", + "Table 1: Current layout detection models in the LayoutParser model zoo\n", + "Dataset\n", + "Base Model1 Large Model\n", + "Notes\n", + "PubLayNet [38]\n", + "F / M\n", + "M\n", + "Layouts of modern scientific documents\n", + "PRImA [3]\n", + "M\n", + "-\n", + "Layouts of scanned modern magazines and scientific reports\n", + "Newspaper [17]\n", + "F\n", + "-\n", + "Layouts of scanned US newspapers from the 20th century\n", + "TableBank [18]\n", + "F\n", + "F\n", + "Table region on modern scientific and business document\n", + "HJDataset [31]\n", + "F / M\n", + "-\n", + "Layouts of history Japanese documents\n", + "1 For each dataset, we train several models of different sizes for different needs (the trade-offbetween accuracy\n", + "vs. computational cost). For “base model” and “large model”, we refer to using the ResNet 50 or ResNet 101\n", + "backbones [13], respectively. One can train models of different architectures, like Faster R-CNN [28] (F) and Mask\n", + "R-CNN [12] (M). For example, an F in the Large Model column indicates it has a Faster R-CNN model trained\n", + "using the ResNet 101 backbone. The platform is maintained and a number of additions will be made to the model\n", + "zoo in coming months.\n", + "layout data structures, which are optimized for efficiency and versatility. 3) When\n", + "necessary, users can employ existing or customized OCR models via the unified\n", + "API provided in the OCR module. 4) LayoutParser comes with a set of utility\n", + "functions for the visualization and storage of the layout data. 5) LayoutParser\n", + "is also highly customizable, via its integration with functions for layout data\n", + "annotation and model training. We now provide detailed descriptions for each\n", + "component.\n", + "3.1\n", + "Layout Detection Models\n", + "In LayoutParser, a layout model takes a document image as an input and\n", + "generates a list of rectangular boxes for the target content regions. Different\n", + "from traditional methods, it relies on deep convolutional neural networks rather\n", + "than manually curated rules to identify content regions. It is formulated as an\n", + "object detection problem and state-of-the-art models like Faster R-CNN [28] and\n", + "Mask R-CNN [12] are used. This yields prediction results of high accuracy and\n", + "makes it possible to build a concise, generalized interface for layout detection.\n", + "LayoutParser, built upon Detectron2 [35], provides a minimal API that can\n", + "perform layout detection with only four lines of code in Python:\n", + "1 import\n", + "layoutparser as lp\n", + "2 image = cv2.imread(\"image_file\") # load\n", + "images\n", + "3 model = lp. Detectron2LayoutModel (\n", + "4\n", + "\"lp:// PubLayNet/ faster_rcnn_R_50_FPN_3x /config\")\n", + "5 layout = model.detect(image)\n", + "LayoutParser provides a wealth of pre-trained model weights using various\n", + "datasets covering different languages, time periods, and document types. Due to\n", + "domain shift [7], the prediction performance can notably drop when models are ap-\n", + "plied to target samples that are significantly different from the training dataset. As\n", + "document structures and layouts vary greatly in different domains, it is important\n", + "to select models trained on a dataset similar to the test samples. A semantic syntax\n", + "is used for initializing the model weights in LayoutParser, using both the dataset\n", + "name and model name lp:///.\n", + "\n", + "\n", + "|Dataset|Base Model1|Large Model|Notes|\n", + "|---|---|---|---|\n", + "|PubLayNet [38] PRImA [3] Newspaper [17] TableBank [18] HJDataset [31]|F / M M F F F / M|M &#45; &#45; F &#45;|Layouts of modern scientific documents Layouts of scanned modern magazines and scientific reports Layouts of scanned US newspapers from the 20th century Table region on modern scientific and business document Layouts of history Japanese documents|\n" + ] + } + ], + "source": [ + "loader = PyMuPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " extract_tables=\"markdown\",\n", + ")\n", + "docs = loader.load()\n", + "print(docs[4].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Working with Files\n", + "\n", + "Many document loaders involve parsing files. The difference between such loaders usually stems from how the file is parsed, rather than how the file is loaded. For example, you can use `open` to read the binary content of either a PDF or a markdown file, but you need different parsing logic to convert that binary data into text.\n", + "\n", + "As a result, it can be helpful to decouple the parsing logic from the loading logic, which makes it easier to re-use a given parser regardless of how the data was loaded.\n", + "You can use this strategy to analyze different files, with the same parsing parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-17T12:46:34.866868Z", + "start_time": "2025-01-17T12:46:34.819048Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LayoutParser: A Unified Toolkit for Deep\n", + "Learning Based Document Image Analysis\n", + "Zejiang Shen1 (\u0000), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\n", + "Lee4, Jacob Carlson3, and Weining Li5\n", + "1 Allen Institute for AI\n", + "shannons@allenai.org\n", + "2 Brown University\n", + "ruochen zhang@brown.edu\n", + "3 Harvard University\n", + "{melissadell,jacob carlson}@fas.harvard.edu\n", + "4 University of Washington\n", + "bcgl@cs.washington.edu\n", + "5 University of Waterloo\n", + "w422li@uwaterloo.ca\n", + "Abstract. Recent advances in document image analysis (DIA) have been\n", + "primarily driven by the application of neural networks. Ideally, research\n", + "outcomes could be easily deployed in production and extended for further\n", + "investigation. However, various factors like loosely organized codebases\n", + "and sophisticated model configurations complicate the easy reuse of im-\n", + "portant innovations by a wide audience. Though there have been on-going\n", + "efforts to improve reusability and simplify deep learning (DL) model\n", + "development in disciplines like natural language processing and computer\n", + "vision, none of them are optimized for challenges in the domain of DIA.\n", + "This represents a major gap in the existing toolkit, as DIA is central to\n", + "academic research across a wide range of disciplines in the social sciences\n", + "and humanities. This paper introduces LayoutParser, an open-source\n", + "library for streamlining the usage of DL in DIA research and applica-\n", + "tions. The core LayoutParser library comes with a set of simple and\n", + "intuitive interfaces for applying and customizing DL models for layout de-\n", + "tection, character recognition, and many other document processing tasks.\n", + "To promote extensibility, LayoutParser also incorporates a community\n", + "platform for sharing both pre-trained models and full document digiti-\n", + "zation pipelines. We demonstrate that LayoutParser is helpful for both\n", + "lightweight and large-scale digitization pipelines in real-word use cases.\n", + "The library is publicly available at https://layout-parser.github.io.\n", + "Keywords: Document Image Analysis · Deep Learning · Layout Analysis\n", + "· Character Recognition · Open Source library · Toolkit.\n", + "1\n", + "Introduction\n", + "Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of\n", + "document image analysis (DIA) tasks including document image classification [11,\n", + "arXiv:2103.15348v2 [cs.CV] 21 Jun 2021\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'source': 'example_data/layout-parser-paper.pdf',\n", + " 'file_path': 'example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'format': 'PDF 1.5',\n", + " 'title': '',\n", + " 'author': '',\n", + " 'subject': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'trapped': '',\n", + " 'page': 0}\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders import FileSystemBlobLoader\n", + "from langchain_community.document_loaders.generic import GenericLoader\n", + "from langchain_community.document_loaders.parsers import PyMuPDFParser\n", + "\n", + "loader = GenericLoader(\n", + " blob_loader=FileSystemBlobLoader(\n", + " path=\"./example_data/\",\n", + " glob=\"*.pdf\",\n", + " ),\n", + " blob_parser=PyMuPDFParser(),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[0].page_content)\n", + "pprint.pp(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is possible to work with files from cloud storage." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import CloudBlobLoader\n", + "from langchain_community.document_loaders.generic import GenericLoader\n", + "\n", + "loader = GenericLoader(\n", + " blob_loader=CloudBlobLoader(\n", + " url=\"s3:/mybucket\", # Supports s3://, az://, gs://, file:// schemes.\n", + " glob=\"*.pdf\",\n", + " ),\n", + " blob_parser=PyMuPDFParser(),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[0].page_content)\n", + "pprint.pp(docs[0].metadata)" ] }, { @@ -157,13 +1277,13 @@ "source": [ "## API reference\n", "\n", - "For detailed documentation of all PyMuPDFLoader features and configurations head to the API reference: https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyMuPDFLoader.html" + "For detailed documentation of all `PyMuPDFLoader` features and configurations head to the API reference: https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyMuPDFLoader.html" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -177,9 +1297,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.12.7" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/libs/community/extended_testing_deps.txt b/libs/community/extended_testing_deps.txt index 975bbfb0b069b..e04a857ae17fd 100644 --- a/libs/community/extended_testing_deps.txt +++ b/libs/community/extended_testing_deps.txt @@ -60,12 +60,14 @@ oracle-ads>=2.9.1,<3 oracledb>=2.2.0,<3 pandas>=2.0.1,<3 pdfminer-six>=20221105,<20240706 +pdfplumber>=0.11 pgvector>=0.1.6,<0.2 playwright>=1.48.0,<2 praw>=7.7.1,<8 premai>=0.3.25,<0.4 psychicapi>=0.8.0,<0.9 pydantic>=2.7.4,<3 +pytesseract>=0.3.13 py-trello>=0.19.0,<0.20 pyjwt>=2.8.0,<3 pymupdf>=1.22.3,<2 diff --git a/libs/community/langchain_community/document_loaders/parsers/__init__.py b/libs/community/langchain_community/document_loaders/parsers/__init__.py index 13261622d18f8..9712718e19714 100644 --- a/libs/community/langchain_community/document_loaders/parsers/__init__.py +++ b/libs/community/langchain_community/document_loaders/parsers/__init__.py @@ -17,6 +17,12 @@ from langchain_community.document_loaders.parsers.html import ( BS4HTMLParser, ) + from langchain_community.document_loaders.parsers.images import ( + BaseImageBlobParser, + LLMImageBlobParser, + RapidOCRBlobParser, + TesseractBlobParser, + ) from langchain_community.document_loaders.parsers.language import ( LanguageParser, ) @@ -35,15 +41,19 @@ _module_lookup = { "AzureAIDocumentIntelligenceParser": "langchain_community.document_loaders.parsers.doc_intelligence", # noqa: E501 "BS4HTMLParser": "langchain_community.document_loaders.parsers.html", + "BaseImageBlobParser": "langchain_community.document_loaders.parsers.images", "DocAIParser": "langchain_community.document_loaders.parsers.docai", "GrobidParser": "langchain_community.document_loaders.parsers.grobid", "LanguageParser": "langchain_community.document_loaders.parsers.language", + "LLMImageBlobParser": "langchain_community.document_loaders.parsers.images", "OpenAIWhisperParser": "langchain_community.document_loaders.parsers.audio", "PDFMinerParser": "langchain_community.document_loaders.parsers.pdf", "PDFPlumberParser": "langchain_community.document_loaders.parsers.pdf", "PyMuPDFParser": "langchain_community.document_loaders.parsers.pdf", "PyPDFParser": "langchain_community.document_loaders.parsers.pdf", "PyPDFium2Parser": "langchain_community.document_loaders.parsers.pdf", + "RapidOCRBlobParser": "langchain_community.document_loaders.parsers.images", + "TesseractBlobParser": "langchain_community.document_loaders.parsers.images", "VsdxParser": "langchain_community.document_loaders.parsers.vsdx", } @@ -57,15 +67,19 @@ def __getattr__(name: str) -> Any: __all__ = [ "AzureAIDocumentIntelligenceParser", + "BaseImageBlobParser", "BS4HTMLParser", "DocAIParser", "GrobidParser", "LanguageParser", + "LLMImageBlobParser", "OpenAIWhisperParser", "PDFMinerParser", "PDFPlumberParser", "PyMuPDFParser", "PyPDFParser", "PyPDFium2Parser", + "RapidOCRBlobParser", + "TesseractBlobParser", "VsdxParser", ] diff --git a/libs/community/langchain_community/document_loaders/parsers/images.py b/libs/community/langchain_community/document_loaders/parsers/images.py new file mode 100644 index 0000000000000..b053b94d4915b --- /dev/null +++ b/libs/community/langchain_community/document_loaders/parsers/images.py @@ -0,0 +1,220 @@ +import base64 +import io +import logging +from abc import abstractmethod +from typing import TYPE_CHECKING, Iterable, Iterator + +import numpy +import numpy as np +from langchain_core.documents import Document +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import HumanMessage + +from langchain_community.document_loaders.base import BaseBlobParser +from langchain_community.document_loaders.blob_loaders import Blob + +if TYPE_CHECKING: + from PIL.Image import Image + +logger = logging.getLogger(__name__) + + +class BaseImageBlobParser(BaseBlobParser): + """Abstract base class for parsing image blobs into text.""" + + @abstractmethod + def _analyze_image(self, img: "Image") -> str: + """Abstract method to analyze an image and extract textual content. + + Args: + img: The image to be analyzed. + + Returns: + The extracted text content. + """ + + def lazy_parse(self, blob: Blob) -> Iterator[Document]: + """Lazily parse a blob and yields Documents containing the parsed content. + + Args: + blob (Blob): The blob to be parsed. + + Yields: + Document: + A document containing the parsed content and metadata. + """ + try: + from PIL import Image as Img + + with blob.as_bytes_io() as buf: + if blob.mimetype == "application/x-npy": + img = Img.fromarray(numpy.load(buf)) + else: + img = Img.open(buf) + content = self._analyze_image(img) + logger.debug("Image text: %s", content.replace("\n", "\\n")) + yield Document( + page_content=content, + metadata={**blob.metadata, **{"source": blob.source}}, + ) + except ImportError: + raise ImportError( + "`Pillow` package not found, please install it with " + "`pip install Pillow`" + ) + + +class RapidOCRBlobParser(BaseImageBlobParser): + """Parser for extracting text from images using the RapidOCR library. + + Attributes: + ocr: + The RapidOCR instance for performing OCR. + """ + + def __init__( + self, + ) -> None: + """ + Initializes the RapidOCRBlobParser. + """ + super().__init__() + self.ocr = None + + def _analyze_image(self, img: "Image") -> str: + """ + Analyzes an image and extracts text using RapidOCR. + + Args: + img (Image): + The image to be analyzed. + + Returns: + str: + The extracted text content. + """ + if not self.ocr: + try: + from rapidocr_onnxruntime import RapidOCR + + self.ocr = RapidOCR() + except ImportError: + raise ImportError( + "`rapidocr-onnxruntime` package not found, please install it with " + "`pip install rapidocr-onnxruntime`" + ) + ocr_result, _ = self.ocr(np.array(img)) # type: ignore + content = "" + if ocr_result: + content = ("\n".join([text[1] for text in ocr_result])).strip() + return content + + +class TesseractBlobParser(BaseImageBlobParser): + """Parse for extracting text from images using the Tesseract OCR library.""" + + def __init__( + self, + *, + langs: Iterable[str] = ("eng",), + ): + """Initialize the TesseractBlobParser. + + Args: + langs (list[str]): + The languages to use for OCR. + """ + super().__init__() + self.langs = list(langs) + + def _analyze_image(self, img: "Image") -> str: + """Analyze an image and extracts text using Tesseract OCR. + + Args: + img: The image to be analyzed. + + Returns: + str: The extracted text content. + """ + try: + import pytesseract + except ImportError: + raise ImportError( + "`pytesseract` package not found, please install it with " + "`pip install pytesseract`" + ) + return pytesseract.image_to_string(img, lang="+".join(self.langs)).strip() + + +_PROMPT_IMAGES_TO_DESCRIPTION: str = ( + "You are an assistant tasked with summarizing images for retrieval. " + "1. These summaries will be embedded and used to retrieve the raw image. " + "Give a concise summary of the image that is well optimized for retrieval\n" + "2. extract all the text from the image. " + "Do not exclude any content from the page.\n" + "Format answer in markdown without explanatory text " + "and without markdown delimiter ``` at the beginning. " +) + + +class LLMImageBlobParser(BaseImageBlobParser): + """Parser for analyzing images using a language model (LLM). + + Attributes: + model (BaseChatModel): + The language model to use for analysis. + prompt (str): + The prompt to provide to the language model. + """ + + def __init__( + self, + *, + model: BaseChatModel, + prompt: str = _PROMPT_IMAGES_TO_DESCRIPTION, + ): + """Initializes the LLMImageBlobParser. + + Args: + model (BaseChatModel): + The language model to use for analysis. + prompt (str): + The prompt to provide to the language model. + """ + super().__init__() + self.model = model + self.prompt = prompt + + def _analyze_image(self, img: "Image") -> str: + """Analyze an image using the provided language model. + + Args: + img: The image to be analyzed. + + Returns: + The extracted textual content. + """ + image_bytes = io.BytesIO() + img.save(image_bytes, format="PNG") + img_base64 = base64.b64encode(image_bytes.getvalue()).decode("utf-8") + msg = self.model.invoke( + [ + HumanMessage( + content=[ + { + "type": "text", + "text": self.prompt.format(format=format), + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{img_base64}" + }, + }, + ] + ) + ] + ) + result = msg.content + assert isinstance(result, str) + return result diff --git a/libs/community/langchain_community/document_loaders/parsers/pdf.py b/libs/community/langchain_community/document_loaders/parsers/pdf.py index 063f863869f87..254849df80273 100644 --- a/libs/community/langchain_community/document_loaders/parsers/pdf.py +++ b/libs/community/langchain_community/document_loaders/parsers/pdf.py @@ -2,12 +2,18 @@ from __future__ import annotations +import html +import io +import logging +import threading import warnings +from datetime import datetime from typing import ( TYPE_CHECKING, Any, Iterable, Iterator, + Literal, Mapping, Optional, Sequence, @@ -15,16 +21,21 @@ ) from urllib.parse import urlparse +import numpy import numpy as np from langchain_core.documents import Document from langchain_community.document_loaders.base import BaseBlobParser from langchain_community.document_loaders.blob_loaders import Blob +from langchain_community.document_loaders.parsers.images import ( + BaseImageBlobParser, + RapidOCRBlobParser, +) if TYPE_CHECKING: - import fitz import pdfminer import pdfplumber + import pymupdf import pypdf import pypdfium2 from textractor.data.text_linearization_config import TextLinearizationConfig @@ -78,6 +89,156 @@ def extract_from_images_with_rapidocr( return text +logger = logging.getLogger(__name__) + +_FORMAT_IMAGE_STR = "\n\n{image_text}\n\n" +_JOIN_IMAGES = "\n" +_JOIN_TABLES = "\n" +_DEFAULT_PAGES_DELIMITER = "\n\f" + +_STD_METADATA_KEYS = {"source", "total_pages", "creationdate", "creator", "producer"} + + +def _format_inner_image(blob: Blob, content: str, format: str) -> str: + """Format the content of the image with the source of the blob. + + blob: The blob containing the image. + format:: + The format for the parsed output. + - "text" = return the content as is + - "markdown-img" = wrap the content into an image markdown link, w/ link + pointing to (`![body)(#)`] + - "html-img" = wrap the content as the `alt` text of an tag and link to + (`{body}`) + """ + if content: + source = blob.source or "#" + if format == "markdown-img": + content = content.replace("]", r"\\]") + content = f"![{content}]({source})" + elif format == "html-img": + content = ( + f'{html.escape(content, quote=True)} ' f'src=' + ) + return content + + +def _validate_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + """Validate that the metadata has all the standard keys and the page is an integer. + + The standard keys are: + - source + - total_page + - creationdate + - creator + - producer + + Validate that page is an integer if it is present. + """ + if not _STD_METADATA_KEYS.issubset(metadata.keys()): + raise ValueError("The PDF parser must valorize the standard metadata.") + if not isinstance(metadata.get("page", 0), int): + raise ValueError("The PDF metadata page must be a integer.") + return metadata + + +def _purge_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + """Purge metadata from unwanted keys and normalize key names. + + Args: + metadata: The original metadata dictionary. + + Returns: + The cleaned and normalized the key format of metadata dictionary. + """ + new_metadata: dict[str, Any] = {} + map_key = { + "page_count": "total_pages", + "file_path": "source", + } + for k, v in metadata.items(): + if type(v) not in [str, int]: + v = str(v) + if k.startswith("/"): + k = k[1:] + k = k.lower() + if k in ["creationdate", "moddate"]: + try: + new_metadata[k] = datetime.strptime( + v.replace("'", ""), "D:%Y%m%d%H%M%S%z" + ).isoformat("T") + except ValueError: + new_metadata[k] = v + elif k in map_key: + # Normalize key with others PDF parser + new_metadata[map_key[k]] = v + new_metadata[k] = v + elif isinstance(v, str): + new_metadata[k] = v.strip() + elif isinstance(v, int): + new_metadata[k] = v + return new_metadata + + +_PARAGRAPH_DELIMITER = [ + "\n\n\n", + "\n\n", +] # To insert images or table in the middle of the page. + + +def _merge_text_and_extras(extras: list[str], text_from_page: str) -> str: + """Insert extras such as image/table in a text between two paragraphs if possible, + else at the end of the text. + + Args: + extras: List of extra content (images/tables) to insert. + text_from_page: The text content from the page. + + Returns: + The merged text with extras inserted. + """ + + def _recurs_merge_text_and_extras( + extras: list[str], text_from_page: str, recurs: bool + ) -> Optional[str]: + if extras: + for delim in _PARAGRAPH_DELIMITER: + pos = text_from_page.rfind(delim) + if pos != -1: + # search penultimate, to bypass an error in footer + previous_text = None + if recurs: + previous_text = _recurs_merge_text_and_extras( + extras, text_from_page[:pos], False + ) + if previous_text: + all_text = previous_text + text_from_page[pos:] + else: + all_extras = "" + str_extras = "\n\n".join(filter(lambda x: x, extras)) + if str_extras: + all_extras = delim + str_extras + all_text = ( + text_from_page[:pos] + all_extras + text_from_page[pos:] + ) + break + else: + all_text = None + else: + all_text = text_from_page + return all_text + + all_text = _recurs_merge_text_and_extras(extras, text_from_page, True) + if not all_text: + all_extras = "" + str_extras = "\n\n".join(filter(lambda x: x, extras)) + if str_extras: + all_extras = _PARAGRAPH_DELIMITER[-1] + str_extras + all_text = text_from_page + all_extras + + return all_text + + class PyPDFParser(BaseBlobParser): """Load `PDF` using `pypdf`""" @@ -105,9 +266,7 @@ def lazy_parse(self, blob: Blob) -> Iterator[Document]: # type: ignore[valid-ty ) def _extract_text_from_page(page: pypdf.PageObject) -> str: - """ - Extract text from image given the version of pypdf. - """ + """Extract text from image given the version of pypdf.""" if pypdf.__version__.startswith("3"): return page.extract_text() else: @@ -275,92 +434,363 @@ def get_image(layout_object: Any) -> Any: class PyMuPDFParser(BaseBlobParser): - """Parse `PDF` using `PyMuPDF`.""" + """Parse a blob from a PDF using `PyMuPDF` library. + + This class provides methods to parse a blob from a PDF document, supporting various + configurations such as handling password-protected PDFs, extracting images, and + defining extraction mode. + It integrates the 'PyMuPDF' library for PDF processing and offers synchronous blob + parsing. + + Examples: + Setup: + + .. code-block:: bash + + pip install -U langchain-community pymupdf + + Load a blob from a PDF file: + + .. code-block:: python + + from langchain_core.documents.base import Blob + + blob = Blob.from_path("./example_data/layout-parser-paper.pdf") + + Instantiate the parser: + + .. code-block:: python + + from langchain_community.document_loaders.parsers import PyMuPDFParser + + parser = PyMuPDFParser( + # password = None, + mode = "single", + pages_delimiter = "\n\f", + # extract_images = True, + # images_parser = TesseractBlobParser(), + # extract_tables="markdown", + # extract_tables_settings=None, + # text_kwargs=None, + ) + + Lazily parse the blob: + + .. code-block:: python + + docs = [] + docs_lazy = parser.lazy_parse(blob) + + for doc in docs_lazy: + docs.append(doc) + print(docs[0].page_content[:100]) + print(docs[0].metadata) + """ + + # PyMuPDF is not thread safe. + # See https://pymupdf.readthedocs.io/en/latest/recipes-multiprocessing.html + _lock = threading.Lock() def __init__( self, - text_kwargs: Optional[Mapping[str, Any]] = None, + text_kwargs: Optional[dict[str, Any]] = None, extract_images: bool = False, + *, + password: Optional[str] = None, + mode: Literal["single", "page"] = "page", + pages_delimiter: str = _DEFAULT_PAGES_DELIMITER, + images_parser: Optional[BaseImageBlobParser] = None, + images_inner_format: Literal["text", "markdown-img", "html-img"] = "text", + extract_tables: Union[Literal["csv", "markdown", "html"], None] = None, + extract_tables_settings: Optional[dict[str, Any]] = None, ) -> None: - """Initialize the parser. + """Initialize a parser based on PyMuPDF. Args: - text_kwargs: Keyword arguments to pass to ``fitz.Page.get_text()``. + password: Optional password for opening encrypted PDFs. + mode: The extraction mode, either "single" for the entire document or "page" + for page-wise extraction. + pages_delimiter: A string delimiter to separate pages in single-mode + extraction. + extract_images: Whether to extract images from the PDF. + images_parser: Optional image blob parser. + images_inner_format: The format for the parsed output. + - "text" = return the content as is + - "markdown-img" = wrap the content into an image markdown link, w/ link + pointing to (`![body)(#)`] + - "html-img" = wrap the content as the `alt` text of an tag and link to + (`{body}`) + extract_tables: Whether to extract tables in a specific format, such as + "csv", "markdown", or "html". + extract_tables_settings: Optional dictionary of settings for customizing + table extraction. + + Returns: + This method does not directly return data. Use the `parse` or `lazy_parse` + methods to retrieve parsed documents with content and metadata. + + Raises: + ValueError: If the mode is not "single" or "page". + ValueError: If the extract_tables format is not "markdown", "html", + or "csv". """ + super().__init__() + if mode not in ["single", "page"]: + raise ValueError("mode must be single or page") + if extract_tables and extract_tables not in ["markdown", "html", "csv"]: + raise ValueError("mode must be markdown") + + self.mode = mode + self.pages_delimiter = pages_delimiter + self.password = password self.text_kwargs = text_kwargs or {} + if extract_images and not images_parser: + images_parser = RapidOCRBlobParser() self.extract_images = extract_images + self.images_inner_format = images_inner_format + self.images_parser = images_parser + self.extract_tables = extract_tables + self.extract_tables_settings = extract_tables_settings def lazy_parse(self, blob: Blob) -> Iterator[Document]: # type: ignore[valid-type] - """Lazily parse the blob.""" + return self._lazy_parse( + blob, + ) - import fitz + def _lazy_parse( + self, + blob: Blob, + # text-kwargs is present for backwards compatibility. + # Users should not use it directly. + text_kwargs: Optional[dict[str, Any]] = None, + ) -> Iterator[Document]: # type: ignore[valid-type] + """Lazily parse the blob. + Insert image, if possible, between two paragraphs. + In this way, a paragraph can be continued on the next page. - with blob.as_bytes_io() as file_path: # type: ignore[attr-defined] - if blob.data is None: # type: ignore[attr-defined] - doc = fitz.open(file_path) - else: - doc = fitz.open(stream=file_path, filetype="pdf") + Args: + blob: The blob to parse. + text_kwargs: Optional keyword arguments to pass to the `get_text` method. + If provided at run time, it will override the default text_kwargs. - yield from [ - Document( - page_content=self._get_page_content(doc, page, blob), - metadata=self._extract_metadata(doc, page, blob), - ) - for page in doc - ] + Raises: + ImportError: If the `pypdf` package is not found. - def _get_page_content(self, doc: fitz.Document, page: fitz.Page, blob: Blob) -> str: + Yield: + An iterator over the parsed documents. """ - Get the text of the page using PyMuPDF and RapidOCR and issue a warning + try: + import pymupdf + + text_kwargs = text_kwargs or self.text_kwargs + if not self.extract_tables_settings: + from pymupdf.table import ( + DEFAULT_JOIN_TOLERANCE, + DEFAULT_MIN_WORDS_HORIZONTAL, + DEFAULT_MIN_WORDS_VERTICAL, + DEFAULT_SNAP_TOLERANCE, + ) + + self.extract_tables_settings = { + # See https://pymupdf.readthedocs.io/en/latest/page.html#Page.find_tables + "clip": None, + "vertical_strategy": "lines", + "horizontal_strategy": "lines", + "vertical_lines": None, + "horizontal_lines": None, + "snap_tolerance": DEFAULT_SNAP_TOLERANCE, + "snap_x_tolerance": None, + "snap_y_tolerance": None, + "join_tolerance": DEFAULT_JOIN_TOLERANCE, + "join_x_tolerance": None, + "join_y_tolerance": None, + "edge_min_length": 3, + "min_words_vertical": DEFAULT_MIN_WORDS_VERTICAL, + "min_words_horizontal": DEFAULT_MIN_WORDS_HORIZONTAL, + "intersection_tolerance": 3, + "intersection_x_tolerance": None, + "intersection_y_tolerance": None, + "text_tolerance": 3, + "text_x_tolerance": 3, + "text_y_tolerance": 3, + "strategy": None, # offer abbreviation + "add_lines": None, # optional user-specified lines + } + except ImportError: + raise ImportError( + "pymupdf package not found, please install it " + "with `pip install pymupdf`" + ) + + with PyMuPDFParser._lock: + with blob.as_bytes_io() as file_path: # type: ignore[attr-defined] + if blob.data is None: # type: ignore[attr-defined] + doc = pymupdf.open(file_path) + else: + doc = pymupdf.open(stream=file_path, filetype="pdf") + if doc.is_encrypted: + doc.authenticate(self.password) + doc_metadata = self._extract_metadata(doc, blob) + full_content = [] + for page in doc: + all_text = self._get_page_content(doc, page, text_kwargs).strip() + if self.mode == "page": + yield Document( + page_content=all_text, + metadata=_validate_metadata( + doc_metadata | {"page": page.number} + ), + ) + else: + full_content.append(all_text) + + if self.mode == "single": + yield Document( + page_content=self.pages_delimiter.join(full_content), + metadata=_validate_metadata(doc_metadata), + ) + + def _get_page_content( + self, + doc: pymupdf.Document, + page: pymupdf.Page, + text_kwargs: dict[str, Any], + ) -> str: + """Get the text of the page using PyMuPDF and RapidOCR and issue a warning if it is empty. + + Args: + doc: The PyMuPDF document object. + page: The PyMuPDF page object. + blob: The blob being parsed. + + Returns: + str: The text content of the page. """ - content = page.get_text(**self.text_kwargs) + self._extract_images_from_page( - doc, page - ) + text_from_page = page.get_text(**{**self.text_kwargs, **text_kwargs}) + images_from_page = self._extract_images_from_page(doc, page) + tables_from_page = self._extract_tables_from_page(page) + extras = [] + if images_from_page: + extras.append(images_from_page) + if tables_from_page: + extras.append(tables_from_page) + all_text = _merge_text_and_extras(extras, text_from_page) - if not content: - warnings.warn( - f"Warning: Empty content on page " - f"{page.number} of document {blob.source}" - ) + return all_text - return content - - def _extract_metadata( - self, doc: fitz.Document, page: fitz.Page, blob: Blob - ) -> dict: - """Extract metadata from the document and page.""" - return dict( - { - "source": blob.source, # type: ignore[attr-defined] - "file_path": blob.source, # type: ignore[attr-defined] - "page": page.number, - "total_pages": len(doc), - }, - **{ - k: doc.metadata[k] - for k in doc.metadata - if isinstance(doc.metadata[k], (str, int)) - }, + def _extract_metadata(self, doc: pymupdf.Document, blob: Blob) -> dict: + """Extract metadata from the document and page. + + Args: + doc: The PyMuPDF document object. + blob: The blob being parsed. + + Returns: + dict: The extracted metadata. + """ + return _purge_metadata( + dict( + { + "producer": "PyMuPDF", + "creator": "PyMuPDF", + "creationdate": "", + "source": blob.source, # type: ignore[attr-defined] + "file_path": blob.source, # type: ignore[attr-defined] + "total_pages": len(doc), + }, + **{ + k: doc.metadata[k] + for k in doc.metadata + if isinstance(doc.metadata[k], (str, int)) + }, + ) ) - def _extract_images_from_page(self, doc: fitz.Document, page: fitz.Page) -> str: - """Extract images from page and get the text with RapidOCR.""" - if not self.extract_images: + def _extract_images_from_page( + self, doc: pymupdf.Document, page: pymupdf.Page + ) -> str: + """Extract images from a PDF page and get the text using images_to_text. + + Args: + doc: The PyMuPDF document object. + page: The PyMuPDF page object. + + Returns: + str: The extracted text from the images on the page. + """ + if not self.images_parser: return "" - import fitz + import pymupdf img_list = page.get_images() - imgs = [] + images = [] for img in img_list: - xref = img[0] - pix = fitz.Pixmap(doc, xref) - imgs.append( - np.frombuffer(pix.samples, dtype=np.uint8).reshape( + if self.images_parser: + xref = img[0] + pix = pymupdf.Pixmap(doc, xref) + image = np.frombuffer(pix.samples, dtype=np.uint8).reshape( pix.height, pix.width, -1 ) - ) - return extract_from_images_with_rapidocr(imgs) + image_bytes = io.BytesIO() + numpy.save(image_bytes, image) + blob = Blob.from_data( + image_bytes.getvalue(), mime_type="application/x-npy" + ) + image_text = next(self.images_parser.lazy_parse(blob)).page_content + + images.append( + _format_inner_image(blob, image_text, self.images_inner_format) + ) + return _FORMAT_IMAGE_STR.format( + image_text=_JOIN_IMAGES.join(filter(None, images)) + ) + + def _extract_tables_from_page(self, page: pymupdf.Page) -> str: + """Extract tables from a PDF page. + + Args: + page: The PyMuPDF page object. + + Returns: + str: The extracted tables in the specified format. + """ + if self.extract_tables is None: + return "" + import pymupdf + + tables_list = list( + pymupdf.table.find_tables(page, **self.extract_tables_settings) + ) + if tables_list: + if self.extract_tables == "markdown": + return _JOIN_TABLES.join([table.to_markdown() for table in tables_list]) + elif self.extract_tables == "html": + return _JOIN_TABLES.join( + [ + table.to_pandas().to_html( + header=False, + index=False, + bold_rows=False, + ) + for table in tables_list + ] + ) + elif self.extract_tables == "csv": + return _JOIN_TABLES.join( + [ + table.to_pandas().to_csv( + header=False, + index=False, + ) + for table in tables_list + ] + ) + else: + raise ValueError( + f"extract_tables {self.extract_tables} not implemented" + ) + return "" class PyPDFium2Parser(BaseBlobParser): diff --git a/libs/community/langchain_community/document_loaders/pdf.py b/libs/community/langchain_community/document_loaders/pdf.py index 25b3e72a3d8a1..af0aa86b4b5b3 100644 --- a/libs/community/langchain_community/document_loaders/pdf.py +++ b/libs/community/langchain_community/document_loaders/pdf.py @@ -12,6 +12,7 @@ Any, BinaryIO, Iterator, + Literal, Mapping, Optional, Sequence, @@ -27,7 +28,9 @@ from langchain_community.document_loaders.base import BaseLoader from langchain_community.document_loaders.blob_loaders import Blob from langchain_community.document_loaders.dedoc import DedocBaseLoader +from langchain_community.document_loaders.parsers.images import BaseImageBlobParser from langchain_community.document_loaders.parsers.pdf import ( + _DEFAULT_PAGES_DELIMITER, AmazonTextractPDFParser, DocumentIntelligenceParser, PDFMinerParser, @@ -113,7 +116,8 @@ def __init__( if "~" in self.file_path: self.file_path = os.path.expanduser(self.file_path) - # If the file is a web path or S3, download it to a temporary file, and use that + # If the file is a web path or S3, download it to a temporary file, + # and use that. It's better to use a BlobLoader. if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path): self.temp_dir = tempfile.TemporaryDirectory() _, suffix = os.path.splitext(self.file_path) @@ -180,8 +184,7 @@ def load(self) -> list[Document]: class PyPDFLoader(BasePDFLoader): - """ - PyPDFLoader document loader integration + """PyPDFLoader document loader integration Setup: Install ``langchain-community``. @@ -429,44 +432,139 @@ def lazy_load(self) -> Iterator[Document]: class PyMuPDFLoader(BasePDFLoader): - """Load `PDF` files using `PyMuPDF`.""" + """Load and parse a PDF file using 'PyMuPDF' library. + + This class provides methods to load and parse PDF documents, supporting various + configurations such as handling password-protected files, extracting tables, + extracting images, and defining extraction mode. It integrates the `PyMuPDF` + library for PDF processing and offers both synchronous and asynchronous document + loading. + + Examples: + Setup: + + .. code-block:: bash + + pip install -U langchain-community pymupdf + + Instantiate the loader: + + .. code-block:: python + + from langchain_community.document_loaders import PyMuPDFLoader + + loader = PyMuPDFLoader( + file_path = "./example_data/layout-parser-paper.pdf", + # headers = None + # password = None, + mode = "single", + pages_delimiter = "\n\f", + # extract_images = True, + # images_parser = TesseractBlobParser(), + # extract_tables = "markdown", + # extract_tables_settings = None, + ) + + Lazy load documents: + + .. code-block:: python + + docs = [] + docs_lazy = loader.lazy_load() + + for doc in docs_lazy: + docs.append(doc) + print(docs[0].page_content[:100]) + print(docs[0].metadata) + + Load documents asynchronously: + + .. code-block:: python + + docs = await loader.aload() + print(docs[0].page_content[:100]) + print(docs[0].metadata) + """ def __init__( self, file_path: Union[str, PurePath], *, - headers: Optional[dict] = None, + password: Optional[str] = None, + mode: Literal["single", "page"] = "page", + pages_delimiter: str = _DEFAULT_PAGES_DELIMITER, extract_images: bool = False, + images_parser: Optional[BaseImageBlobParser] = None, + images_inner_format: Literal["text", "markdown-img", "html-img"] = "text", + extract_tables: Union[Literal["csv", "markdown", "html"], None] = None, + headers: Optional[dict] = None, + extract_tables_settings: Optional[dict[str, Any]] = None, **kwargs: Any, ) -> None: - """Initialize with a file path.""" - try: - import fitz # noqa:F401 - except ImportError: - raise ImportError( - "`PyMuPDF` package not found, please install it with " - "`pip install pymupdf`" - ) + """Initialize with a file path. + + Args: + file_path: The path to the PDF file to be loaded. + headers: Optional headers to use for GET request to download a file from a + web path. + password: Optional password for opening encrypted PDFs. + mode: The extraction mode, either "single" for the entire document or "page" + for page-wise extraction. + pages_delimiter: A string delimiter to separate pages in single-mode + extraction. + extract_images: Whether to extract images from the PDF. + images_parser: Optional image blob parser. + images_inner_format: The format for the parsed output. + - "text" = return the content as is + - "markdown-img" = wrap the content into an image markdown link, w/ link + pointing to (`![body)(#)`] + - "html-img" = wrap the content as the `alt` text of an tag and link to + (`{body}`) + extract_tables: Whether to extract tables in a specific format, such as + "csv", "markdown", or "html". + extract_tables_settings: Optional dictionary of settings for customizing + table extraction. + **kwargs: Additional keyword arguments for customizing text extraction + behavior. + + Returns: + This method does not directly return data. Use the `load`, `lazy_load`, or + `aload` methods to retrieve parsed documents with content and metadata. + + Raises: + ValueError: If the `mode` argument is not one of "single" or "page". + """ + if mode not in ["single", "page"]: + raise ValueError("mode must be single or page") super().__init__(file_path, headers=headers) - self.extract_images = extract_images - self.text_kwargs = kwargs + self.parser = PyMuPDFParser( + password=password, + mode=mode, + pages_delimiter=pages_delimiter, + text_kwargs=kwargs, + extract_images=extract_images, + images_parser=images_parser, + images_inner_format=images_inner_format, + extract_tables=extract_tables, + extract_tables_settings=extract_tables_settings, + ) def _lazy_load(self, **kwargs: Any) -> Iterator[Document]: + """Lazy load given path as pages or single document (see `mode`). + Insert image, if possible, between two paragraphs. + In this way, a paragraph can be continued on the next page. + """ if kwargs: logger.warning( f"Received runtime arguments {kwargs}. Passing runtime args to `load`" f" is deprecated. Please pass arguments during initialization instead." ) - - text_kwargs = {**self.text_kwargs, **kwargs} - parser = PyMuPDFParser( - text_kwargs=text_kwargs, extract_images=self.extract_images - ) + parser = self.parser if self.web_path: blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path) # type: ignore[attr-defined] else: blob = Blob.from_path(self.file_path) # type: ignore[attr-defined] - yield from parser.lazy_parse(blob) + yield from parser._lazy_parse(blob, text_kwargs=kwargs) def load(self, **kwargs: Any) -> list[Document]: return list(self._lazy_load(**kwargs)) @@ -772,8 +870,8 @@ def lazy_load( ) -> Iterator[Document]: """Lazy load documents""" # the self.file_path is local, but the blob has to include - # the S3 location if the file originated from S3 for multi-page documents - # raises ValueError when multi-page and not on S3""" + # the S3 location if the file originated from S3 for multipage documents + # raises ValueError when multipage and not on S3""" if self.web_path and self._is_s3_url(self.web_path): blob = Blob(path=self.web_path) # type: ignore[call-arg] # type: ignore[misc] @@ -818,8 +916,7 @@ def _get_number_of_pages(blob: Blob) -> int: # type: ignore[valid-type] class DedocPDFLoader(DedocBaseLoader): - """ - DedocPDFLoader document loader integration to load PDF files using `dedoc`. + """DedocPDFLoader document loader integration to load PDF files using `dedoc`. The file loader can automatically detect the correctness of a textual layer in the PDF document. Note that `__init__` method supports parameters that differ from ones of @@ -925,8 +1022,7 @@ def __init__( model: str = "prebuilt-document", headers: Optional[dict] = None, ) -> None: - """ - Initialize the object for file processing with Azure Document Intelligence + """Initialize the object for file processing with Azure Document Intelligence (formerly Form Recognizer). This constructor initializes a DocumentIntelligenceParser object to be used @@ -968,11 +1064,10 @@ def lazy_load( class ZeroxPDFLoader(BasePDFLoader): - """ - Document loader utilizing Zerox library: + """Document loader utilizing Zerox library: https://github.com/getomni-ai/zerox - Zerox converts PDF document to serties of images (page-wise) and + Zerox converts PDF document to series of images (page-wise) and uses vision-capable LLM model to generate Markdown representation. Zerox utilizes anyc operations. Therefore when using this loader @@ -991,9 +1086,8 @@ def __init__( **zerox_kwargs: Any, ) -> None: super().__init__(file_path=file_path) - """ - Initialize the parser with arguments to be passed to the zerox function. - Make sure to set necessary environmnet variables such as API key, endpoint, etc. + """Initialize the parser with arguments to be passed to the zerox function. + Make sure to set necessary environment variables such as API key, endpoint, etc. Check zerox documentation for list of necessary environment variables for any given model. @@ -1014,13 +1108,7 @@ def __init__( self.model = model def lazy_load(self) -> Iterator[Document]: - """ - Loads documnts from pdf utilizing zerox library: - https://github.com/getomni-ai/zerox - - Returns: - Iterator[Document]: An iterator over parsed Document instances. - """ + """Lazily load pages.""" import asyncio from pyzerox import zerox diff --git a/libs/community/tests/integration_tests/document_loaders/parsers/test_images.py b/libs/community/tests/integration_tests/document_loaders/parsers/test_images.py new file mode 100644 index 0000000000000..e6d71fae69240 --- /dev/null +++ b/libs/community/tests/integration_tests/document_loaders/parsers/test_images.py @@ -0,0 +1,60 @@ +import re +from pathlib import Path +from typing import Any, Type + +import pytest +from langchain_core.documents.base import Blob +from langchain_core.language_models import FakeMessagesListChatModel +from langchain_core.messages import ChatMessage + +from langchain_community.document_loaders.parsers.images import ( + LLMImageBlobParser, + RapidOCRBlobParser, + TesseractBlobParser, +) + +path_base = Path(__file__).parent.parent.parent +building_image = Blob.from_path(path_base / "examples/building.jpg") +text_image = Blob.from_path(path_base / "examples/text.png") +page_image = Blob.from_path(path_base / "examples/page.png") + + +@pytest.mark.parametrize( + "blob,body", + [ + (building_image, ""), + (text_image, r"(?ms).*MAKE.*TEXT.*STAND.*OUT.*FROM.*BACKGROUNDS.*"), + ], +) +@pytest.mark.parametrize( + "blob_loader,kw", + [ + (RapidOCRBlobParser, {}), + (TesseractBlobParser, {}), + ( + LLMImageBlobParser, + { + "model": FakeMessagesListChatModel( + responses=[ + ChatMessage( + id="ai1", + role="system", + content="A building. MAKE TEXT STAND OUT FROM BACKGROUNDS", + ), + ] + ) + }, + ), + ], +) +def test_image_parser_with_differents_files( + blob_loader: Type, + kw: dict[str, Any], + blob: Blob, + body: str, +) -> None: + if blob_loader == LLMImageBlobParser and "building" in str(blob.path): + body = ".*building.*" + documents = list(blob_loader(**kw).lazy_parse(blob)) + assert len(documents) == 1 + assert re.compile(body).match(documents[0].page_content) diff --git a/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py b/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py index 47163c859520b..44cc8294643f1 100644 --- a/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py +++ b/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py @@ -1,18 +1,26 @@ """Tests for the various PDF parsers.""" +import re from pathlib import Path -from typing import Iterator +from typing import TYPE_CHECKING, Iterator +import pytest + +import langchain_community.document_loaders.parsers as pdf_parsers from langchain_community.document_loaders.base import BaseBlobParser from langchain_community.document_loaders.blob_loaders import Blob -from langchain_community.document_loaders.parsers.pdf import ( +from langchain_community.document_loaders.parsers import ( + BaseImageBlobParser, PDFMinerParser, PDFPlumberParser, - PyMuPDFParser, PyPDFium2Parser, PyPDFParser, ) +if TYPE_CHECKING: + from PIL.Image import Image + + # PDFs to test parsers on. HELLO_PDF = Path(__file__).parent.parent.parent / "examples" / "hello.pdf" @@ -20,6 +28,12 @@ Path(__file__).parent.parent.parent / "examples" / "layout-parser-paper.pdf" ) +LAYOUT_PARSER_PAPER_PASSWORD_PDF = ( + Path(__file__).parent.parent.parent + / "examples" + / "layout-parser-paper-password.pdf" +) + DUPLICATE_CHARS = ( Path(__file__).parent.parent.parent / "examples" / "duplicate-chars.pdf" ) @@ -41,7 +55,7 @@ def _assert_with_parser(parser: BaseBlobParser, splits_by_page: bool = True) -> assert isinstance(page_content, str) # The different parsers return different amount of whitespace, so using # startswith instead of equals. - assert docs[0].page_content.startswith("Hello world!") + assert re.findall(r"Hello\s+world!", docs[0].page_content) blob = Blob.from_path(LAYOUT_PARSER_PAPER_PDF) doc_generator = parser.lazy_parse(blob) @@ -84,11 +98,6 @@ def _assert_with_duplicate_parser(parser: BaseBlobParser, dedupe: bool = False) assert "11000000 SSeerriieess" == docs[0].page_content.split("\n")[0] -def test_pymupdf_loader() -> None: - """Test PyMuPDF loader.""" - _assert_with_parser(PyMuPDFParser()) - - def test_pypdf_parser() -> None: """Test PyPDF parser.""" _assert_with_parser(PyPDFParser()) @@ -123,11 +132,210 @@ def test_extract_images_text_from_pdf_pdfminerparser() -> None: _assert_with_parser(PDFMinerParser(extract_images=True)) -def test_extract_images_text_from_pdf_pymupdfparser() -> None: - """Test extract image from pdf and recognize text with rapid ocr - PyMuPDFParser""" - _assert_with_parser(PyMuPDFParser(extract_images=True)) - - def test_extract_images_text_from_pdf_pypdfium2parser() -> None: """Test extract image from pdf and recognize text with rapid ocr - PyPDFium2Parser""" # noqa: E501 _assert_with_parser(PyPDFium2Parser(extract_images=True)) + + +class EmptyImageBlobParser(BaseImageBlobParser): + def _analyze_image(self, img: "Image") -> str: + return "Hello world" + + +@pytest.mark.parametrize( + "mode,image_parser", + [("single", EmptyImageBlobParser()), ("page", None)], +) +@pytest.mark.parametrize( + "parser_factory,params", + [ + ("PyMuPDFParser", {}), + ], +) +@pytest.mark.requires("pillow") +def test_mode_and_extract_images_variations( + parser_factory: str, + params: dict, + mode: str, + image_parser: BaseImageBlobParser, +) -> None: + _test_matrix( + parser_factory, + params, + mode, + image_parser, + images_inner_format="text", + ) + + +@pytest.mark.parametrize( + "images_inner_format", + ["text", "markdown-img", "html-img"], +) +@pytest.mark.parametrize( + "parser_factory,params", + [ + ("PyMuPDFParser", {}), + ], +) +@pytest.mark.requires("pillow") +def test_mode_and_image_formats_variations( + parser_factory: str, + params: dict, + images_inner_format: str, +) -> None: + mode = "single" + image_parser = EmptyImageBlobParser() + + _test_matrix( + parser_factory, + params, + mode, + image_parser, + images_inner_format, + ) + + +def _test_matrix( + parser_factory: str, + params: dict, + mode: str, + image_parser: BaseImageBlobParser, + images_inner_format: str, +) -> None: + """Apply the same test for all *standard* PDF parsers. + + - Try with mode `single` and `page` + - Try with image_parser `None` or others + """ + + def _std_assert_with_parser(parser: BaseBlobParser) -> None: + """Standard tests to verify that the given parser works. + + Args: + parser (BaseBlobParser): The parser to test. + """ + blob = Blob.from_path(LAYOUT_PARSER_PAPER_PDF) + doc_generator = parser.lazy_parse(blob) + docs = list(doc_generator) + metadata = docs[0].metadata + assert metadata["source"] == str(LAYOUT_PARSER_PAPER_PDF) + assert "creationdate" in metadata + assert "creator" in metadata + assert "producer" in metadata + assert "total_pages" in metadata + if len(docs) > 1: + assert metadata["page"] == 0 + if hasattr(parser, "extract_images") and parser.extract_images: + images = [] + for doc in docs: + _HTML_image = ( + r"]*" + r'src="([^"]+)"(?:\s+alt="([^"]*)")?(?:\s+' + r'title="([^"]*)")?[^>]*>' + ) + _markdown_image = r"!\[([^\]]*)\]\(([^)\s]+)(?:\s+\"([^\"]+)\")?\)" + match = re.findall(_markdown_image, doc.page_content) + if match: + images.extend(match) + assert len(images) >= 1 + + if hasattr(parser, "password"): + old_password = parser.password + parser.password = "password" + blob = Blob.from_path(LAYOUT_PARSER_PAPER_PASSWORD_PDF) + doc_generator = parser.lazy_parse(blob) + docs = list(doc_generator) + assert len(docs) + parser.password = old_password + + parser_class = getattr(pdf_parsers, parser_factory) + + parser = parser_class( + mode=mode, + images_parser=image_parser, + images_inner_format=images_inner_format, + **params, + ) + _assert_with_parser(parser, splits_by_page=(mode == "page")) + _std_assert_with_parser(parser) + + +@pytest.mark.parametrize( + "mode", + ["single", "page"], +) +@pytest.mark.parametrize( + "extract_tables", + ["markdown", "html", "csv", None], +) +@pytest.mark.parametrize( + "parser_factory,params", + [ + ("PyMuPDFParser", {}), + ], +) +def test_parser_with_table( + parser_factory: str, + params: dict, + mode: str, + extract_tables: str, +) -> None: + from PIL.Image import Image + + from langchain_community.document_loaders.parsers.images import BaseImageBlobParser + + def _std_assert_with_parser(parser: BaseBlobParser) -> None: + """Standard tests to verify that the given parser works. + + Args: + parser (BaseBlobParser): The parser to test. + """ + blob = Blob.from_path(LAYOUT_PARSER_PAPER_PDF) + doc_generator = parser.lazy_parse(blob) + docs = list(doc_generator) + tables = [] + for doc in docs: + if extract_tables == "markdown": + pattern = ( + r"(?s)(" + r"(?:(?:[^\n]*\|)\n)" + r"(?:\|(?:\s?:?---*:?\s?\|)+)\n" + r"(?:(?:[^\n]*\|)\n)+" + r")" + ) + elif extract_tables == "html": + pattern = r"(?s)(]*>(?:.*?)<\/table>)" + elif extract_tables == "csv": + pattern = ( + r"((?:(?:" + r'(?:"(?:[^"]*(?:""[^"]*)*)"' + r"|[^\n,]*),){2,}" + r"(?:" + r'(?:"(?:[^"]*(?:""[^"]*)*)"' + r"|[^\n]*))\n){2,})" + ) + else: + pattern = None + if pattern: + matches = re.findall(pattern, doc.page_content) + if matches: + tables.extend(matches) + if extract_tables: + assert len(tables) >= 1 + else: + assert not len(tables) + + class EmptyImageBlobParser(BaseImageBlobParser): + def _analyze_image(self, img: Image) -> str: + return "![image](.)" + + parser_class = getattr(pdf_parsers, parser_factory) + + parser = parser_class( + mode=mode, + extract_tables=extract_tables, + images_parser=EmptyImageBlobParser(), + **params, + ) + _std_assert_with_parser(parser) diff --git a/libs/community/tests/integration_tests/document_loaders/test_pdf.py b/libs/community/tests/integration_tests/document_loaders/test_pdf.py index 66ce4ce3fa587..7eae7ef710d42 100644 --- a/libs/community/tests/integration_tests/document_loaders/test_pdf.py +++ b/libs/community/tests/integration_tests/document_loaders/test_pdf.py @@ -4,12 +4,12 @@ import pytest +import langchain_community.document_loaders as pdf_loaders from langchain_community.document_loaders import ( AmazonTextractPDFLoader, MathpixPDFLoader, PDFMinerLoader, PDFMinerPDFasHTMLLoader, - PyMuPDFLoader, PyPDFium2Loader, UnstructuredPDFLoader, ) @@ -100,30 +100,6 @@ def test_pypdfium2_loader() -> None: assert len(docs) == 16 -def test_pymupdf_loader() -> None: - """Test PyMuPDF loader.""" - file_path = Path(__file__).parent.parent / "examples/hello.pdf" - loader = PyMuPDFLoader(file_path) - - docs = loader.load() - assert len(docs) == 1 - - file_path = Path(__file__).parent.parent / "examples/layout-parser-paper.pdf" - loader = PyMuPDFLoader(file_path) - - docs = loader.load() - assert len(docs) == 16 - assert loader.web_path is None - - web_path = "https://people.sc.fsu.edu/~jpeterson/hello_world.pdf" - loader = PyMuPDFLoader(web_path) - - docs = loader.load() - assert loader.web_path == web_path - assert loader.file_path != web_path - assert len(docs) == 1 - - @pytest.mark.skipif( not os.environ.get("MATHPIX_API_KEY"), reason="Mathpix API key not found" ) @@ -230,3 +206,51 @@ def test_amazontextract_loader_failures() -> None: loader = AmazonTextractPDFLoader(two_page_pdf) with pytest.raises(ValueError): loader.load() + + +@pytest.mark.parametrize( + "parser_factory,params", + [ + ("PyMuPDFLoader", {}), + ], +) +def test_standard_parameters( + parser_factory: str, + params: dict, +) -> None: + loader_class = getattr(pdf_loaders, parser_factory) + + file_path = Path(__file__).parent.parent / "examples/hello.pdf" + loader = loader_class(file_path) + docs = loader.load() + assert len(docs) == 1 + + file_path = Path(__file__).parent.parent / "examples/layout-parser-paper.pdf" + loader = loader_class( + file_path, + mode="page", + page_delimiter="---", + images_parser=None, + images_inner_format="text", + password=None, + extract_tables=None, + extract_tables_settings=None, + ) + docs = loader.load() + assert len(docs) == 16 + assert loader.web_path is None + + web_path = "https://people.sc.fsu.edu/~jpeterson/hello_world.pdf" + loader = loader_class(web_path) + docs = loader.load() + assert loader.web_path == web_path + assert loader.file_path != web_path + assert len(docs) == 1 + + +def test_pymupdf_deprecated_kwards() -> None: + from langchain_community.document_loaders import PyMuPDFLoader + + file_path = Path(__file__).parent.parent / "examples/hello.pdf" + loader = PyMuPDFLoader(file_path=file_path) + loader.load(sort=True) diff --git a/libs/community/tests/integration_tests/examples/building.jpg b/libs/community/tests/integration_tests/examples/building.jpg new file mode 100644 index 0000000000000..2fdfcbff8e942 Binary files /dev/null and b/libs/community/tests/integration_tests/examples/building.jpg differ diff --git a/libs/community/tests/integration_tests/examples/layout-parser-paper-password.pdf b/libs/community/tests/integration_tests/examples/layout-parser-paper-password.pdf new file mode 100644 index 0000000000000..ae9ca9b4c684e Binary files /dev/null and b/libs/community/tests/integration_tests/examples/layout-parser-paper-password.pdf differ diff --git a/libs/community/tests/integration_tests/examples/page.png b/libs/community/tests/integration_tests/examples/page.png new file mode 100644 index 0000000000000..68b115f989ecc Binary files /dev/null and b/libs/community/tests/integration_tests/examples/page.png differ diff --git a/libs/community/tests/integration_tests/examples/text.png b/libs/community/tests/integration_tests/examples/text.png new file mode 100644 index 0000000000000..897e1f70448b7 Binary files /dev/null and b/libs/community/tests/integration_tests/examples/text.png differ diff --git a/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py b/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py index 5b96e25015f51..3044305a34159 100644 --- a/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py +++ b/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py @@ -1,17 +1,19 @@ """Tests for the various PDF parsers.""" +import importlib from pathlib import Path -from typing import Iterator +from typing import Any, Iterator import pytest +import langchain_community.document_loaders.parsers as pdf_parsers from langchain_community.document_loaders.base import BaseBlobParser from langchain_community.document_loaders.blob_loaders import Blob from langchain_community.document_loaders.parsers.pdf import ( PDFMinerParser, - PyMuPDFParser, PyPDFium2Parser, PyPDFParser, + _merge_text_and_extras, ) _THIS_DIR = Path(__file__).parents[3] @@ -23,7 +25,19 @@ LAYOUT_PARSER_PAPER_PDF = _EXAMPLES_DIR / "layout-parser-paper.pdf" -def _assert_with_parser(parser: BaseBlobParser, splits_by_page: bool = True) -> None: +def test_merge_text_and_extras() -> None: + assert "abc\n\n\n\n\n\n\n\ndef\n\n\nghi" == _merge_text_and_extras( + ["", "
"], "abc\n\n\ndef\n\n\nghi" + ) + assert "abc\n\n\n\n
\n\ndef\n\nghi" == _merge_text_and_extras( + ["", "
"], "abc\n\ndef\n\nghi" + ) + assert "abc\ndef\n\n\n\n
\n\nghi" == _merge_text_and_extras( + ["", "
"], "abc\ndef\n\nghi" + ) + + +def _assert_with_parser(parser: BaseBlobParser, *, splits_by_page: bool = True) -> None: """Standard tests to verify that the given parser works. Args: @@ -75,14 +89,29 @@ def test_pdfminer_parser() -> None: _assert_with_parser(PDFMinerParser(), splits_by_page=False) -@pytest.mark.requires("fitz") # package is PyMuPDF -def test_pymupdf_loader() -> None: - """Test PyMuPDF loader.""" - _assert_with_parser(PyMuPDFParser()) - - @pytest.mark.requires("pypdfium2") def test_pypdfium2_parser() -> None: """Test PyPDFium2 parser.""" # Does not follow defaults to split by page. _assert_with_parser(PyPDFium2Parser()) + + +@pytest.mark.parametrize( + "parser_factory,require,params", + [ + ("PyMuPDFParser", "pymupdf", {}), + ], +) +def test_parsers( + parser_factory: str, + require: str, + params: dict[str, Any], +) -> None: + try: + require = require.replace("-", "") + importlib.import_module(require, package=None) + parser_class = getattr(pdf_parsers, parser_factory) + parser = parser_class() + _assert_with_parser(parser, **params) + except ModuleNotFoundError: + pytest.skip(f"{parser_factory} skiped. Require '{require}'") diff --git a/libs/community/tests/unit_tests/document_loaders/parsers/test_public_api.py b/libs/community/tests/unit_tests/document_loaders/parsers/test_public_api.py index efaf28ecc89ce..edb5d1a35d858 100644 --- a/libs/community/tests/unit_tests/document_loaders/parsers/test_public_api.py +++ b/libs/community/tests/unit_tests/document_loaders/parsers/test_public_api.py @@ -5,15 +5,19 @@ def test_parsers_public_api_correct() -> None: """Test public API of parsers for breaking changes.""" assert set(__all__) == { "AzureAIDocumentIntelligenceParser", + "BaseImageBlobParser", "BS4HTMLParser", "DocAIParser", "GrobidParser", "LanguageParser", + "LLMImageBlobParser", "OpenAIWhisperParser", "PyPDFParser", "PDFMinerParser", "PyMuPDFParser", "PyPDFium2Parser", "PDFPlumberParser", + "RapidOCRBlobParser", + "TesseractBlobParser", "VsdxParser", } diff --git a/libs/community/tests/unit_tests/document_loaders/test_pdf.py b/libs/community/tests/unit_tests/document_loaders/test_pdf.py index 62d4fe8cc6e45..d3e9e2b586d2f 100644 --- a/libs/community/tests/unit_tests/document_loaders/test_pdf.py +++ b/libs/community/tests/unit_tests/document_loaders/test_pdf.py @@ -25,12 +25,12 @@ @pytest.mark.requires("pypdf") def test_pypdf_loader() -> None: """Test PyPDFLoader.""" - loader = PyPDFLoader(str(path_to_simple_pdf)) + loader = PyPDFLoader(path_to_simple_pdf) docs = loader.load() assert len(docs) == 1 - loader = PyPDFLoader(str(path_to_layout_pdf)) + loader = PyPDFLoader(path_to_layout_pdf) docs = loader.load() assert len(docs) == 16 @@ -48,7 +48,7 @@ def test_pypdf_loader() -> None: @pytest.mark.requires("pypdf") def test_pypdf_loader_with_layout() -> None: """Test PyPDFLoader with layout mode.""" - loader = PyPDFLoader(str(path_to_layout_pdf), extraction_mode="layout") + loader = PyPDFLoader(path_to_layout_pdf, extraction_mode="layout") docs = loader.load() assert len(docs) == 16