community: support advanced text extraction options for pdf documents

- Updated constructors in PyPDFParser and PyPDFLoader to handle `extraction_mode` and additional kwargs, aligning with the capabilities of `PageObject.extract_text()` from pypdf.

- Added `test_pypdf_loader_with_layout` along with a corresponding example text file to validate layout extraction from PDFs.

Author: bricefotzo

.gitignore

@@ -133,6 +133,7 @@ env.bak/
 
 # mypy
 .mypy_cache/
+.mypy_cache_test/
 .dmypy.json
 dmypy.json
 

libs/community/langchain_community/document_loaders/parsers/pdf.py

@@ -82,10 +82,16 @@ class PyPDFParser(BaseBlobParser):
     """Load `PDF` using `pypdf`"""
 
     def __init__(
-        self, password: Optional[Union[str, bytes]] = None, extract_images: bool = False
+        self,
+        password: Optional[Union[str, bytes]] = None,
+        extract_images: bool = False,
+        extraction_mode: str = "plain",
+        **kwargs: Any,
     ):
         self.password = password
         self.extract_images = extract_images
+        self.extraction_mode = extraction_mode
+        self.extraction_kwargs = kwargs
 
     def lazy_parse(self, blob: Blob) -> Iterator[Document]:
         """Lazily parse the blob."""
@@ -95,7 +101,9 @@ def lazy_parse(self, blob: Blob) -> Iterator[Document]:
             pdf_reader = pypdf.PdfReader(pdf_file_obj, password=self.password)
             yield from [
                 Document(
-                    page_content=page.extract_text()
+                    page_content=page.extract_text(
+                        extraction_mode=self.extraction_mode, **self.extraction_kwargs
+                    )
                     + self._extract_images_from_page(page),
                     metadata={"source": blob.source, "page": page_number},
                 )

libs/community/langchain_community/document_loaders/pdf.py

@@ -171,6 +171,8 @@ def __init__(
         password: Optional[Union[str, bytes]] = None,
         headers: Optional[Dict] = None,
         extract_images: bool = False,
+        extraction_mode: str = "plain",
+        kwargs: Optional[Dict] = None,
     ) -> None:
         """Initialize with a file path."""
         try:
@@ -180,7 +182,12 @@ def __init__(
                 "pypdf package not found, please install it with " "`pip install pypdf`"
             )
         super().__init__(file_path, headers=headers)
-        self.parser = PyPDFParser(password=password, extract_images=extract_images)
+        self.parser = PyPDFParser(
+            password=password,
+            extract_images=extract_images,
+            extraction_mode=extraction_mode,
+            kwargs=kwargs,
+        )
 
     def lazy_load(
         self,

libs/community/poetry.lock

@@ -26,7 +26,7 @@ jinja2 = {version = "^3", optional = true}
 cohere = {version = "^4", optional = true}
 openai = {version = "<2", optional = true}
 arxiv = {version = "^1.4", optional = true}
-pypdf = {version = "^3.4.0", optional = true}
+pypdf = {version = "^4.0.0", optional = true}
 aleph-alpha-client = {version="^2.15.0", optional = true}
 gradientai = {version="^1.4.0", optional = true}
 pgvector = {version = "^0.1.6", optional = true}

libs/community/pyproject.toml

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 from typing import Sequence, Union
 
@@ -100,6 +101,22 @@ def test_pypdf_loader() -> None:
     assert len(docs) == 16
 
 
+def test_pypdf_loader_with_layout() -> None:
+    """Test PyPDFLoader with layout mode."""
+    file_path = Path(__file__).parent.parent / "examples/layout-parser-paper.pdf"
+    loader = PyPDFLoader(str(file_path), extraction_mode="layout")
+
+    docs = loader.load()
+    first_page = docs[0].page_content
+
+    expected = (
+        Path(__file__).parent.parent / "examples/layout-parser-paper-page-1.txt"
+    ).read_text(encoding="utf-8")
+    cleaned_first_page = re.sub(r"\x00", "", first_page)
+    cleaned_expected = re.sub(r"\x00", "", expected)
+    assert cleaned_first_page == cleaned_expected
+
+
 def test_pypdfium2_loader() -> None:
     """Test PyPDFium2Loader."""
     file_path = Path(__file__).parent.parent / "examples/hello.pdf"

libs/community/tests/integration_tests/document_loaders/test_pdf.py

@@ -0,0 +1,49 @@
+             LayoutParser         : A Unified Toolkit for Deep
+          Learning Based Document Image Analysis
+
+
+Zejiang Shen           1  (     ), Ruochen Zhang                2, Melissa Dell         3, Benjamin Charles Germain
+                                         Lee   4, Jacob Carlson            3, and Weining Li              5
+
+                                                           1  Allen Institute for AI
+                                                           shannons@allenai.org
+                                                               2  Brown University
+                                                        ruochen          zhang@brown.edu
+                                                             3  Harvard University
+                                  {melissadell,jacob                       carlson       }@fas.harvard.edu
+                                                       4  University of Washington
+                                                         bcgl@cs.washington.edu
+                                                          5  University of Waterloo
+                                                            w422li@uwaterloo.ca
+
+
+
+             Abstract.        Recentadvancesindocumentimageanalysis(DIA)havebeen
+             primarily driven by the application of neural networks. Ideally, research
+             outcomes could be easily deployed in production and extended for further
+             investigation. However, various factors like loosely organized codebases
+             and sophisticated model configurations complicate the easy reuse of im-
+             portant innovations by awide audience. Though there havebeen on-going
+             efforts to improve reusability and simplify deep learning (DL) model
+             development in disciplines like natural language processing and computer
+             vision, none of them are optimized for challenges in the domain of DIA.
+             This represents a major gap in the existing toolkit, as DIA is central to
+             academic research across a wide range of disciplines in the social sciences
+             and humanities. This paper introduces                           LayoutParser           , an open-source
+             library for streamlining the usage of DL in DIA research and applica-
+             tions. The core          LayoutParser            library comes with a set of simple and
+             intuitive interfaces for applying and customizing DL models for layout de-
+             tection,characterrecognition,andmanyotherdocumentprocessingtasks.
+             To promote extensibility,                 LayoutParser            also incorporates a community
+             platform for sharing both pre-trained models and full document digiti-
+             zation pipelines. We demonstrate that                         LayoutParser            is helpful for both
+             lightweight and large-scale digitization pipelines in real-word use cases.
+             The library is publicly available at                    https://layout-parser.github.io                            .
+
+             Keywords:          DocumentImageAnalysis                                    ·DeepLearning                    ·LayoutAnalysis
+             · Character Recognition                              · Open Source library                           · Toolkit.
+
+1   Introduction
+
+Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of
+documentimageanalysis(DIA)tasksincludingdocumentimageclassification[                                              11 ,

libs/community/tests/integration_tests/examples/layout-parser-paper-page-1.txt

@@ -42,7 +42,7 @@ nlpcloud = {version = "^1", optional = true}
 huggingface_hub = {version = "^0", optional = true}
 sentence-transformers = {version = "^2", optional = true}
 arxiv = {version = "^1.4", optional = true}
-pypdf = {version = "^3.4.0", optional = true}
+pypdf = {version = "^4.0.0", optional = true}
 aleph-alpha-client = {version="^2.15.0", optional = true}
 pgvector = {version = "^0.1.6", optional = true}
 async-timeout = {version = "^4.0.0", python = "<3.11"}

libs/langchain/poetry.lock

@@ -10,7 +10,7 @@ python = ">=3.8.1,<4.0"
 langchain = "^0.1"
 openai = "<2"
 pymongo = "^4.6.0"
-pypdf = "^3.17.0"
+pypdf = "^4.0.0"
 tiktoken = "^0.5.1"
 langchain-text-splitters = ">=0.0.1,<0.1"
 

libs/langchain/pyproject.toml

@@ -10,7 +10,7 @@ python = ">=3.8.1,<4.0"
 langchain = "^0.1"
 pymilvus = ">=2.3.0"
 langchain-nvidia-aiplay = "^0.0.2"
-pypdf = ">=3.1"
+pypdf = ">=4.0.1"
 langchain-text-splitters = ">=0.0.1,<0.1"
 
 [tool.poetry.group.dev.dependencies]