Merge pull request #983 from dhdaines/issue-982

Respect `use_text_flow` in `extract_text`

pdfplumber/utils/clustering.py

@@ -40,7 +40,10 @@ R = TypeVar("R")
 
 
 def cluster_objects(
-    xs: List[R], key_fn: Union[Hashable, Callable[[R], T_num]], tolerance: T_num
+    xs: List[R],
+    key_fn: Union[Hashable, Callable[[R], T_num]],
+    tolerance: T_num,
+    preserve_order: bool = False,
 ) -> List[List[R]]:
 
     if not callable(key_fn):
@@ -51,7 +54,12 @@ def cluster_objects(
 
     get_0, get_1 = itemgetter(0), itemgetter(1)
 
-    cluster_tuples = sorted(((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1)
+    if preserve_order:
+        cluster_tuples = [(x, cluster_dict.get(key_fn(x))) for x in xs]
+    else:
+        cluster_tuples = sorted(
+            ((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1
+        )
 
     grouped = itertools.groupby(cluster_tuples, key=get_1)
 

pdfplumber/utils/text.py

@@ -225,7 +225,10 @@ class WordMap:
 
         for i, ws in enumerate(
             cluster_objects(
-                words_sorted_doctop, lambda x: float(x[0]["doctop"]), y_tolerance
+                words_sorted_doctop,
+                lambda x: float(x[0]["doctop"]),
+                y_tolerance,
+                preserve_order=presorted or use_text_flow,
             )
         ):
             y_dist = (

tests/test_issues.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import logging
 import os
+import re
 import unittest
 
 import pdfplumber
@@ -257,3 +258,20 @@ class Test(unittest.TestCase):
         with pdfplumber.open(path) as pdf:
             page = pdf.pages[0]
             page.search(r"\d+", regex=True)
+
+    def test_issue_982(self):
+        """
+        extract_text(use_text_flow=True) apparently does nothing
+
+        This is because, while we took care not to sort the words by
+        `doctop` in `WordExtractor` and `WordMap`, no such precaution
+        was taken in `cluster_objects`.  We thus add an option to
+        `cluster_objects` to preserve the ordering (which could come
+        from `use_text_flow` or from `presorted`) of the input objects.
+        """
+        path = os.path.join(HERE, "pdfs/issue-982-example.pdf")
+        with pdfplumber.open(path) as pdf:
+            page = pdf.pages[0]
+            text = re.sub(r"\s+", " ", page.extract_text(use_text_flow=True))
+            words = " ".join(w["text"] for w in page.extract_words(use_text_flow=True))
+            assert text[0:100] == words[0:100]