Add .extract_words(...) and decimalize tolerances

README.md

@@ -83,6 +83,8 @@ The `pdfplumber.Page` class is at the core of `pdfplumber`. Most things you'll d
 
 - `.extract_text(x_tolerance=0, y_tolerance=0)`: Collates all of the page's character objects into a single string. Adds spaces where the difference between the `x1` of one character and the `x0` of the next is greater than `x_tolerance`. Adds newline characters where the difference between the `doctop` of one character and the `doctop` of the next is greater than `y_tolerance`.
 
+- `.extract_words(x_tolerance=0, y_tolerance=0)`: Returns a list of all word-looking things and their bounding boxes. Words are considered to be sequences of characters where the difference between the `x1` of one character and the `x0` of the next is less than or equal to `x_tolerance` *and* where the `doctop` of one character and the `doctop` of the next is less than or equal to `y_tolerance`.
+
 - `.extract_table(...)`: Extracts tabular data from the page. For more details see "[Extracting tables](#extracting-tables)" below.
 
 ### Objects

pdfplumber/helpers.py

@@ -1,28 +0,0 @@
-import itertools
-def cluster_list(xs, tolerance=0):
-    if tolerance == 0: return [ [x] for x in sorted(xs) ]
-    if len(xs) < 2: return [ [x] for x in sorted(xs) ]
-    groups = []
-    xs = list(sorted(xs))
-    current_group = [xs[0]]
-    last = xs[0]
-    for x in xs[1:]:
-        if x <= (last + tolerance):
-            current_group.append(x)
-        else:
-            groups.append(current_group)
-            current_group = [x]
-        last = x
-    groups.append(current_group)
-    return groups
-
-def make_cluster_dict(values, tolerance):
-    clusters = cluster_list(set(values), tolerance)
-
-    nested_tuples = [ [ (val, i) for val in value_cluster ]
-        for i, value_cluster in enumerate(clusters) ]
-
-    cluster_dict = dict(itertools.chain(*nested_tuples))
-    return cluster_dict 
-
-

pdfplumber/page.py

@@ -1,5 +1,4 @@
 from pdfplumber import utils
-from pdfplumber import helpers
 from pdfplumber.container import Container
 
 from six import string_types
@@ -99,7 +98,7 @@ class Page(Container):
 
         pos_var = "x0" if orientation == "v" else "top"
         edges_uniq = set(e[pos_var] for e in edges)
-        edges_clust = helpers.cluster_list(edges_uniq, tolerance=tolerance)
+        edges_clust = utils.cluster_list(edges_uniq, tolerance=tolerance)
         edge_means = list(sorted(sum(c) / len(c) for c in edges_clust))
 
         return edge_means
@@ -176,6 +175,12 @@ class Page(Container):
             x_tolerance=x_tolerance,
             y_tolerance=y_tolerance)
 
+    def extract_words(self, x_tolerance=0, y_tolerance=0):
+        return utils.extract_words(self.chars,
+            x_tolerance=x_tolerance,
+            y_tolerance=y_tolerance)
+
+
     def crop(self, bbox, strict=False):
         return CroppedPage(self, bbox, strict=strict)
 

pdfplumber/utils.py

@@ -1,10 +1,38 @@
-from pdfplumber import helpers
 from pdfminer.utils import PDFDocEncoding
 from decimal import Decimal, ROUND_HALF_UP
 from operator import itemgetter
 import itertools
 import six
 
+def cluster_list(xs, tolerance=0):
+    tolerance = decimalize(tolerance)
+    if tolerance == 0: return [ [x] for x in sorted(xs) ]
+    if len(xs) < 2: return [ [x] for x in sorted(xs) ]
+    groups = []
+    xs = list(sorted(xs))
+    current_group = [xs[0]]
+    last = xs[0]
+    for x in xs[1:]:
+        if x <= (last + tolerance):
+            current_group.append(x)
+        else:
+            groups.append(current_group)
+            current_group = [x]
+        last = x
+    groups.append(current_group)
+    return groups
+
+def make_cluster_dict(values, tolerance):
+    tolerance = decimalize(tolerance)
+    clusters = cluster_list(set(values), tolerance)
+
+    nested_tuples = [ [ (val, i) for val in value_cluster ]
+        for i, value_cluster in enumerate(clusters) ]
+
+    cluster_dict = dict(itertools.chain(*nested_tuples))
+    return cluster_dict 
+
+
 def decode_text(s):
     """
     Decodes a PDFDocEncoding string to Unicode.
@@ -38,6 +66,7 @@ def to_list(collection):
         return collection
 
 def collate_line(line_chars, tolerance=0):
+    tolerance = decimalize(tolerance)
     coll = ""
     last_x1 = None
     for char in sorted(line_chars, key=itemgetter("x0")):
@@ -47,6 +76,68 @@ def collate_line(line_chars, tolerance=0):
         coll += char["text"]
     return coll
 
+def get_bbox(objs):
+    return (
+        min(map(itemgetter("x0"), objs)),
+        min(map(itemgetter("top"), objs)),
+        max(map(itemgetter("x1"), objs)),
+        max(map(itemgetter("bottom"), objs)),
+    )
+
+def extract_words(chars, x_tolerance=0, y_tolerance=0):
+    x_tolerance = decimalize(x_tolerance)
+    y_tolerance = decimalize(y_tolerance)
+
+    def process_word_chars(chars):
+        x0, top, x1, bottom = get_bbox(chars)
+        return {
+            "x0": x0,
+            "x1": x1,
+            "top": top,
+            "bottom": bottom,
+            "text": "".join(map(itemgetter("text"), chars))
+        }
+        
+
+    def get_line_words(chars, tolerance=0):
+        chars_sorted = sorted(chars, key=itemgetter("x0"))
+        words = []
+        current_word = []
+        for char in chars_sorted:
+            if len(current_word) == 0:
+                current_word.append(char)
+            else:
+                last_char = current_word[-1]
+                if char["x0"] > (last_char["x1"] + tolerance):
+                    words.append(current_word)
+                    current_word = [ char ]
+                else:
+                    current_word.append(char)
+        words.append(current_word)
+        processed_words = list(map(process_word_chars, words))
+        return processed_words
+
+    chars = to_list(chars)
+
+    doctops = map(itemgetter("doctop"), chars)
+    doctop_clusters = make_cluster_dict(doctops, y_tolerance)
+
+    with_cluster = ((char, doctop_clusters.get(char["doctop"]))
+        for char in chars)
+
+    get_0 = itemgetter(0)
+    get_1 = itemgetter(1)
+
+    with_cluster_sorted = sorted(with_cluster, key=get_1)
+
+    grouped = itertools.groupby(with_cluster_sorted, key=get_1)
+
+    nested = [ get_line_words(map(get_0, line_chars), tolerance=x_tolerance)
+        for k, line_chars in grouped ]
+
+    words = list(itertools.chain(*nested))
+    return words
+
 def extract_text(chars, x_tolerance=0, y_tolerance=0):
     if len(chars) == 0:
         return None
@@ -57,14 +148,14 @@ def extract_text(chars, x_tolerance=0, y_tolerance=0):
     chars = to_list(chars)
 
     doctops = map(itemgetter("doctop"), chars)
-    doctop_clusters = helpers.make_cluster_dict(doctops, y_tolerance)
+    doctop_clusters = make_cluster_dict(doctops, y_tolerance)
 
     with_cluster = ((char, doctop_clusters.get(char["doctop"]))
         for char in chars)
 
-    groups = itertools.groupby(sorted(with_cluster, key=get_1), key=get_1)
+    grouped = itertools.groupby(sorted(with_cluster, key=get_1), key=get_1)
     lines = (collate_line(map(get_0, items), x_tolerance)
-        for k, items in groups)
+        for k, items in grouped)
 
     coll = "\n".join(lines)
     return coll