From d4534faf08ff94bbb0fe246120500a02f7b118f8 Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Mon, 7 Mar 2016 23:06:08 -0500 Subject: [PATCH] Add .extract_words(...) and decimalize tolerances --- README.md | 2 + pdfplumber/helpers.py | 28 ------------ pdfplumber/page.py | 9 +++- pdfplumber/utils.py | 99 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index c31fa67..a92b264 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ The `pdfplumber.Page` class is at the core of `pdfplumber`. Most things you'll d - `.extract_text(x_tolerance=0, y_tolerance=0)`: Collates all of the page's character objects into a single string. Adds spaces where the difference between the `x1` of one character and the `x0` of the next is greater than `x_tolerance`. Adds newline characters where the difference between the `doctop` of one character and the `doctop` of the next is greater than `y_tolerance`. +- `.extract_words(x_tolerance=0, y_tolerance=0)`: Returns a list of all word-looking things and their bounding boxes. Words are considered to be sequences of characters where the difference between the `x1` of one character and the `x0` of the next is less than or equal to `x_tolerance` *and* where the `doctop` of one character and the `doctop` of the next is less than or equal to `y_tolerance`. + - `.extract_table(...)`: Extracts tabular data from the page. For more details see "[Extracting tables](#extracting-tables)" below. ### Objects diff --git a/pdfplumber/helpers.py b/pdfplumber/helpers.py index 1584c06..e69de29 100644 --- a/pdfplumber/helpers.py +++ b/pdfplumber/helpers.py @@ -1,28 +0,0 @@ -import itertools -def cluster_list(xs, tolerance=0): - if tolerance == 0: return [ [x] for x in sorted(xs) ] - if len(xs) < 2: return [ [x] for x in sorted(xs) ] - groups = [] - xs = list(sorted(xs)) - current_group = [xs[0]] - last = xs[0] - for x in xs[1:]: - if x <= (last + tolerance): - current_group.append(x) - else: - groups.append(current_group) - current_group = [x] - last = x - groups.append(current_group) - return groups - -def make_cluster_dict(values, tolerance): - clusters = cluster_list(set(values), tolerance) - - nested_tuples = [ [ (val, i) for val in value_cluster ] - for i, value_cluster in enumerate(clusters) ] - - cluster_dict = dict(itertools.chain(*nested_tuples)) - return cluster_dict - - diff --git a/pdfplumber/page.py b/pdfplumber/page.py index b56b2cc..7cd45b6 100644 --- a/pdfplumber/page.py +++ b/pdfplumber/page.py @@ -1,5 +1,4 @@ from pdfplumber import utils -from pdfplumber import helpers from pdfplumber.container import Container from six import string_types @@ -99,7 +98,7 @@ class Page(Container): pos_var = "x0" if orientation == "v" else "top" edges_uniq = set(e[pos_var] for e in edges) - edges_clust = helpers.cluster_list(edges_uniq, tolerance=tolerance) + edges_clust = utils.cluster_list(edges_uniq, tolerance=tolerance) edge_means = list(sorted(sum(c) / len(c) for c in edges_clust)) return edge_means @@ -176,6 +175,12 @@ class Page(Container): x_tolerance=x_tolerance, y_tolerance=y_tolerance) + def extract_words(self, x_tolerance=0, y_tolerance=0): + return utils.extract_words(self.chars, + x_tolerance=x_tolerance, + y_tolerance=y_tolerance) + + def crop(self, bbox, strict=False): return CroppedPage(self, bbox, strict=strict) diff --git a/pdfplumber/utils.py b/pdfplumber/utils.py index e69cb6e..aa29b2d 100644 --- a/pdfplumber/utils.py +++ b/pdfplumber/utils.py @@ -1,10 +1,38 @@ -from pdfplumber import helpers from pdfminer.utils import PDFDocEncoding from decimal import Decimal, ROUND_HALF_UP from operator import itemgetter import itertools import six +def cluster_list(xs, tolerance=0): + tolerance = decimalize(tolerance) + if tolerance == 0: return [ [x] for x in sorted(xs) ] + if len(xs) < 2: return [ [x] for x in sorted(xs) ] + groups = [] + xs = list(sorted(xs)) + current_group = [xs[0]] + last = xs[0] + for x in xs[1:]: + if x <= (last + tolerance): + current_group.append(x) + else: + groups.append(current_group) + current_group = [x] + last = x + groups.append(current_group) + return groups + +def make_cluster_dict(values, tolerance): + tolerance = decimalize(tolerance) + clusters = cluster_list(set(values), tolerance) + + nested_tuples = [ [ (val, i) for val in value_cluster ] + for i, value_cluster in enumerate(clusters) ] + + cluster_dict = dict(itertools.chain(*nested_tuples)) + return cluster_dict + + def decode_text(s): """ Decodes a PDFDocEncoding string to Unicode. @@ -38,6 +66,7 @@ def to_list(collection): return collection def collate_line(line_chars, tolerance=0): + tolerance = decimalize(tolerance) coll = "" last_x1 = None for char in sorted(line_chars, key=itemgetter("x0")): @@ -47,6 +76,68 @@ def collate_line(line_chars, tolerance=0): coll += char["text"] return coll +def get_bbox(objs): + return ( + min(map(itemgetter("x0"), objs)), + min(map(itemgetter("top"), objs)), + max(map(itemgetter("x1"), objs)), + max(map(itemgetter("bottom"), objs)), + ) + +def extract_words(chars, x_tolerance=0, y_tolerance=0): + x_tolerance = decimalize(x_tolerance) + y_tolerance = decimalize(y_tolerance) + + def process_word_chars(chars): + x0, top, x1, bottom = get_bbox(chars) + return { + "x0": x0, + "x1": x1, + "top": top, + "bottom": bottom, + "text": "".join(map(itemgetter("text"), chars)) + } + + + def get_line_words(chars, tolerance=0): + chars_sorted = sorted(chars, key=itemgetter("x0")) + words = [] + current_word = [] + for char in chars_sorted: + if len(current_word) == 0: + current_word.append(char) + else: + last_char = current_word[-1] + if char["x0"] > (last_char["x1"] + tolerance): + words.append(current_word) + current_word = [ char ] + else: + current_word.append(char) + words.append(current_word) + processed_words = list(map(process_word_chars, words)) + return processed_words + + chars = to_list(chars) + + doctops = map(itemgetter("doctop"), chars) + doctop_clusters = make_cluster_dict(doctops, y_tolerance) + + with_cluster = ((char, doctop_clusters.get(char["doctop"])) + for char in chars) + + get_0 = itemgetter(0) + get_1 = itemgetter(1) + + with_cluster_sorted = sorted(with_cluster, key=get_1) + + grouped = itertools.groupby(with_cluster_sorted, key=get_1) + + nested = [ get_line_words(map(get_0, line_chars), tolerance=x_tolerance) + for k, line_chars in grouped ] + + words = list(itertools.chain(*nested)) + return words + def extract_text(chars, x_tolerance=0, y_tolerance=0): if len(chars) == 0: return None @@ -57,14 +148,14 @@ def extract_text(chars, x_tolerance=0, y_tolerance=0): chars = to_list(chars) doctops = map(itemgetter("doctop"), chars) - doctop_clusters = helpers.make_cluster_dict(doctops, y_tolerance) + doctop_clusters = make_cluster_dict(doctops, y_tolerance) with_cluster = ((char, doctop_clusters.get(char["doctop"])) for char in chars) - groups = itertools.groupby(sorted(with_cluster, key=get_1), key=get_1) + grouped = itertools.groupby(sorted(with_cluster, key=get_1), key=get_1) lines = (collate_line(map(get_0, items), x_tolerance) - for k, items in groups) + for k, items in grouped) coll = "\n".join(lines) return coll