mirror of
https://github.com/jsvine/pdfplumber.git
synced 2026-08-29 16:40:24 +08:00
Fix utils.extract_words & .resize_object, + tests
- Fixes `.extract_words`, which had been returning incorrect results when `horizontal_ltr = False` - Fixes `.resize_object`, which had been failing in various permutations - Brings utils.py test coverage to 100%
This commit is contained in:
Jeremy Singer-Vine
+4
-22
@@ -216,29 +216,11 @@ class Page(Container):
|
||||
largest = list(sorted(tables, key=sorter))[0]
|
||||
return largest.extract()
|
||||
|
||||
def extract_text(
|
||||
self,
|
||||
x_tolerance=utils.DEFAULT_X_TOLERANCE,
|
||||
y_tolerance=utils.DEFAULT_Y_TOLERANCE,
|
||||
):
|
||||
def extract_text(self, **kwargs):
|
||||
return utils.extract_text(self.chars, **kwargs)
|
||||
|
||||
return utils.extract_text(
|
||||
self.chars, x_tolerance=x_tolerance, y_tolerance=y_tolerance
|
||||
)
|
||||
|
||||
def extract_words(
|
||||
self,
|
||||
x_tolerance=utils.DEFAULT_X_TOLERANCE,
|
||||
y_tolerance=utils.DEFAULT_Y_TOLERANCE,
|
||||
keep_blank_chars=False,
|
||||
):
|
||||
|
||||
return utils.extract_words(
|
||||
self.chars,
|
||||
x_tolerance=x_tolerance,
|
||||
y_tolerance=y_tolerance,
|
||||
keep_blank_chars=keep_blank_chars,
|
||||
)
|
||||
def extract_words(self, **kwargs):
|
||||
return utils.extract_words(self.chars, **kwargs)
|
||||
|
||||
def crop(self, bbox):
|
||||
return CroppedPage(self, self.decimalize(bbox))
|
||||
|
||||
+41
-56
@@ -3,7 +3,7 @@ from pdfminer.psparser import PSLiteral
|
||||
from pdfminer.pdftypes import PDFObjRef
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
import numbers
|
||||
from operator import itemgetter
|
||||
from operator import itemgetter, gt, lt, add, sub
|
||||
import itertools
|
||||
from functools import lru_cache as cache
|
||||
|
||||
@@ -13,7 +13,7 @@ DEFAULT_Y_TOLERANCE = 3
|
||||
|
||||
def cluster_list(xs, tolerance=0):
|
||||
tolerance = decimalize(tolerance)
|
||||
if tolerance == 0:
|
||||
if tolerance == Decimal(0):
|
||||
return [[x] for x in sorted(xs)]
|
||||
if len(xs) < 2:
|
||||
return [[x] for x in sorted(xs)]
|
||||
@@ -112,16 +112,8 @@ def resolve_all(x):
|
||||
|
||||
@cache(maxsize=int(10e4))
|
||||
def _decimalize(v, q=None):
|
||||
# If already a decimal, just return itself
|
||||
if type(v) == Decimal:
|
||||
return v
|
||||
|
||||
# If tuple/list passed, bulk-convert
|
||||
elif isinstance(v, (tuple, list)):
|
||||
return type(v)(decimalize(x, q) for x in v)
|
||||
|
||||
# Convert int-like
|
||||
elif isinstance(v, numbers.Integral):
|
||||
if isinstance(v, numbers.Integral):
|
||||
return Decimal(int(v))
|
||||
|
||||
# Convert float-like
|
||||
@@ -211,37 +203,39 @@ def extract_words(
|
||||
def process_word_chars(chars, upright):
|
||||
x0, top, x1, bottom = objects_to_bbox(chars)
|
||||
|
||||
if upright:
|
||||
if horizontal_ltr:
|
||||
sorted_chars = chars
|
||||
else:
|
||||
sorted_chars = sorted(chars, key=lambda x: -x["x1"])
|
||||
else:
|
||||
if vertical_ttb:
|
||||
sorted_chars = sorted(chars, key=itemgetter("doctop"))
|
||||
else:
|
||||
sorted_chars = sorted(chars, key=lambda x: -x["bottom"])
|
||||
|
||||
return {
|
||||
"x0": x0,
|
||||
"x1": x1,
|
||||
"top": top,
|
||||
"bottom": bottom,
|
||||
"upright": upright,
|
||||
"text": "".join(map(itemgetter("text"), sorted_chars)),
|
||||
"text": "".join(map(itemgetter("text"), chars)),
|
||||
}
|
||||
|
||||
def get_line_words(chars, upright, tolerance=DEFAULT_X_TOLERANCE):
|
||||
def get_line_words(chars, upright, tolerance):
|
||||
get_text = itemgetter("text")
|
||||
min_key = "x0" if upright else "top"
|
||||
max_key = "x1" if upright else "bottom"
|
||||
|
||||
chars_sorted = sorted(chars, key=itemgetter(min_key))
|
||||
if upright:
|
||||
min_key, max_key = ("x0", "x1") if horizontal_ltr else ("x1", "x0")
|
||||
else:
|
||||
min_key, max_key = ("top", "bottom") if vertical_ttb else ("bottom", "top")
|
||||
|
||||
words = []
|
||||
current_word = []
|
||||
|
||||
for char in chars_sorted:
|
||||
asc_order = (
|
||||
(upright and horizontal_ltr)
|
||||
or (not upright and vertical_ttb)
|
||||
)
|
||||
|
||||
comp_fn = gt if asc_order else lt
|
||||
tol_fn = add if asc_order else sub
|
||||
|
||||
def sort_key(x):
|
||||
return tol_fn(0, x[min_key])
|
||||
|
||||
sorted_chars = sorted(chars, key=sort_key)
|
||||
|
||||
for char in sorted_chars:
|
||||
if not keep_blank_chars and get_text(char).isspace():
|
||||
if len(current_word) > 0:
|
||||
words.append(current_word)
|
||||
@@ -252,7 +246,8 @@ def extract_words(
|
||||
current_word.append(char)
|
||||
else:
|
||||
last_char = current_word[-1]
|
||||
if char[min_key] > (last_char[max_key] + tolerance):
|
||||
prev_pos = tol_fn(last_char[max_key], tolerance)
|
||||
if comp_fn(char[min_key], prev_pos):
|
||||
words.append(current_word)
|
||||
current_word = []
|
||||
current_word.append(char)
|
||||
@@ -417,36 +412,26 @@ def resize_object(obj, key, value):
|
||||
assert key in ("x0", "x1", "top", "bottom")
|
||||
old_value = obj[key]
|
||||
diff = value - old_value
|
||||
if key in ("x0", "x1"):
|
||||
if key == "x0":
|
||||
assert value <= obj["x1"]
|
||||
else:
|
||||
assert value >= obj["x0"]
|
||||
new_items = (
|
||||
(key, value),
|
||||
("width", obj["width"] + diff),
|
||||
)
|
||||
if key == "top":
|
||||
new_items = [
|
||||
(key, value),
|
||||
]
|
||||
if key == "x0":
|
||||
assert value <= obj["x1"]
|
||||
new_items.append(("width", obj["x1"] - value))
|
||||
elif key == "x1":
|
||||
assert value >= obj["x0"]
|
||||
new_items.append(("width", value - obj["x0"]))
|
||||
elif key == "top":
|
||||
assert value <= obj["bottom"]
|
||||
new_items = [
|
||||
(key, value),
|
||||
("doctop", obj["doctop"] + diff),
|
||||
("height", obj["height"] - diff),
|
||||
]
|
||||
new_items.append(("doctop", obj["doctop"] + diff))
|
||||
new_items.append(("height", obj["height"] - diff))
|
||||
if "y1" in obj:
|
||||
new_items += [
|
||||
("y1", obj["y1"] - diff),
|
||||
]
|
||||
if key == "bottom":
|
||||
new_items.append(("y1", obj["y1"] - diff))
|
||||
elif key == "bottom":
|
||||
assert value >= obj["top"]
|
||||
new_items = [
|
||||
(key, value),
|
||||
("height", obj["height"] + diff),
|
||||
]
|
||||
new_items.append(("height", obj["height"] + diff))
|
||||
if "y0" in obj:
|
||||
new_items += [
|
||||
("y0", obj["y0"] - diff),
|
||||
]
|
||||
new_items.append(("y0", obj["y0"] - diff))
|
||||
return obj.__class__(tuple(obj.items()) + tuple(new_items))
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python
|
||||
import unittest
|
||||
import pytest
|
||||
import pandas as pd
|
||||
import pdfplumber
|
||||
from pdfplumber import utils
|
||||
from pdfminer.pdfparser import PDFObjRef
|
||||
from pdfminer.psparser import PSLiteral
|
||||
from decimal import Decimal
|
||||
import sys, os
|
||||
|
||||
import logging
|
||||
logging.disable(logging.ERROR)
|
||||
|
||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
class Test(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setup_class(self):
|
||||
path = os.path.join(HERE, "pdfs/pdffill-demo.pdf")
|
||||
self.pdf = pdfplumber.open(path)
|
||||
|
||||
@classmethod
|
||||
def teardown_class(self):
|
||||
self.pdf.close()
|
||||
|
||||
def test_cluster_list(self):
|
||||
a = [1, 2, 3, 4]
|
||||
assert utils.cluster_list(a) == [[x] for x in a]
|
||||
assert utils.cluster_list(a, tolerance=1) == [a]
|
||||
|
||||
a = [1, 2, 5, 6]
|
||||
assert utils.cluster_list(a, tolerance=1) == [[1, 2], [5, 6]]
|
||||
|
||||
def test_cluster_objects(self):
|
||||
a = ["a", "ab", "abc", "b"]
|
||||
assert utils.cluster_objects(a, len, 0) == [["a", "b"], ["ab"], ["abc"]]
|
||||
|
||||
def test_resolve(self):
|
||||
annot = self.pdf.annots[0]
|
||||
annot_ad0 = utils.resolve(annot["data"]["A"]["D"][0])
|
||||
assert annot_ad0["MediaBox"] == [0, 0, 612, 792]
|
||||
|
||||
def test_resolve_all(self):
|
||||
info = self.pdf.doc.xrefs[0].trailer["Info"]
|
||||
assert type(info) == PDFObjRef
|
||||
a = [ { "info": info } ]
|
||||
a_res = utils.resolve_all(a)
|
||||
assert a_res[0]["info"]["Producer"] == self.pdf.doc.info[0]["Producer"]
|
||||
|
||||
def test_decimalize(self):
|
||||
d = Decimal("1.011")
|
||||
assert utils.decimalize(1.011) == d
|
||||
assert [ utils.decimalize(1.011) ] == [ d ]
|
||||
assert utils.decimalize(d) == d
|
||||
assert id(utils.decimalize(d)) == id(d)
|
||||
assert utils.decimalize(1) == Decimal("1")
|
||||
with pytest.raises(ValueError):
|
||||
utils.decimalize("1")
|
||||
|
||||
def test_decode_psl_list(self):
|
||||
a = [ PSLiteral("test"), "test_2" ]
|
||||
assert utils.decode_psl_list(a) == ["test", "test_2"]
|
||||
|
||||
def test_extract_words(self):
|
||||
path = os.path.join(HERE, "pdfs/issue-192-example.pdf")
|
||||
with pdfplumber.open(path) as pdf:
|
||||
p = pdf.pages[0]
|
||||
words = p.extract_words(vertical_ttb=False)
|
||||
words_rtl = p.extract_words(horizontal_ltr=False)
|
||||
|
||||
assert words[0]["text"] == "Agaaaaa:"
|
||||
vertical = [w for w in words if w["upright"] == 0]
|
||||
assert vertical[0]["text"] == "Aaaaaabag8"
|
||||
assert words_rtl[1]["text"] == "baaabaaA/AAA"
|
||||
|
||||
def test_extract_text(self):
|
||||
text = self.pdf.pages[0].extract_text()
|
||||
goal = "\n".join([
|
||||
"First Page Previous Page Next Page Last Page",
|
||||
"Print",
|
||||
"PDFill: PDF Drawing",
|
||||
"You can open a PDF or create a blank PDF by PDFill.",
|
||||
"Online Help",
|
||||
"Here are the PDF drawings created by PDFill",
|
||||
"Please save into a new PDF to see the effect!",
|
||||
"Goto Page 2: Line Tool",
|
||||
"Goto Page 3: Arrow Tool",
|
||||
"Goto Page 4: Tool for Rectangle, Square and Rounded Corner",
|
||||
"Goto Page 5: Tool for Circle, Ellipse, Arc, Pie",
|
||||
"Goto Page 6: Tool for Basic Shapes",
|
||||
"Goto Page 7: Tool for Curves",
|
||||
"Here are the tools to change line width, style, arrow style and colors",
|
||||
])
|
||||
|
||||
assert text == goal
|
||||
assert self.pdf.pages[0].crop((0, 0, 0, 0)).extract_text() == None
|
||||
|
||||
def test_resize_object(self):
|
||||
obj = {
|
||||
"x0": 5,
|
||||
"x1": 10,
|
||||
"top": 20,
|
||||
"bottom": 30,
|
||||
"width": 5,
|
||||
"height": 10,
|
||||
"doctop": 120,
|
||||
"y0": 40,
|
||||
"y1": 50,
|
||||
}
|
||||
assert utils.resize_object(obj, "x0", 0) == {
|
||||
"x0": 0,
|
||||
"x1": 10,
|
||||
"top": 20,
|
||||
"doctop": 120,
|
||||
"bottom": 30,
|
||||
"width": 10,
|
||||
"height": 10,
|
||||
"y0": 40,
|
||||
"y1": 50,
|
||||
}
|
||||
assert utils.resize_object(obj, "x1", 50) == {
|
||||
"x0": 5,
|
||||
"x1": 50,
|
||||
"top": 20,
|
||||
"doctop": 120,
|
||||
"bottom": 30,
|
||||
"width": 45,
|
||||
"height": 10,
|
||||
"y0": 40,
|
||||
"y1": 50,
|
||||
}
|
||||
assert utils.resize_object(obj, "top", 0) == {
|
||||
"x0": 5,
|
||||
"x1": 10,
|
||||
"top": 0,
|
||||
"doctop": 100,
|
||||
"bottom": 30,
|
||||
"height": 30,
|
||||
"width": 5,
|
||||
"y0": 40,
|
||||
"y1": 70,
|
||||
}
|
||||
assert utils.resize_object(obj, "bottom", 40) == {
|
||||
"x0": 5,
|
||||
"x1": 10,
|
||||
"top": 20,
|
||||
"doctop": 120,
|
||||
"bottom": 40,
|
||||
"height": 20,
|
||||
"width": 5,
|
||||
"y0": 30,
|
||||
"y1": 50,
|
||||
}
|
||||
|
||||
def test_filter_edges(self):
|
||||
with pytest.raises(ValueError):
|
||||
utils.filter_edges([], "x")
|
||||
Reference in New Issue
Block a user