From 87b947f8f6271ab54f463d701d936627aab38f29 Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Sun, 7 Feb 2021 16:33:54 -0500 Subject: [PATCH] Remove decimalizing (but let CLI adjust precision) Per discussion at https://github.com/jsvine/pdfplumber/discussions/346 and input from @ramcdona, this commit changes pdfplumber's behavior regarding floating point numbers. Specifically, it removes all conversion of floats to Decimal objects. This brings several advantages: - Increased precision (where applicable) - Decreased code complexity - Increased performance (~10% speedup on test suite) - Increased fidelity to `pdfminer.six` output These seem to outweigh the disadvantages: - Some tests break (but have been easily fixed) due to increased precision and/or floating point arithmetic artifacts - Some users' scripts may also break, if they depend on strict equality testing, though these *should* also be easily fixable Because some form of automatic rounding may still be desirable for the pdfplumber CLI utility, the conversion methods (.to_csv, .to_json) have been adjusted to accept a `precision` argument. --- CHANGELOG.md | 3 ++ README.md | 2 + pdfplumber/cli.py | 8 ++- pdfplumber/convert.py | 108 ++++++++++++++++++++----------------- pdfplumber/display.py | 8 ++- pdfplumber/page.py | 61 +++++++-------------- pdfplumber/pdf.py | 2 - pdfplumber/table.py | 8 +-- pdfplumber/utils.py | 57 ++++---------------- tests/test_basics.py | 4 +- tests/test_convert.py | 10 ++-- tests/test_dedupe_chars.py | 29 +++++----- tests/test_display.py | 7 +++ tests/test_issues.py | 4 +- tests/test_utils.py | 13 +---- 15 files changed, 140 insertions(+), 184 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4869a17..ceec78a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,11 @@ All notable changes to this project will be documented in this file. The format ## [0.6.0] - [unreleased] ### Added - Add `utils.merge_bboxes(bboxes)`, which returns the smallest bounding box that contains all bounding boxes in the `bboxes` argument. +- Add `--precision` argument to CLI ## Changed - Upgrade `pdfminer.six` from `20200517` to `20211012`; see [that library's changelog](https://github.com/pdfminer/pdfminer.six/blob/develop/CHANGELOG.md) for details, but a key difference is an improvement in how it assigns `line`, `rect`, and `curve` objects. (Diagonal two-point lines, for instance, are now `line` objects instead of `curve` objects.) +- Remove Decimal-ization of parsed object attributes, which are now represented with as much precision as is returned by `pdfminer.six` [#346](https://github.com/jsvine/pdfplumber/discussions/346) - Change behavior of horizontal `text_strategy`, so that it uses the top and bottom of *every* word, not just the top of every word and the bottom of the last. ([#467](https://github.com/jsvine/pdfplumber/pull/467) + [#466](https://github.com/jsvine/pdfplumber/issues/466) + [#265](https://github.com/jsvine/pdfplumber/issues/265)) [h/t @bobluda + @samkit-jain] ### Fixed @@ -42,6 +44,7 @@ All notable changes to this project will be documented in this file. The format - Add `Page.close/__enter__/__exit__` methods, by generalizing that behavior through the `Container` class ([b1849f4](https://github.com/jsvine/pdfplumber/commit/b1849f4)) ### Changed +- Change handling of floating point numbers; no longer convert them to `Decimal` objects and do not round them - Change `TableFinder` to return tables in order of topmost-and-then-leftmost, rather than leftmost-and-then-topmost ([#336](https://github.com/jsvine/pdfplumber/issues/336)) - Change `Page.to_image()`'s handling of alpha layer, to remove aliasing artifacts ([#340](https://github.com/jsvine/pdfplumber/pull/340)) [h/t @arlyon] diff --git a/README.md b/README.md index ac9b0f3..6b5b0a0 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ The output will be a CSV containing info about every character, line, and rectan |`--pages [list of pages]`| A space-delimited, `1`-indexed list of pages or hyphenated page ranges. E.g., `1, 11-15`, which would return data for pages 1, 11, 12, 13, 14, and 15.| |`--types [list of object types to extract]`| Choices are `char`, `rect`, `line`, `curve`, `image`, `annot`, et cetera. Defaults to all available.| |`--laparams`| A JSON-formatted string (e.g., `'{"detect_vertical": true}'`) to pass to `pdfplumber.open(..., laparams=...)`.| +|`--precision [integer]`| The number of decimal places to round floating-point numbers. Defaults to no rounding.| ## Python library @@ -432,6 +433,7 @@ Many thanks to the following users who've contributed ideas, features, and fixes - [Alexander Regueiro](https://github.com/alexreg) - [Daniel Peña](https://github.com/trifling) - [bobluda](https://github.com/bobluda) +- [@ramcdona](https://github.com/ramcdona) ## Contributing diff --git a/pdfplumber/cli.py b/pdfplumber/cli.py index 5c16576..583becd 100644 --- a/pdfplumber/cli.py +++ b/pdfplumber/cli.py @@ -28,6 +28,8 @@ def parse_args(args_raw): parser.add_argument("--laparams", type=json.loads) + parser.add_argument("--precision", type=int) + parser.add_argument("--pages", nargs="+", type=parse_page_spec) parser.add_argument( @@ -43,7 +45,11 @@ def parse_args(args_raw): def main(args_raw=sys.argv[1:]): args = parse_args(args_raw) converter = {"csv": convert.to_csv, "json": convert.to_json}[args.format] - kwargs = {"csv": {}, "json": {"indent": args.indent}}[args.format] + kwargs = { + "csv": {"precision": args.precision}, + "json": {"precision": args.precision, "indent": args.indent}, + }[args.format] + with PDF.open(args.infile, pages=args.pages, laparams=args.laparams) as pdf: converter(pdf, sys.stdout, args.types, **kwargs) diff --git a/pdfplumber/convert.py b/pdfplumber/convert.py index 20ff377..86c88a0 100644 --- a/pdfplumber/convert.py +++ b/pdfplumber/convert.py @@ -1,7 +1,4 @@ from .utils import decode_text -from decimal import Decimal, ROUND_HALF_UP -from pdfminer.pdftypes import PDFStream -from pdfminer.psparser import PSLiteral import json import csv import base64 @@ -33,50 +30,62 @@ def to_b64(data_bytes): return base64.b64encode(data_bytes).decode("ascii") -def try_decode_bytes(obj): - for e in ENCODINGS_TO_TRY: - try: - return obj.decode(e) - except UnicodeDecodeError: # pragma: no cover - pass - # If none of the decodings work, raise whatever error - # decoding with utf-8 causes - obj.decode(ENCODINGS_TO_TRY[0]) # pragma: no cover +class Serializer: + def __init__(self, precision=None): + self.precision = precision + + def serialize(self, obj): + if obj is None: + return None + + t = type(obj) + + # Basic types don't need to be converted + if t in (int, str): + return obj + + # Use one of the custom converters, if possible + fn = getattr(self, f"do_{t.__name__}", None) + if fn is not None: + return fn(obj) + + # Otherwise, just use the string-representation + else: + return str(obj) + + def do_float(self, x): + return x if self.precision is None else round(x, self.precision) + + def do_bool(self, x): + return int(x) + + def do_list(self, obj): + return list(self.serialize(x) for x in obj) + + def do_tuple(self, obj): + return tuple(self.serialize(x) for x in obj) + + def do_dict(self, obj): + return {k: self.serialize(v) for k, v in obj.items()} + + def do_PDFStream(self, obj): + return {"rawdata": to_b64(obj.rawdata)} + + def do_PSLiteral(self, obj): + return decode_text(obj.name) + + def do_bytes(self, obj): + for e in ENCODINGS_TO_TRY: + try: + return obj.decode(e) + except UnicodeDecodeError: # pragma: no cover + pass + # If none of the decodings work, raise whatever error + # decoding with utf-8 causes + obj.decode(ENCODINGS_TO_TRY[0]) # pragma: no cover -serializers = { - Decimal: lambda obj: float(obj.quantize(Decimal(".0001"), rounding=ROUND_HALF_UP)), - list: lambda obj: list(serialize(x) for x in obj), - tuple: lambda obj: tuple(serialize(x) for x in obj), - dict: lambda obj: {k: serialize(v) for k, v in obj.items()}, - PDFStream: lambda obj: {"rawdata": to_b64(obj.rawdata)}, - PSLiteral: lambda obj: decode_text(obj.name), - bytes: try_decode_bytes, - bool: int, -} - - -def serialize(obj): - if obj is None: - return None - - t = type(obj) - - # Basic types don't need to be converted - if t in (int, float, str): - return obj - - # Use one of the custom converters above, if possible - fn = serializers.get(t) - if fn is not None: - return fn(obj) - - # Otherwise, just use the string-representation - else: - return str(obj) - - -def to_json(container, stream=None, types=None, indent=None): +def to_json(container, stream=None, types=None, precision=None, indent=None): if types is None: types = list(container.objects.keys()) + ["annot"] @@ -103,7 +112,7 @@ def to_json(container, stream=None, types=None, indent=None): else: data = page_to_dict(container) - serialized = serialize(data) + serialized = Serializer(precision=precision).serialize(data) if stream is None: return json.dumps(serialized, indent=indent) @@ -111,7 +120,7 @@ def to_json(container, stream=None, types=None, indent=None): return json.dump(serialized, stream, indent=indent) -def to_csv(container, stream=None, types=None): +def to_csv(container, stream=None, types=None, precision=None): if stream is None: stream = StringIO() to_string = True @@ -133,11 +142,14 @@ def to_csv(container, stream=None, types=None): new_keys = [k for k, v in new_objs[0].items() if type(v) is not dict] fields = fields.union(set(new_keys)) + serialized = Serializer(precision=precision).serialize(objs) + cols = COLS_TO_PREPEND + list(sorted(set(fields) - set(COLS_TO_PREPEND))) w = csv.DictWriter(stream, fieldnames=cols, extrasaction="ignore") w.writeheader() - w.writerows(serialize(objs)) + w.writerows(serialized) + if to_string: stream.seek(0) return stream.read() diff --git a/pdfplumber/display.py b/pdfplumber/display.py index ad44767..854d9d8 100644 --- a/pdfplumber/display.py +++ b/pdfplumber/display.py @@ -56,15 +56,13 @@ class PageImage(object): else: self.original = original - d = self.page.decimalize - self.decimalize = d if page.is_original: self.root = page cropped = False else: self.root = page.root_page cropped = page.root_page.bbox != page.bbox - self.scale = d(self.original.size[0]) / d(self.root.width) + self.scale = self.original.size[0] / self.root.width if cropped: cropbox = ( (page.bbox[0] - page.root_page.bbox[0]) * self.scale, @@ -160,7 +158,7 @@ class PageImage(object): bbox = (obj["x0"], obj["top"], obj["x1"], obj["bottom"]) x0, top, x1, bottom = bbox - half = self.decimalize(stroke_width / 2) + half = stroke_width / 2 x0 += half top += half x1 -= half @@ -194,7 +192,7 @@ class PageImage(object): obj = center_or_obj center = ((obj["x0"] + obj["x1"]) / 2, (obj["top"] + obj["bottom"]) / 2) cx, cy = center - bbox = self.decimalize((cx - radius, cy - radius, cx + radius, cy + radius)) + bbox = (cx - radius, cy - radius, cx + radius, cy + radius) self.draw.ellipse(self._reproject_bbox(bbox), fill, stroke) return self diff --git a/pdfplumber/page.py b/pdfplumber/page.py index cf6b0f5..d9886e7 100644 --- a/pdfplumber/page.py +++ b/pdfplumber/page.py @@ -8,7 +8,7 @@ import re lt_pat = re.compile(r"^LT") -DECIMAL_ATTRS = set( +ALL_ATTRS = set( [ "adv", "height", @@ -21,11 +21,6 @@ DECIMAL_ATTRS = set( "x1", "y0", "y1", - ] -) - -ALL_ATTRS = DECIMAL_ATTRS | set( - [ "bits", "upright", "font", @@ -53,42 +48,33 @@ class Page(Container): self.pdf = pdf self.page_obj = page_obj self.page_number = page_number - _rotation = self.decimalize(resolve_all(self.page_obj.attrs.get("Rotate", 0))) + _rotation = resolve_all(self.page_obj.attrs.get("Rotate", 0)) self.rotation = _rotation % 360 self.page_obj.rotate = self.rotation - self.initial_doctop = self.decimalize(initial_doctop) + self.initial_doctop = initial_doctop cropbox = page_obj.attrs.get("CropBox") mediabox = page_obj.attrs.get("MediaBox") - self.cropbox = ( - self.decimalize(resolve_all(cropbox)) if cropbox is not None else None - ) - self.mediabox = self.decimalize(resolve_all(mediabox) or self.cropbox) + self.cropbox = resolve_all(cropbox) if cropbox is not None else None + self.mediabox = resolve_all(mediabox) or self.cropbox m = self.mediabox if self.rotation in [90, 270]: - self.bbox = self.decimalize( - ( - min(m[1], m[3]), - min(m[0], m[2]), - max(m[1], m[3]), - max(m[0], m[2]), - ) + self.bbox = ( + min(m[1], m[3]), + min(m[0], m[2]), + max(m[1], m[3]), + max(m[0], m[2]), ) else: - self.bbox = self.decimalize( - ( - min(m[0], m[2]), - min(m[1], m[3]), - max(m[0], m[2]), - max(m[1], m[3]), - ) + self.bbox = ( + min(m[0], m[2]), + min(m[1], m[3]), + max(m[0], m[2]), + max(m[1], m[3]), ) - def decimalize(self, x): - return utils.decimalize(x, self.pdf.precision) - @property def width(self): return self.bbox[2] - self.bbox[0] @@ -114,7 +100,7 @@ class Page(Container): @property def annots(self): def parse(annot): - rect = self.decimalize(annot["Rect"]) + rect = annot["Rect"] a = annot.get("A", {}) extras = { @@ -164,16 +150,11 @@ class Page(Container): def process_object(self, obj): kind = re.sub(lt_pat, "", obj.__class__.__name__).lower() - d = self.decimalize - def process_attr(item): k, v = item if k in ALL_ATTRS: res = resolve_all(v) - if k in DECIMAL_ATTRS: - return (k, d(res)) - else: - return (k, res) + return (k, res) else: return None @@ -194,7 +175,7 @@ class Page(Container): def point2coord(pt): x, y = pt - return (self.decimalize(x), self.height - self.decimalize(y)) + return (x, self.height - y) attr["points"] = list(map(point2coord, obj.pts)) @@ -272,15 +253,13 @@ class Page(Container): return utils.extract_words(self.chars, **kwargs) def crop(self, bbox, relative=False): - return CroppedPage(self, self.decimalize(bbox), relative=relative) + return CroppedPage(self, bbox, relative=relative) def within_bbox(self, bbox, relative=False): """ Same as .crop, except only includes objects fully within the bbox """ - return CroppedPage( - self, self.decimalize(bbox), relative=relative, crop_fn=utils.within_bbox - ) + return CroppedPage(self, bbox, relative=relative, crop_fn=utils.within_bbox) def filter(self, test_function): return FilteredPage(self, test_function) diff --git a/pdfplumber/pdf.py b/pdfplumber/pdf.py index dc86cd1..68d1f9c 100644 --- a/pdfplumber/pdf.py +++ b/pdfplumber/pdf.py @@ -22,14 +22,12 @@ class PDF(Container): stream, pages=None, laparams=None, - precision=0.001, password="", strict_metadata=False, ): self.laparams = None if laparams is None else LAParams(**laparams) self.stream = stream self.pages_to_parse = pages - self.precision = precision self.doc = PDFDocument(PDFParser(stream), password=password) self.rsrcmgr = PDFResourceManager() self.metadata = {} diff --git a/pdfplumber/table.py b/pdfplumber/table.py index af5593d..79b327d 100644 --- a/pdfplumber/table.py +++ b/pdfplumber/table.py @@ -496,8 +496,8 @@ class TableFinder(object): else: v_explicit.append( { - "x0": utils.decimalize(desc), - "x1": utils.decimalize(desc), + "x0": desc, + "x1": desc, "top": self.page.bbox[1], "bottom": self.page.bbox[3], "height": self.page.bbox[3] - self.page.bbox[1], @@ -530,8 +530,8 @@ class TableFinder(object): "x0": self.page.bbox[0], "x1": self.page.bbox[2], "width": self.page.bbox[2] - self.page.bbox[0], - "top": utils.decimalize(desc), - "bottom": utils.decimalize(desc), + "top": desc, + "bottom": desc, "orientation": "h", } ) diff --git a/pdfplumber/utils.py b/pdfplumber/utils.py index e17ad95..adc5acd 100644 --- a/pdfplumber/utils.py +++ b/pdfplumber/utils.py @@ -1,19 +1,15 @@ from pdfminer.utils import PDFDocEncoding from pdfminer.psparser import PSLiteral from pdfminer.pdftypes import PDFObjRef -from decimal import Decimal, ROUND_HALF_UP -import numbers from operator import itemgetter import itertools -from functools import lru_cache as cache DEFAULT_X_TOLERANCE = 3 DEFAULT_Y_TOLERANCE = 3 def cluster_list(xs, tolerance=0): - tolerance = decimalize(tolerance) - if tolerance == Decimal(0): + if tolerance == 0: return [[x] for x in sorted(xs)] if len(xs) < 2: return [[x] for x in sorted(xs)] @@ -33,7 +29,6 @@ def cluster_list(xs, tolerance=0): def make_cluster_dict(values, tolerance): - tolerance = decimalize(tolerance) clusters = cluster_list(set(values), tolerance) nested_tuples = [ @@ -145,34 +140,6 @@ def resolve_all(x): return x -@cache(maxsize=int(10e4)) -def _decimalize(v, q=None): - # Convert int-like - if isinstance(v, numbers.Integral): - return Decimal(int(v)) - - # Convert float-like - elif isinstance(v, numbers.Real): - if q is not None: - return Decimal(repr(v)).quantize(Decimal(repr(q)), rounding=ROUND_HALF_UP) - else: - return Decimal(repr(v)) - else: - raise ValueError(f"Cannot convert {v} to Decimal.") - - -def decimalize(v, q=None): - # If already a decimal, just return itself - if type(v) == Decimal: - return v - - # If tuple/list passed, bulk-convert - if isinstance(v, (tuple, list)): - return type(v)(decimalize(x, q) for x in v) - else: - return _decimalize(v, q) - - def is_dataframe(collection): cls = collection.__class__ name = ".".join([cls.__module__, cls.__name__]) @@ -193,13 +160,12 @@ def dedupe_chars(chars, tolerance=1): """ key = itemgetter("fontname", "size", "upright", "text") pos_key = itemgetter("doctop", "x0") - t = decimalize(tolerance) def yield_unique_chars(chars): sorted_chars = sorted(chars, key=key) for grp, grp_chars in itertools.groupby(sorted_chars, key=key): - for y_cluster in cluster_objects(grp_chars, "doctop", t): - for x_cluster in cluster_objects(y_cluster, "x0", t): + for y_cluster in cluster_objects(grp_chars, "doctop", tolerance): + for x_cluster in cluster_objects(y_cluster, "x0", tolerance): yield sorted(x_cluster, key=pos_key)[0] deduped = yield_unique_chars(chars) @@ -207,7 +173,6 @@ def dedupe_chars(chars, tolerance=1): def collate_line(line_chars, tolerance=DEFAULT_X_TOLERANCE): - tolerance = decimalize(tolerance) coll = "" last_x1 = None for char in sorted(line_chars, key=itemgetter("x0")): @@ -273,9 +238,6 @@ class WordExtractor: if s not in DEFAULT_WORD_EXTRACTION_SETTINGS: raise ValueError(f"{s} is not a valid WordExtractor parameter") - if s in ["x_tolerance", "y_tolerance"]: - val = decimalize(val) - setattr(self, s, val) def merge_chars(self, ordered_chars): @@ -418,8 +380,8 @@ def filter_objects(objs, fn): def get_bbox_overlap(a, b): - a_left, a_top, a_right, a_bottom = decimalize(a) - b_left, b_top, b_right, b_bottom = decimalize(b) + a_left, a_top, a_right, a_bottom = a + b_left, b_top, b_right, b_bottom = b o_left = max(a_left, b_left) o_right = min(a_right, b_right) o_bottom = min(a_bottom, b_bottom) @@ -440,7 +402,6 @@ def calculate_area(bbox): def clip_obj(obj, bbox): - bbox = decimalize(bbox) overlap = get_bbox_overlap(obj_to_bbox(obj), bbox) if overlap is None: @@ -587,7 +548,7 @@ def rect_to_edges(rect): top.update( { "object_type": "rect_edge", - "height": decimalize(0), + "height": 0, "y0": rect["y1"], "bottom": rect["top"], "orientation": "h", @@ -596,7 +557,7 @@ def rect_to_edges(rect): bottom.update( { "object_type": "rect_edge", - "height": decimalize(0), + "height": 0, "y1": rect["y0"], "top": rect["top"] + rect["height"], "doctop": rect["doctop"] + rect["height"], @@ -606,7 +567,7 @@ def rect_to_edges(rect): left.update( { "object_type": "rect_edge", - "width": decimalize(0), + "width": 0, "x1": rect["x0"], "orientation": "v", } @@ -614,7 +575,7 @@ def rect_to_edges(rect): right.update( { "object_type": "rect_edge", - "width": decimalize(0), + "width": 0, "x0": rect["x1"], "orientation": "v", } diff --git a/tests/test_basics.py b/tests/test_basics.py index 4ab0a7b..cf48a96 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -78,7 +78,7 @@ class Test(unittest.TestCase): page = self.pdf.pages[0] cropped = page.crop((10, 10, 40, 40)) recropped = cropped.crop((10, 15, 20, 25), relative=True) - target_bbox = pdfplumber.utils.decimalize((20, 25, 30, 35)) + target_bbox = (20, 25, 30, 35) assert recropped.bbox == target_bbox recropped_wi = cropped.within_bbox((10, 15, 20, 25), relative=True) @@ -148,7 +148,7 @@ class Test(unittest.TestCase): path = os.path.join(HERE, "pdfs/cupertino_usd_4-6-16.pdf") laparams = dict(line_margin=0.2) with pdfplumber.open(path, laparams=laparams) as pdf: - assert float(pdf.pages[0].chars[0]["top"]) == 66.384 + assert round(pdf.pages[0].chars[0]["top"], 3) == 66.384 def test_loading_pathobj(self): from pathlib import Path diff --git a/tests/test_convert.py b/tests/test_convert.py index 8b64e6f..3894ed2 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -30,8 +30,8 @@ class Test(unittest.TestCase): def test_json(self): c = json.loads(self.pdf.to_json()) - assert c["pages"][0]["rects"][0]["bottom"] == float( - self.pdf.pages[0].rects[0]["bottom"] + assert ( + c["pages"][0]["rects"][0]["bottom"] == self.pdf.pages[0].rects[0]["bottom"] ) def test_json_all_types(self): @@ -45,7 +45,7 @@ class Test(unittest.TestCase): def test_single_pages(self): c = json.loads(self.pdf.pages[0].to_json()) - assert c["rects"][0]["bottom"] == float(self.pdf.pages[0].rects[0]["bottom"]) + assert c["rects"][0]["bottom"] == self.pdf.pages[0].rects[0]["bottom"] def test_additional_attr_types(self): path = os.path.join(HERE, "pdfs/issue-67-example.pdf") @@ -54,14 +54,14 @@ class Test(unittest.TestCase): assert len(c["pages"][0]["images"]) def test_csv(self): - c = self.pdf.to_csv() + c = self.pdf.to_csv(precision=3) assert c.split("\r\n")[9] == ( "char,1,45.83,58.826,656.82,674.82,117.18,117.18,135.18,12.996," '18.0,12.996,,,,,,TimesNewRomanPSMT,,,,"(0, 0, 0)",,,18.0,,,,,Y,,1,' ) io = StringIO() - self.pdf.to_csv(io) + self.pdf.to_csv(io, precision=3) io.seek(0) c_from_io = io.read() assert c == c_from_io diff --git a/tests/test_dedupe_chars.py b/tests/test_dedupe_chars.py index a9492e1..64a53b7 100644 --- a/tests/test_dedupe_chars.py +++ b/tests/test_dedupe_chars.py @@ -4,7 +4,6 @@ import os import logging import pdfplumber -from pdfplumber.utils import Decimal logging.disable(logging.ERROR) @@ -34,26 +33,26 @@ class Test(unittest.TestCase): def test_extract_words(self): page = self.pdf.pages[0] - x0 = Decimal("440.143") - x1_without_drop = Decimal("534.992") - x1_with_drop = Decimal("534.719") - top_windows = Decimal("791.849") - top_linux = Decimal("794.357") - bottom = Decimal("802.961") + x0 = 440.143 + x1_without_drop = 534.992 + x1_with_drop = 534.719 + top_windows = 791.849 + top_linux = 794.357 + bottom = 802.961 last_words_without_drop = page.extract_words()[-1] last_words_with_drop = page.dedupe_chars().extract_words()[-1] - assert last_words_without_drop["x0"] == x0 - assert last_words_without_drop["x1"] == x1_without_drop - assert last_words_without_drop["top"] in (top_windows, top_linux) - assert last_words_without_drop["bottom"] == bottom + assert round(last_words_without_drop["x0"], 3) == x0 + assert round(last_words_without_drop["x1"], 3) == x1_without_drop + assert round(last_words_without_drop["top"], 3) in (top_windows, top_linux) + assert round(last_words_without_drop["bottom"], 3) == bottom assert last_words_without_drop["upright"] == 1 assert last_words_without_drop["text"] == "名名模模意意义义一一些些有有意意义义一一些些" - assert last_words_with_drop["x0"] == x0 - assert last_words_with_drop["x1"] == x1_with_drop - assert last_words_with_drop["top"] in (top_windows, top_linux) - assert last_words_with_drop["bottom"] == bottom + assert round(last_words_with_drop["x0"], 3) == x0 + assert round(last_words_with_drop["x1"], 3) == x1_with_drop + assert round(last_words_with_drop["top"], 3) in (top_windows, top_linux) + assert round(last_words_with_drop["bottom"], 3) == bottom assert last_words_with_drop["upright"] == 1 assert last_words_with_drop["text"] == "名模意义一些有意义一些" diff --git a/tests/test_display.py b/tests/test_display.py index 81b8d6b..b2a4cf5 100644 --- a/tests/test_display.py +++ b/tests/test_display.py @@ -45,3 +45,10 @@ class Test(unittest.TestCase): page = pdfplumber.open(path).pages[0] im = page.to_image() im.draw_lines(page.curves) + + def test_cropped(self): + im = self.pdf.pages[0].crop((10, 20, 30, 50)).to_image() + assert im.original.size == (20, 30) + + def test_copy(self): + assert self.im.copy().original == self.im.original diff --git a/tests/test_issues.py b/tests/test_issues.py index d04d852..a0ab999 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -72,7 +72,9 @@ class Test(unittest.TestCase): p0 = pdf.pages[0] checklines = [ - line for line in p0.lines if line["height"] == line["width"] + line + for line in p0.lines + if round(line["height"], 2) == round(line["width"], 2) ] # These are diagonals rects = filter_rects(p0.objects["rect"]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7bd2ede..b0ad273 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,6 @@ import pdfplumber from pdfplumber import utils from pdfminer.pdfparser import PDFObjRef from pdfminer.psparser import PSLiteral -from decimal import Decimal from itertools import groupby from operator import itemgetter import os @@ -52,16 +51,6 @@ class Test(unittest.TestCase): a_res = utils.resolve_all(a) assert a_res[0]["info"]["Producer"] == self.pdf.doc.info[0]["Producer"] - def test_decimalize(self): - d = Decimal("1.011") - assert utils.decimalize(1.011) == d - assert [utils.decimalize(1.011)] == [d] - assert utils.decimalize(d) == d - assert id(utils.decimalize(d)) == id(d) - assert utils.decimalize(1) == Decimal("1") - with pytest.raises(ValueError): - utils.decimalize("1") - def test_decode_psl_list(self): a = [PSLiteral("test"), "test_2"] assert utils.decode_psl_list(a) == ["test", "test_2"] @@ -79,7 +68,7 @@ class Test(unittest.TestCase): assert words[0]["direction"] == 1 assert "size" not in words[0] - assert float(words_attr[0]["size"]) == 9.960 + assert round(words_attr[0]["size"], 2) == 9.96 assert words_w_spaces[0]["text"] == "Agaaaaa: AAAA"