diff --git a/pdfplumber/utils.py b/pdfplumber/utils.py index 006f956..20fea02 100644 --- a/pdfplumber/utils.py +++ b/pdfplumber/utils.py @@ -163,7 +163,7 @@ def is_dataframe(collection): def to_list(collection): if is_dataframe(collection): - return collection.to_dict("records") + return collection.to_dict("records") # pragma: nocover else: return list(collection) diff --git a/requirements-dev.txt b/requirements-dev.txt index 99199a9..46d8893 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ -pandas>=1.0.0 pytest pytest-cov pytest-parallel diff --git a/tests/pdfs/pdffill-demo.pdf b/tests/pdfs/pdffill-demo.pdf index a63d021..dcc7eb3 100644 Binary files a/tests/pdfs/pdffill-demo.pdf and b/tests/pdfs/pdffill-demo.pdf differ diff --git a/tests/test_basics.py b/tests/test_basics.py index 2fb747d..0a04bcb 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest import pytest -import pandas as pd import pdfplumber import sys, os diff --git a/tests/test_ca_warn_report.py b/tests/test_ca_warn_report.py index ce418ff..2b3ff92 100644 --- a/tests/test_ca_warn_report.py +++ b/tests/test_ca_warn_report.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import unittest -import pandas as pd import pdfplumber from pdfplumber import utils from pdfplumber import table @@ -37,7 +36,7 @@ class Test(unittest.TestCase): assert len(self.pdf.figures) assert len(self.pdf.images) - def test_pandas(self): + def test_parse(self): rect_x0_clusters = utils.cluster_list([ r["x0"] for r in self.pdf.pages[1].rects ], tolerance=3) diff --git a/tests/test_convert.py b/tests/test_convert.py index 8f5c481..cf11497 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest import pytest -import pandas as pd import pdfplumber from subprocess import Popen, PIPE from io import StringIO diff --git a/tests/test_display.py b/tests/test_display.py index 2a015b4..b0742a3 100644 --- a/tests/test_display.py +++ b/tests/test_display.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import unittest -import pandas as pd import pdfplumber import sys, os, io diff --git a/tests/test_issues.py b/tests/test_issues.py index a334bc5..d639208 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import unittest -import pandas as pd import pdfplumber import sys, os import six diff --git a/tests/test_la_precinct_bulletin.py b/tests/test_la_precinct_bulletin.py deleted file mode 100644 index c4ddfa0..0000000 --- a/tests/test_la_precinct_bulletin.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -import unittest -import pandas as pd -import pdfplumber -from pdfplumber.utils import intersects_bbox, within_bbox, collate_chars -import sys, os -import re - -import logging -logging.disable(logging.ERROR) - -HERE = os.path.abspath(os.path.dirname(__file__)) - -def parse_results_line(chars): - _left = chars[chars["x0rel"] < 125] - left = collate_chars(_left) if len(_left) else None - _right = chars[(chars["x0rel"] > 155)] - right = int(collate_chars(_right)) if len(_right) else None - _mid = chars[(chars["x0rel"] > 125) & (chars["x0rel"] < 155)] - mid = collate_chars(_mid) if len(_mid) else None - return { "text": left, "aff": mid, "votes": right } - -class PrecinctPage(object): - def __init__(self, page): - self.chars = pd.DataFrame(page.chars) - self.lines = pd.DataFrame(page.lines) - self.rects = pd.DataFrame(page.rects) - self.bboxes = self.get_bboxes() - - def get_bboxes(self): - outer, inner = [ r for i, r in self.rects.iterrows() ] - col_top = inner["top"] + inner["height"] - col_bot = outer["top"] + outer["height"] - line_xs = self.lines["x0"].tolist() - return { - "h1": (outer["x0"], outer["top"], outer["x1"], inner["top"]), - "h2": (outer["x0"], inner["top"], outer["x1"], col_top), - "c1": (outer["x0"], col_top, line_xs[0], col_bot), - "c2": (line_xs[0], col_top, line_xs[1], col_bot), - "c3": (line_xs[1], col_top, line_xs[2], col_bot), - "c4": (line_xs[2], col_top, outer["x1"], col_bot), - } - - @property - def precinct(self): - h1_left = list(self.bboxes["h1"]) - h1_left[-2] = float(h1_left[-2]) / 2 - h1_left_chars = intersects_bbox(self.chars, h1_left) - txt = h1_left_chars.groupby("top").apply(collate_chars).iloc[-1] - p_id = "|".join(re.split(r"\s{2,}", txt)[1:3]) - return p_id - - @property - def ballots_cast(self): - h2_chars = within_bbox(self.chars, self.bboxes["h2"]) - txt = h2_chars.groupby("top").apply(collate_chars).iloc[0] - return int(re.match(r"(\d+) BALLOTS CAST", txt).group(1)) - - @property - def registered_voters(self): - h2_chars = within_bbox(self.chars, self.bboxes["h2"]) - txt = h2_chars.groupby("top").apply(collate_chars).iloc[1] - return int(re.match(r"(\d+) REGISTERED VOTERS", txt).group(1)) - - def parse_col(self, col_chars): - c = col_chars.copy() - c["x0rel"] = c["x0"] - c["x0"].min() - results_lines = c.groupby("top").apply(parse_results_line) - items = [] - item = {} - vote_seen = False - for i, r in results_lines.iteritems(): - if r["votes"] == None: - if vote_seen == True: - items.append(item) - item = {} - vote_seen = False - item["desc"] = item["desc"] + "|" + r["text"] if item.get("desc", False) else r["text"] - if type(r["votes"]) == int: - vote_seen = True - item["options"] = item.get("options", []) - item["options"].append(r) - items.append(item) - return items - - @property - def results(self): - r = [] - for col in [ "c1", "c2", "c3", "c4" ]: - b = within_bbox(self.chars, self.bboxes[col]) - r += self.parse_col(b) - return r - - def to_dict(self): - return { - "precinct": self.precinct, - "registered_voters": self.registered_voters, - "ballots_cast": self.ballots_cast, - "results": self.results - } - -class Test(unittest.TestCase): - @classmethod - def setup_class(self): - path = os.path.join(HERE, "pdfs/la-precinct-bulletin-2014-p1.pdf") - self.pdf = pdfplumber.open(path) - self.PDF_WIDTH = self.pdf.pages[0].width - - @classmethod - def teardown_class(self): - self.pdf.close() - - def test_pandas(self): - p1 = PrecinctPage(self.pdf.pages[0]).to_dict() - assert(p1["registered_voters"] == 1100) - assert(p1["ballots_cast"] == 327) - assert(p1["precinct"] == "0050003A|ACTON") - last = p1["results"][-1] - assert(last["desc"] == "ANTELOPE VALLEY HEALTH BD") - assert(last["options"][-1]["text"] == "ROE LEER") - assert(last["options"][-1]["votes"] == 39) diff --git a/tests/test_nics_report.py b/tests/test_nics_report.py index a2a7344..dfa4815 100644 --- a/tests/test_nics_report.py +++ b/tests/test_nics_report.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import unittest -import pandas as pd import pdfplumber from operator import itemgetter from pdfplumber.utils import within_bbox, collate_chars @@ -86,36 +85,6 @@ class Test(unittest.TestCase): month_text = collate_chars(month_chars) assert(month_text == "November - 2015") - def test_pandas(self): - page = self.pdf.pages[0] - cropped = page.crop((0, 80, self.PDF_WIDTH, 485)) - table = cropped.extract_table({ - "horizontal_strategy": "text", - "explicit_vertical_lines": [ - min(map(itemgetter("x0"), cropped.chars)) - ], - "intersection_tolerance": 5 - }) - - table = pd.DataFrame(table) - - def parse_value(x): - if pd.isnull(x) or x == "": return None - return int(x.replace(",", "")) - - table.columns = COLUMNS - table[table.columns[1:]] = table[table.columns[1:]].applymap(parse_value) - - # [1:] because first column is state name - for c in COLUMNS[1:]: - total = table[c].iloc[-1] - colsum = table[c].sum() - assert(colsum == (total * 2)) - - month_chars = within_bbox(page.chars, (0, 35, self.PDF_WIDTH, 65)) - month_text = collate_chars(month_chars) - assert(month_text == "November - 2015") - def test_filter(self): page = self.pdf.pages[0] def test(obj): diff --git a/tests/test_table.py b/tests/test_table.py index 7f1e1fd..114582c 100644 --- a/tests/test_table.py +++ b/tests/test_table.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest import pytest -import pandas as pd import pdfplumber from pdfplumber import table import sys, os diff --git a/tests/test_utils.py b/tests/test_utils.py index 8ffb49a..ece054a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest import pytest -import pandas as pd import pdfplumber from pdfplumber import utils from pdfminer.pdfparser import PDFObjRef @@ -98,6 +97,55 @@ class Test(unittest.TestCase): assert text == goal assert self.pdf.pages[0].crop((0, 0, 1, 1)).extract_text() == None + def test_intersects_bbox(self): + objs = [ + # Is same as bbox + { + "x0": 0, + "top": 0, + "x1": 20, + "bottom": 20, + }, + # Inside bbox + { + "x0": 10, + "top": 10, + "x1": 15, + "bottom": 15, + }, + # Overlaps bbox + { + "x0": 10, + "top": 10, + "x1": 30, + "bottom": 30, + }, + # Touching on one side + { + "x0": 20, + "top": 0, + "x1": 40, + "bottom": 20, + }, + # Touching on one corner + { + "x0": 20, + "top": 20, + "x1": 40, + "bottom": 40, + }, + # Fully outside + { + "x0": 21, + "top": 21, + "x1": 40, + "bottom": 40, + }, + ] + bbox = utils.obj_to_bbox(objs[0]) + + assert utils.intersects_bbox(objs, bbox) == objs[:4] + def test_resize_object(self): obj = { "x0": 5,