saberbud/pdfplumber
1
0
Fork 0
mirror of https://github.com/jsvine/pdfplumber.git synced 2026-08-29 16:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove pandas from dev requirements and tests

Browse Source 
Results in needing one more # pragma: nocover statement, but on a
simple line of code. See PR #253 for details and motivation.
This commit is contained in:
Jeremy Singer-Vine
2020-08-14 21:10:17 -04:00
parent 90f767ff24
commit a5e7d7fa52
12 changed files with 51 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pdfplumber/utils.py
+1 -1
View File
@@ -163,7 +163,7 @@ def is_dataframe(collection):
def to_list(collection):
if is_dataframe(collection):
return collection.to_dict("records")
return collection.to_dict("records") # pragma: nocover
else:
return list(collection)
requirements-dev.txt
-1
View File
@@ -1,4 +1,3 @@
pandas>=1.0.0
pytest
pytest-cov
pytest-parallel
tests/pdfs/pdffill-demo.pdf
BIN
View File
Binary file not shown.
tests/test_basics.py
-1
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env python
import unittest
import pytest
import pandas as pd
import pdfplumber
import sys, os
tests/test_ca_warn_report.py
+1 -2
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python
import unittest
import pandas as pd
import pdfplumber
from pdfplumber import utils
from pdfplumber import table
@@ -37,7 +36,7 @@ class Test(unittest.TestCase):
assert len(self.pdf.figures)
assert len(self.pdf.images)
def test_pandas(self):
def test_parse(self):
rect_x0_clusters = utils.cluster_list([ r["x0"]
for r in self.pdf.pages[1].rects ], tolerance=3)
tests/test_convert.py
-1
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env python
import unittest
import pytest
import pandas as pd
import pdfplumber
from subprocess import Popen, PIPE
from io import StringIO
tests/test_display.py
-1
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python
import unittest
import pandas as pd
import pdfplumber
import sys, os, io
tests/test_issues.py
-1
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python
import unittest
import pandas as pd
import pdfplumber
import sys, os
import six
tests/test_la_precinct_bulletin.py
-121
View File
@@ -1,121 +0,0 @@
#!/usr/bin/env python
import unittest
import pandas as pd
import pdfplumber
from pdfplumber.utils import intersects_bbox, within_bbox, collate_chars
import sys, os
import re
import logging
logging.disable(logging.ERROR)
HERE = os.path.abspath(os.path.dirname(__file__))
def parse_results_line(chars):
_left = chars[chars["x0rel"] < 125]
left = collate_chars(_left) if len(_left) else None
_right = chars[(chars["x0rel"] > 155)]
right = int(collate_chars(_right)) if len(_right) else None
_mid = chars[(chars["x0rel"] > 125) & (chars["x0rel"] < 155)]
mid = collate_chars(_mid) if len(_mid) else None
return { "text": left, "aff": mid, "votes": right }
class PrecinctPage(object):
def __init__(self, page):
self.chars = pd.DataFrame(page.chars)
self.lines = pd.DataFrame(page.lines)
self.rects = pd.DataFrame(page.rects)
self.bboxes = self.get_bboxes()
def get_bboxes(self):
outer, inner = [ r for i, r in self.rects.iterrows() ]
col_top = inner["top"] + inner["height"]
col_bot = outer["top"] + outer["height"]
line_xs = self.lines["x0"].tolist()
return {
"h1": (outer["x0"], outer["top"], outer["x1"], inner["top"]),
"h2": (outer["x0"], inner["top"], outer["x1"], col_top),
"c1": (outer["x0"], col_top, line_xs[0], col_bot),
"c2": (line_xs[0], col_top, line_xs[1], col_bot),
"c3": (line_xs[1], col_top, line_xs[2], col_bot),
"c4": (line_xs[2], col_top, outer["x1"], col_bot),
}
@property
def precinct(self):
h1_left = list(self.bboxes["h1"])
h1_left[-2] = float(h1_left[-2]) / 2
h1_left_chars = intersects_bbox(self.chars, h1_left)
txt = h1_left_chars.groupby("top").apply(collate_chars).iloc[-1]
p_id = "|".join(re.split(r"\s{2,}", txt)[1:3])
return p_id
@property
def ballots_cast(self):
h2_chars = within_bbox(self.chars, self.bboxes["h2"])
txt = h2_chars.groupby("top").apply(collate_chars).iloc[0]
return int(re.match(r"(\d+) BALLOTS CAST", txt).group(1))
@property
def registered_voters(self):
h2_chars = within_bbox(self.chars, self.bboxes["h2"])
txt = h2_chars.groupby("top").apply(collate_chars).iloc[1]
return int(re.match(r"(\d+) REGISTERED VOTERS", txt).group(1))
def parse_col(self, col_chars):
c = col_chars.copy()
c["x0rel"] = c["x0"] - c["x0"].min()
results_lines = c.groupby("top").apply(parse_results_line)
items = []
item = {}
vote_seen = False
for i, r in results_lines.iteritems():
if r["votes"] == None:
if vote_seen == True:
items.append(item)
item = {}
vote_seen = False
item["desc"] = item["desc"] + "|" + r["text"] if item.get("desc", False) else r["text"]
if type(r["votes"]) == int:
vote_seen = True
item["options"] = item.get("options", [])
item["options"].append(r)
items.append(item)
return items
@property
def results(self):
r = []
for col in [ "c1", "c2", "c3", "c4" ]:
b = within_bbox(self.chars, self.bboxes[col])
r += self.parse_col(b)
return r
def to_dict(self):
return {
"precinct": self.precinct,
"registered_voters": self.registered_voters,
"ballots_cast": self.ballots_cast,
"results": self.results
}
class Test(unittest.TestCase):
@classmethod
def setup_class(self):
path = os.path.join(HERE, "pdfs/la-precinct-bulletin-2014-p1.pdf")
self.pdf = pdfplumber.open(path)
self.PDF_WIDTH = self.pdf.pages[0].width
@classmethod
def teardown_class(self):
self.pdf.close()
def test_pandas(self):
p1 = PrecinctPage(self.pdf.pages[0]).to_dict()
assert(p1["registered_voters"] == 1100)
assert(p1["ballots_cast"] == 327)
assert(p1["precinct"] == "0050003A|ACTON")
last = p1["results"][-1]
assert(last["desc"] == "ANTELOPE VALLEY HEALTH BD")
assert(last["options"][-1]["text"] == "ROE LEER")
assert(last["options"][-1]["votes"] == 39)
tests/test_nics_report.py
-31
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python
import unittest
import pandas as pd
import pdfplumber
from operator import itemgetter
from pdfplumber.utils import within_bbox, collate_chars
@@ -86,36 +85,6 @@ class Test(unittest.TestCase):
month_text = collate_chars(month_chars)
assert(month_text == "November - 2015")
def test_pandas(self):
page = self.pdf.pages[0]
cropped = page.crop((0, 80, self.PDF_WIDTH, 485))
table = cropped.extract_table({
"horizontal_strategy": "text",
"explicit_vertical_lines": [
min(map(itemgetter("x0"), cropped.chars))
],
"intersection_tolerance": 5
})
table = pd.DataFrame(table)
def parse_value(x):
if pd.isnull(x) or x == "": return None
return int(x.replace(",", ""))
table.columns = COLUMNS
table[table.columns[1:]] = table[table.columns[1:]].applymap(parse_value)
# [1:] because first column is state name
for c in COLUMNS[1:]:
total = table[c].iloc[-1]
colsum = table[c].sum()
assert(colsum == (total * 2))
month_chars = within_bbox(page.chars, (0, 35, self.PDF_WIDTH, 65))
month_text = collate_chars(month_chars)
assert(month_text == "November - 2015")
def test_filter(self):
page = self.pdf.pages[0]
def test(obj):
tests/test_table.py
-1
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env python
import unittest
import pytest
import pandas as pd
import pdfplumber
from pdfplumber import table
import sys, os
tests/test_utils.py
+49 -1
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env python
import unittest
import pytest
import pandas as pd
import pdfplumber
from pdfplumber import utils
from pdfminer.pdfparser import PDFObjRef
@@ -98,6 +97,55 @@ class Test(unittest.TestCase):
assert text == goal
assert self.pdf.pages[0].crop((0, 0, 1, 1)).extract_text() == None
def test_intersects_bbox(self):
objs = [
# Is same as bbox
{
"x0": 0,
"top": 0,
"x1": 20,
"bottom": 20,
},
# Inside bbox
{
"x0": 10,
"top": 10,
"x1": 15,
"bottom": 15,
},
# Overlaps bbox
{
"x0": 10,
"top": 10,
"x1": 30,
"bottom": 30,
},
# Touching on one side
{
"x0": 20,
"top": 0,
"x1": 40,
"bottom": 20,
},
# Touching on one corner
{
"x0": 20,
"top": 20,
"x1": 40,
"bottom": 40,
},
# Fully outside
{
"x0": 21,
"top": 21,
"x1": 40,
"bottom": 40,
},
]
bbox = utils.obj_to_bbox(objs[0])
assert utils.intersects_bbox(objs, bbox) == objs[:4]
def test_resize_object(self):
obj = {
"x0": 5,