From a5e7d7fa528061b778ab4e94ddffeb36f7966ab7 Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Fri, 14 Aug 2020 21:10:17 -0400 Subject: [PATCH] Remove pandas from dev requirements and tests Results in needing one more # pragma: nocover statement, but on a simple line of code. See PR #253 for details and motivation. --- pdfplumber/utils.py | 2 +- requirements-dev.txt | 1 - tests/pdfs/pdffill-demo.pdf | Bin 120145 -> 108017 bytes tests/test_basics.py | 1 - tests/test_ca_warn_report.py | 3 +- tests/test_convert.py | 1 - tests/test_display.py | 1 - tests/test_issues.py | 1 - tests/test_la_precinct_bulletin.py | 121 ----------------------------- tests/test_nics_report.py | 31 -------- tests/test_table.py | 1 - tests/test_utils.py | 50 +++++++++++- 12 files changed, 51 insertions(+), 162 deletions(-) delete mode 100644 tests/test_la_precinct_bulletin.py diff --git a/pdfplumber/utils.py b/pdfplumber/utils.py index 006f956..20fea02 100644 --- a/pdfplumber/utils.py +++ b/pdfplumber/utils.py @@ -163,7 +163,7 @@ def is_dataframe(collection): def to_list(collection): if is_dataframe(collection): - return collection.to_dict("records") + return collection.to_dict("records") # pragma: nocover else: return list(collection) diff --git a/requirements-dev.txt b/requirements-dev.txt index 99199a9..46d8893 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ -pandas>=1.0.0 pytest pytest-cov pytest-parallel diff --git a/tests/pdfs/pdffill-demo.pdf b/tests/pdfs/pdffill-demo.pdf index a63d021dd2d9ec548ffa63563555871152cd8fde..dcc7eb3227c06c849da100606ca146eaaa00280c 100644 GIT binary patch delta 11 Tcmcb(iv8nhwuUW?AD03EB>V;t delta 12178 zcmcIq30zI-`>#aASZ1tY97Y*i-Fwe=?;zS`E3y=lyQZ|5io)c`SSAV~k!_MKWGiDy zk$oH4V-QL9oyn5M@B5y6Z;JWPG@t*c+j7r&&bvIz_j%sudC&XeRIPIxEv9CgzE~^B zcuGQrssb&$cek|buhj%79YdoisZ;{;Aj;CtDKsQP8xj!?lGH#Nqzw-pAEwq4n#|ZN zEOeZ{BW<9#g@ukEhZc-z?xInU{MhSSL;jGEJNG$-ZM=qAB8`jB?c z4peuUPNrkz7*x%37>U6sXepD>Qd)-A49DSGB9m}D>EQ%E5;6zC3dC()A(!-cCNUFCXWsu+CEHEc{ z8bm=b!?)fSD1+D(@Z@NjB1$FK@U$;eqHu4(ks@)Ax)%V?I+1%{^;@L4HGB7l-(7!1fN(E+Z^D_8|B z;%Km-;uJj1(}XdM#;FXWx$PrE@KodZW%HpVN%ZV7@7y0 zG^Un=Jp>)FK&DbKaDI%$B1DmIu!-joiwH(5cvgVKP!oRjlJj^ZXE{g+xzGT2yqu_& zld(ioDySUO@e(osBeo-S9ug9b(GpgP-b4VozCUTtF+}B@hRDD&`f<>X$H)RrCS58lVc*9^n={M0D7s3n#s3*LIiB+2sW2_?UzC79B1YM2`;1>=yIe2XSDm_h_MF&)rB+Kb~Vi6#U_DN2Hcs(d0j+=SQ)4nh+> zl%t=}A&075awlHlATY+!fFS`O00USo0V1FuZe_4Fw8I%D)DnrH zTY?SP1d@v&1Y>%J)S~wg(?B!gLM#K)321q^C$4}TxS~u)1Im?u1O|eq2d|5fPfKCc;FmP3Qidf!i^XkI0Y>sbWx#7Mh(0ZrV))F}lJ)dtin;k6=KK_T{45!)azsqv*GCgK&C2x!;4 z8LUj>9s==siBTeCVhRW#F}f!Y9YBLRnrHEihyvR~-I$sP23W($ zJdh#Nu}I4pP7rNQ6om(3Lyc%-(h(Cwsg>Lbrcw|$gCPJ)5^V89!WGyMRVo$7s0=cq zYOEA+4;AL&7&($qVj@CE{>{sD@K0s=ABlND1Of!s4%kW``LqU=A(|1C!zcbLdrxhf@ffMWCkRZ2!vw< z8ekn3Csx-74R|ZAh#koqE`_YPhMN&okd>c63T)v&v}fpp#TWNV5;}2bnklM`9H?S1>Zt2v%0%UV*N}!H{N?g>oE%7A-7 zPK`%wQd<4UKyu3QzuXQZgQpfn|%+k@yEs1Oe#*gE9g4B>tfb?g>o_ zb07^7^`ihVrO0gH^KePTMVdGjVgq6T5*1`kkQjcCz(V33Fatfo93)2Y_>Z-f+=F3# zJXON~K_Idq_LBi%Z7Fgz4emLxL4qqnG3mgm@LUD&mLgQapBup(b# zNhQUNqgeXOjCFj44FZ{=Oef5QM}%pW!In{`{XI_wACc6{nrvw+Jip$1?vuC6>r89) zL$9-|XXTr1z1rscVvj-cUcKkEv);e0a7L`#!sz{NqK-8vxa>7<`TR*|YFM>3y;+if z>${arez(6bvlwF$vgnslMFFU$-x$q#I`F80U*w=QFu zR$UM4!f(}G6&y4E*c~sQb)K(Xp9(qGhS}PUkIr>^b*JvP0~gxWv`Ffj&`jF6s9F5N zRdeE%Lz<<&rB01Zd&!RGFOFSW<|2=-&uhc)ZuU99_Ds-wf8C;;R@&ba10S4NVQX(a z{Q9Wmu{&J$T&cb68WsQB_G`<1Z!Es3yc`g}W7qW?GoLiLniw2MN@{~HECu5hhgkF& z(W2kt5VvN?-hgq7LuTAM_)l9L|G2{;RL2&F%m{R{#X%eQF3NY3#2pU9P6q0~PiXu9 zZDnBVKtv!>qH!k!@T16Ig@RQ?q*2kHLRhHUON+e-+PF306dDv7<~2^K)`F-n*gx~@ zPo*`&?59HfzZK}BEkjkPq5mbs{9}X>sz1d1 zV}uc^Kg1X#jA%52iQS!?9F^f(4JE*s`i;|yI1|;ZZ>rxB{2Mvoj&YpY3hsPo-=H!&X=>ps=fZ02;E3C=ZL8#n4G4Al%iVCpujo!X^w-Jhqa^6R{8 z=y^1EptDQyjkvLOo~5&|3X;o?`P{sEZ&&_R|7X$pdrdq)Ofx?i;cYsVN%3y{^il4} zufI8U_>EcfhNktsO%D|6bmnevoy{(O$nLRp>4x4%GfJM?PSs>x>$2a|Cj9b8A)y`b z)V`L9rOPF*`xL>ZS+Ut7uZ$xtWFO5>zE12Yn<R%o8?wrSugThT$jkWE3U*JnuO$$A_f2QKi(o;`Mhjv@ntY^%3?oDW|OYyq( z>-svMK3%VVQl6&lO~a2x2NVInpE%TI`KklWUd*^ZWZ?Zpb*9IT$(qqU#(Tc4#&WuF zsZeOnb+BkLWyRRWW5)ZLM9)xo?sm609@@ji>AM>3LwZg!i+(hsq15tt&b~3}30uSG z39?Cl-Fx4r?#S68h5a1bdEG6kH%}90y}C<_)SJ_Ta?QS;ZxQjUB@<&SG3zq=t5#F# zaWx7klj&1?H>|O^hHY=NfL`W19PF={SUO0rm`?aPaGc4-m`P{N2TbF})f#C&?7&y! z>eid;*2{8cZeNLMK}Xvg3mM@$g=X#u3KC%(6k@Z+5twPvL#mo#sBv_;V@pSVKRP~~M+AIYU9>Lo+_HM`5{ z8qzWErma<7+P==a!{(k7>Tr#g#dkeCbo9K@Beo@NuW?2BxLd3G#}2f(-D=#F-OWeJ zLf9?lyBg%jWleo_=&M4T7g75<6*PV}twC=1bJb66b1a2c!nZf)wK=-K!~8nJaPzU# z8nj(ZwGV1PxS>_;d$r4^H#*1+jM?B3(QanLiHAq3I&PnP#bKX&E1Lle8@tbJ-)vC( zj!QFR2D=Bg^R?+=lihaVi61m>Vab9XNu7#Ngd(3cmNnV|(`bCwjidFr5n>{kndf$SW z1-riXNbB`r+j;A?ZLJ(Htetp%cV_?0AO6t(A!+T#uPME-@WGMSZSH5kuA{Zqjwt@V zxK*)LamGnuP_2WK(QoNNE!M^@jLV9f_lmi@C$A{)8n-oI)W>}t6W?_T>GV@#yF^{0 zY2v(%{f2~IYJ6$M#^8+}7bSr4{x5AJ4c;~~chuikXPTKuG z@XtdJdpuldm28z`)xv6sRT6WQZIK+3d?L9x`5tpDbIkC%I|cLM#l!FI93@Fy`tMN0aXC+oYOvtM&#Q;&4@h7522)Bzc1zVTgPJA!xEHFB3%q@zb)J8wI`tplH> z*biKlzG}^WxBVl8lLyXHe~g~vuq<~&%IuW=%9YC0LiKkW6E>aelsY`M+fljtw;yM} zurC^6vn_Uo`vUj)6aBre2p0TTD3d^WpY{#9|P`|+Pr zCoX%}xo(d+-g8zoz0uUK%abl%T_1I6 zH}39CPwRSf!u}`G-I9H`-DsLMASyEYFy_haGiR$&RY8rwL(sBtM5 zhg?f68vf}0>Hhig_a+<;K78X!kE><7OFK8{+aSSh&ap-77v8hVt2^qKq^-jehHc%o z;En28;IsAiKiJ3G$J>9CGa+YFwB1`}nOWI{_cvY%`OojY&oA{bmz!^{bKi1}xvTk5 z3;VjVI$zfuZ+hqHRmJAVGS+no5OeVazQo^U(ja>=c~%Vd`%m%d5fU3@a`xW)~Sqt^9k z+NNcb$?YeErM(f(-{?E5=<#jKqNld>$i*3t{|MVzoOUX2%Y46FpC3DY+p{B-*a7Tf^n|9U7fu|eYeSIhSPROa&CtjT> zoxNH%^yKf0DhnWNm5WtqHXwuitpE_m%fjmCvZFXSdw)O)ozEa`nqyS;HRZ z3{{T!add-#d762dvJE@3`)mz#yR`n{nujq~^OEmoKb*Yl$%;RYUCZBpGg!D_8eijx zh5fgUg;x87<8M62raJ^5QC=_Hd?@eH>W6#x`wM=8^ON2$67t+;xUD^QaNhV!-7eH*Zu0m+0h#xbSPU@ z^4r4bAxW?x_Ioq$oL{kJ+m?b;F-coTz1S0$b}hT~!;qv+{#z8j-J(}S zXFr))7JvEMtd)!X9#08)u=81P_JeU5Ws8f_=jPOmzW3g@)UKdpZ*=k8k1JZ%m{aTh zQ`?-$A4gs9@gTaO?3_hfgHb}y)05<7W6IQzi&l5A^ysv5W6$)mOKgc{6uA*?mlWx7FiN&Aha{&aRoR58cw;^5Wjb-I%of^^xbN;*u^D&Dh`ZuixGz zp4+r)N@Quazv{%{=VQ-3TH3Qu!G;{g`^zQ4_cZtKT!;*Kwr>6Gf=xL+OJd4WO4Y*a zV+ju)d%X2XTovCl@~1i+arsGyPR*#X7w_iT?D8^vgQQSe=nQ7O_rZ47G2i z#8)I5^KikKh*Q{U!hR9n9#^PUMUpYcu9g&8s6vihohh{Y2Op%2uMZf9v95(*Rhm1kK(T*d>Ey<@MjYAIp84YJ!zM#p`OKc}p=&T^;Ge7IB zz$w_zXbK!G`I4rOwN;IidL0;}wpa~M3p_WK06Ga;PB@G7%Ds?99$T8iD z0}`5?yj-deNN*)H!%+P$c=pk zqdlcK0PwkuLE}ERF 155)] - right = int(collate_chars(_right)) if len(_right) else None - _mid = chars[(chars["x0rel"] > 125) & (chars["x0rel"] < 155)] - mid = collate_chars(_mid) if len(_mid) else None - return { "text": left, "aff": mid, "votes": right } - -class PrecinctPage(object): - def __init__(self, page): - self.chars = pd.DataFrame(page.chars) - self.lines = pd.DataFrame(page.lines) - self.rects = pd.DataFrame(page.rects) - self.bboxes = self.get_bboxes() - - def get_bboxes(self): - outer, inner = [ r for i, r in self.rects.iterrows() ] - col_top = inner["top"] + inner["height"] - col_bot = outer["top"] + outer["height"] - line_xs = self.lines["x0"].tolist() - return { - "h1": (outer["x0"], outer["top"], outer["x1"], inner["top"]), - "h2": (outer["x0"], inner["top"], outer["x1"], col_top), - "c1": (outer["x0"], col_top, line_xs[0], col_bot), - "c2": (line_xs[0], col_top, line_xs[1], col_bot), - "c3": (line_xs[1], col_top, line_xs[2], col_bot), - "c4": (line_xs[2], col_top, outer["x1"], col_bot), - } - - @property - def precinct(self): - h1_left = list(self.bboxes["h1"]) - h1_left[-2] = float(h1_left[-2]) / 2 - h1_left_chars = intersects_bbox(self.chars, h1_left) - txt = h1_left_chars.groupby("top").apply(collate_chars).iloc[-1] - p_id = "|".join(re.split(r"\s{2,}", txt)[1:3]) - return p_id - - @property - def ballots_cast(self): - h2_chars = within_bbox(self.chars, self.bboxes["h2"]) - txt = h2_chars.groupby("top").apply(collate_chars).iloc[0] - return int(re.match(r"(\d+) BALLOTS CAST", txt).group(1)) - - @property - def registered_voters(self): - h2_chars = within_bbox(self.chars, self.bboxes["h2"]) - txt = h2_chars.groupby("top").apply(collate_chars).iloc[1] - return int(re.match(r"(\d+) REGISTERED VOTERS", txt).group(1)) - - def parse_col(self, col_chars): - c = col_chars.copy() - c["x0rel"] = c["x0"] - c["x0"].min() - results_lines = c.groupby("top").apply(parse_results_line) - items = [] - item = {} - vote_seen = False - for i, r in results_lines.iteritems(): - if r["votes"] == None: - if vote_seen == True: - items.append(item) - item = {} - vote_seen = False - item["desc"] = item["desc"] + "|" + r["text"] if item.get("desc", False) else r["text"] - if type(r["votes"]) == int: - vote_seen = True - item["options"] = item.get("options", []) - item["options"].append(r) - items.append(item) - return items - - @property - def results(self): - r = [] - for col in [ "c1", "c2", "c3", "c4" ]: - b = within_bbox(self.chars, self.bboxes[col]) - r += self.parse_col(b) - return r - - def to_dict(self): - return { - "precinct": self.precinct, - "registered_voters": self.registered_voters, - "ballots_cast": self.ballots_cast, - "results": self.results - } - -class Test(unittest.TestCase): - @classmethod - def setup_class(self): - path = os.path.join(HERE, "pdfs/la-precinct-bulletin-2014-p1.pdf") - self.pdf = pdfplumber.open(path) - self.PDF_WIDTH = self.pdf.pages[0].width - - @classmethod - def teardown_class(self): - self.pdf.close() - - def test_pandas(self): - p1 = PrecinctPage(self.pdf.pages[0]).to_dict() - assert(p1["registered_voters"] == 1100) - assert(p1["ballots_cast"] == 327) - assert(p1["precinct"] == "0050003A|ACTON") - last = p1["results"][-1] - assert(last["desc"] == "ANTELOPE VALLEY HEALTH BD") - assert(last["options"][-1]["text"] == "ROE LEER") - assert(last["options"][-1]["votes"] == 39) diff --git a/tests/test_nics_report.py b/tests/test_nics_report.py index a2a7344..dfa4815 100644 --- a/tests/test_nics_report.py +++ b/tests/test_nics_report.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import unittest -import pandas as pd import pdfplumber from operator import itemgetter from pdfplumber.utils import within_bbox, collate_chars @@ -86,36 +85,6 @@ class Test(unittest.TestCase): month_text = collate_chars(month_chars) assert(month_text == "November - 2015") - def test_pandas(self): - page = self.pdf.pages[0] - cropped = page.crop((0, 80, self.PDF_WIDTH, 485)) - table = cropped.extract_table({ - "horizontal_strategy": "text", - "explicit_vertical_lines": [ - min(map(itemgetter("x0"), cropped.chars)) - ], - "intersection_tolerance": 5 - }) - - table = pd.DataFrame(table) - - def parse_value(x): - if pd.isnull(x) or x == "": return None - return int(x.replace(",", "")) - - table.columns = COLUMNS - table[table.columns[1:]] = table[table.columns[1:]].applymap(parse_value) - - # [1:] because first column is state name - for c in COLUMNS[1:]: - total = table[c].iloc[-1] - colsum = table[c].sum() - assert(colsum == (total * 2)) - - month_chars = within_bbox(page.chars, (0, 35, self.PDF_WIDTH, 65)) - month_text = collate_chars(month_chars) - assert(month_text == "November - 2015") - def test_filter(self): page = self.pdf.pages[0] def test(obj): diff --git a/tests/test_table.py b/tests/test_table.py index 7f1e1fd..114582c 100644 --- a/tests/test_table.py +++ b/tests/test_table.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest import pytest -import pandas as pd import pdfplumber from pdfplumber import table import sys, os diff --git a/tests/test_utils.py b/tests/test_utils.py index 8ffb49a..ece054a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest import pytest -import pandas as pd import pdfplumber from pdfplumber import utils from pdfminer.pdfparser import PDFObjRef @@ -98,6 +97,55 @@ class Test(unittest.TestCase): assert text == goal assert self.pdf.pages[0].crop((0, 0, 1, 1)).extract_text() == None + def test_intersects_bbox(self): + objs = [ + # Is same as bbox + { + "x0": 0, + "top": 0, + "x1": 20, + "bottom": 20, + }, + # Inside bbox + { + "x0": 10, + "top": 10, + "x1": 15, + "bottom": 15, + }, + # Overlaps bbox + { + "x0": 10, + "top": 10, + "x1": 30, + "bottom": 30, + }, + # Touching on one side + { + "x0": 20, + "top": 0, + "x1": 40, + "bottom": 20, + }, + # Touching on one corner + { + "x0": 20, + "top": 20, + "x1": 40, + "bottom": 40, + }, + # Fully outside + { + "x0": 21, + "top": 21, + "x1": 40, + "bottom": 40, + }, + ] + bbox = utils.obj_to_bbox(objs[0]) + + assert utils.intersects_bbox(objs, bbox) == objs[:4] + def test_resize_object(self): obj = { "x0": 5,