From 5a85ab7a0c324dd815d2e4380080d538cf4d8a74 Mon Sep 17 00:00:00 2001 From: Andy Friedman Date: Wed, 1 Nov 2023 13:54:10 -0400 Subject: [PATCH] add x_tolerance_ratio --- CHANGELOG.md | 1 + pdfplumber/utils/text.py | 28 +++++++++++++++++++++++++++- tests/pdfs/issue-987-test.pdf | Bin 0 -> 8805 bytes tests/test_utils.py | 6 ++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/pdfs/issue-987-test.pdf diff --git a/CHANGELOG.md b/CHANGELOG.md index 55fea4d..43ebb6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. The format - Add support for marked-content sequences, represented by `mcid` and `tag` attributes on `char`/`rect`/`line`/`curve`/`image` objects (h/t @dhdaines). ([#961](https://github.com/jsvine/pdfplumber/pulls/961)) - Add `gs_path` argument to `pdfplumber.open(...)` and `pdfplumber.repair(...)`, to allow passing a custom Ghostscript path to be used for repairing. ([#953](https://github.com/jsvine/pdfplumber/issues/953)) +- Add `x_tolerance_ratio` to `extract_text` and similar functions, to account for text size when spacing characters (instead of a fixed number of pixels) ### Fixed diff --git a/pdfplumber/utils/text.py b/pdfplumber/utils/text.py index 4d946ca..366eea2 100644 --- a/pdfplumber/utils/text.py +++ b/pdfplumber/utils/text.py @@ -5,7 +5,7 @@ import string from operator import itemgetter from typing import Any, Dict, Generator, List, Match, Optional, Pattern, Tuple, Union -from .._typing import T_num, T_obj, T_obj_iter, T_obj_list +from .._typing import T_num, T_obj, T_obj_iter, T_obj_list, Iterable from .clustering import cluster_objects from .generic import to_list from .geometry import objects_to_bbox @@ -294,6 +294,8 @@ class WordExtractor: self, x_tolerance: T_num = DEFAULT_X_TOLERANCE, y_tolerance: T_num = DEFAULT_Y_TOLERANCE, + x_tolerance_ratio: T_num = None, + y_tolerance_ratio: T_num = None, keep_blank_chars: bool = False, use_text_flow: bool = False, horizontal_ltr: bool = True, # Should words be read left-to-right? @@ -303,7 +305,9 @@ class WordExtractor: expand_ligatures: bool = True, ): self.x_tolerance = x_tolerance + self.x_tolerance_ratio = x_tolerance_ratio self.y_tolerance = y_tolerance + self.y_tolerance_ratio = y_tolerance_ratio self.keep_blank_chars = keep_blank_chars self.use_text_flow = use_text_flow self.horizontal_ltr = horizontal_ltr @@ -343,6 +347,23 @@ class WordExtractor: word[key] = ordered_chars[0][key] return word + + def set_tolerances_from_ratio(self, t: T_obj, axis_range: Iterable='x'): + """ + If there is a `tolerance_ratio` for any axis, overrides the tolerance with ratio * size of `t`. Allows for dynamic tolerances to react to different text sizes within a single call. + + ie: If `x_tolerance_ratio=0.15`, sets `self.x_tolerance` to `set_tolerance(t, x_tolerance_ratio)` + + `axis_range` is there to restrict this to only `x_tolerance` until `y_tolerance` gets implemented (if ever). + """ + for i in axis_range: + if self.__getattribute__(f"{i}_tolerance_ratio") is not None: + self.__setattr__( + f"{i}_tolerance", + set_tolerance( + t, self.__getattribute__(f"{i}_tolerance_ratio") + ) + ) def char_begins_new_word( self, @@ -380,6 +401,7 @@ class WordExtractor: compare, while horizontal_ltr/vertical_ttb determine the direction of the comparison. """ + self.set_tolerances_from_ratio(curr_char) # Note: Due to the grouping step earlier in the process, # curr_char["upright"] will always equal prev_char["upright"]. @@ -462,6 +484,7 @@ class WordExtractor: upright = upright_cluster[0]["upright"] cluster_key = "doctop" if upright else "x0" + self.set_tolerances_from_ratio(upright_cluster[0]) # Cluster by line subclusters = cluster_objects( upright_cluster, itemgetter(cluster_key), self.y_tolerance @@ -583,3 +606,6 @@ def dedupe_chars(chars: T_obj_list, tolerance: T_num = 1) -> T_obj_list: deduped = yield_unique_chars(chars) return sorted(deduped, key=chars.index) + +def set_tolerance(t, tolerance_ratio): + return tolerance_ratio*(t['bottom'] - t['top']) \ No newline at end of file diff --git a/tests/pdfs/issue-987-test.pdf b/tests/pdfs/issue-987-test.pdf new file mode 100644 index 0000000000000000000000000000000000000000..487664fd2f1b9477ac851750179e7b97db1f86f3 GIT binary patch literal 8805 zcmbVS2|QHo_eaW-q=bk}g(%E!?915qz3hxJ7<*&v*|H@&NfeS@DTQnyOO|-;TZ=7Q z_C&Iz{&#qL-?q2k=kuTOxp(e!&+|R!p65KzJ>T=-){;{YfC|ECxf`cGOwH%6-sx&= zqJ@JXAZJTET5)j@SOM=q#1VkA1BQr`!(pARa3HW6&dG*o3xY%8AW2DDS0Vw2aisMm zPmG5;2SFG@JicB-J&}z%9P}ugcORE*m{OEozH3N)36tJk;=9g=hfW2YIL)Mbd^UAQ za>O)9f{nVYrE!V7jAlbO@s{MI4fcQD-D$@K}g*dm)1NxUNvgGESpeoLBpz$b-saKb{7L^;)*j zXX3OtC#%1G2>c>B9{Sw?=y|yS9@oOy>>dd?Cn5+2+|~hsb#SiEZUijO6@=Jpkacz< z0{2}(qyYp3)NxjL4BFWfWDEfspdv8f01<|m(gK6_w=uLhdS%lmy}e(7D^Jrw0WIk(vQ?QdIn{K=?Za z8h~M78B#ns0l@?U%i-MdSe%X``k#n`2|;1MkGb#TEcJ$mp6*NLu+C~!b=T3;Anr9b zZyHdKm=Y7EcDQnY7DZ?!kFzAZqAk7B`t)@C{Ir*vK zNPC*)Lgea5%1*Q4=*yY1!Iw76zJrxyTA!~_))NiL0}vSo$Cl>nOpY?PHGZNv&OlCY zK>lpp+L}^vLyn^FbG1Zra-LFC@%oZrfK}CN{#q>_*MYD8ai>wTTnET#<@<%i{dicW zD6Z=kKWBw~rF^lL$Oj9B!k9B??%qE=(fH6|J$L$iRqM?pp2-Y0!w%M`h_uKmt&tS& z@ytld43nGOV>X$Wsc_YL$|+=1pLlzFH*TwuL5?Mxx6kloIE2IGkFhR=f1n_vzTPPE zlt3S`DPo&O=9yzNd5tn$*)r$I;7$D0yB6!Mm15mZOX&IQ4eH}pJ=HqbO6Z~?ECCOl zZ^)B#8#*T2#w`}IFD@DmOxteOz8pR2r@JH3;=QfE!n4qsCKH)vIO?!)A6(5$w$5dA zi=((EQJ<{D@qqNy_E@ccW}cKQ|P z7qy`xpOV{_wl9d~ut)`-QD%bUNA?O!mw^ zE+;8(3g3BS1}JSy%~RXLxQDJWFz+ujTbu_?t~rgX$(+`mUZ;z(B5>d3XU!Z9cBrE` z8cRjILyHI$gi!EX9AXcEyX+eQkp%}RG3>jsPf&>hFHQMaMskRZRt7pm?jB_4Le>}P z-9@Q!MaboV3FXCShg=TQ1t?0>hSsS<$VUS4)K}g!Fx{0Yh-J zvPg`bTCu)uOwN<(HWkaW*W)VX%#!qF^(^Bj-jnf%nAayzOSe$(>sBzJKCvQ)jy2#S zKXS@G?sOJKzI;k7eb$MJD5bbdllu;YWngX|XKrGh2s4Zvx4eKEuvCHc-^AX$pvE#S zWX%8%+`N))+5afV;Ou=VcN(E1x1t0ZFIdD_7#Afy+Bbw*5M!rre#ZKi-Nmo`xCzV& z`GB(Q@L*(Bz}#NUC2-vFR7OZQI=1+n`n8}GQ~37GUzSxxi2O9oFo~ zgvhGMm{r7dMdw84YoUjDv#(G25;xB|ox7ZPI?+0jJTay~P2aiySbu7PV}WKL#E!=f zWB1Z-yI=5LRA+9dM_Om9IpXW&=dih=%5d-UExV5ma}sk2Y$FlR%c@X!I zRn2#c-WVF*R~Wb>(jeY)In(Rb=J|sX5e5;d%)`v){EPg$0`vTwhJ1$m4IE46inbbF zHKy2`Vtt!K?Iy6r9XZ3(rG@zdqwckfzG5k+MZ?Th8daU^^y{a$dA6Zc5@DWU)%2f+ zi0-^q!M^=R=HHE%cdSHCP@VNT`>0nrcQn=s6m&VK432}l} zf|2H)E*X9LuM;PX=C|6_2cu@(8yy>mha_I^ly9Futa><3F{1f;Zo&*(=RvcpNe?f^ zU3^#;|H*RMZaG&{R5DyLN|Le0z2=@TcmuOTvE#lqygK-PW@~U;lTw)SKFvGYEJ_8+ z3sjN^VKhhgyHPgOk=IpS;RN@XbD8upQ=mV~x`&9#^4}`QZ88`G4VqOEyY+(cGA8MdwZ75@)Ny4TwyLY~>4l|B^oAUFq&PHC|Uk>#Cd4 zU(`M-MRMKZ4l+DgZ1;G?MiH0(Lc9IQ^x9;~q*EJ5>uT$^AJQ7>;bv3g{w4Ht9dgN@ zzA$9qfywI%4G+`0%JGji^D&MfN2QnV+Z5WOa3@Bcu8s6*-_b6rv9K*(o^C={ptZ-V z1v-rB9iBTdIV=ua7Z=*SPC9MY{n4`R?X|a>+;8INM-6>1ZVtDv2QOAL zycwRYTs26xG&FnJ^o-4A6|7Z++SV2m$W9LC33#gxGE9#^Y8v{UTQoy(dF zWQ$3euAcKLTTFe`{Cco<)X~3}JZj%_D#??_*q{2heA2QnlyQ8Hc{6^$p>zK3TxG3| zzq!BMqS8uSr(%d=c5{7;F%R0l)|S( zR^B#ma4x6OiNEx^=36j(b|-!PNte+zql|$!1L04ygz!FUKBKb%y{Q}>90F3ezFuC7 zTu)9Zeqp298P~yd`<6($hu;TuQ^-n0I7^u5MWSS7$z)IJgQ5o`ZGlM-%~mQ1`LC%6E$$MD(!~(f**diblI_b)}?so#zbLs&3@mREz@oA$a6Z;|#Y%K}a_a~SCgBG2|S8ZZK%Voh|6E`1fHtg@r?>w(iqA;gesMr~~89D6zXzls?w#cO3iI7^p zFHb%tcHc|)^VqJov21O8Z{IziETuA%S0l1Du*bHO|eNCEGh8?l3it4Sv1BKX9XhVsyZOk zZ>cMklr?`#j`z}VT_V8^OVlIaa5~P;L=adNB(mG*-+Y&J{kuN$4md{;SQkioow06? zq%0n+1!Uv9y#W|w5R{bk>wtj#clWfH!b5j6ds^sTz75^YuR&1Mp2(iL&^-%)+@91A zwub`SLoo$?mx=Ex_@ikGcpEnYiDvx$RA4O|&>kLOA@m0<68;~tl>P$-_5;lKO!IpW z|KqJcngss=>w7BmTP&^rFbVMk%(uWGQJvqy{Lv)j4_Kt}`hR&z=m!|mc>fXRk0zmh z!1|t#{r2(xYv6=`fcc+c{zsEUzQ@{)+&@0ve+?WkeZGtD-O&Fd&L3?8rpNbS|1-e< z)2JUJ{8tzflXv}ZZcd`xZoujQ1r`^)l`Cms*qzv>-y8Sqw2{RSF%Hf)K!oqY?@fQO zI)-4c>}2h{OVCLdz#<2%U^oB^Sc`sG%V_^zi~e27zD3mzyYEO74;1Sg^SRH3ajMKj@3V4DmQPvhi00{{JwFJ!H7f>Lyz!&jW zL|c*{Q4j zi-ehhceT6$S|) zgrOiP0?-SEBA`GeM;QnbJXXfZ#sLQy1eS5dl3qk2C?u&Cvv(l?170s+S&WMk4sT;i zq(uR*ARtt5ju${sBydL!huIwkm(ed>va$UB}^JF&)qBF zEd~LUl8gk3iZ~eHF2D$#?>+MU+Pha{gn|$-Al$xH9AQW}p!at*$?xk!g%|A|ZrLzN zE^lwf3x_X)YnsX~ud|rji9KQzbC1u-PY}gdxg$)$5ymPh$XQ$#i-NIeqntnSe1ria zFgUc5@RCbwJmyI6^rqr#PR4UP#Q_1F{Azi0+Un?(<4Q`VN7FtPCm@Yc`nKuB%N;&q z%OagV9bV??zH@KrkiGKpc`lQaeXKo_4L%l~93uV~@=G_(-q&Q>M5o>_t5wa4;JD?x zWn1X@nO?i0a00H|TPoyBnI;w;ZFCD)Cl;M9?vpVf^z=YlnuKqbq|v;;Ss&d#U^CmFKt2M?Z|FUa8$4)Yv@JK&bjsu*5!%96;LIXO&&NJ2?op{Cr~T z9=fgaVFh}f@?L!O{YkZ=6I{C2U$x2>Law$Xm6P)+kGn|-%^9G=oMMxEZf3>WJ0^!b zzHTsx&E61|*W(rwTIV^}cQ;bfwchl8EMK760jiV`eI503`cnt=9gDKVjTi?X6$E`A zQsZHKMdVP0b)A$;GQ9Pyn>H@Vp@@pHAgh^;RjhkX1Ja>!cGl24&@43f!mCG?JQAp0 zM?uEWj-HFNNB``mgm8h5(GT++noIoeg%E4sEp1#3^UCpL7_Ijej_4SstS;}V> zn-BvXZ3WF^Z3_@K@$hI*hWP7xO)qpj7aq}+X0`LVzRsPw4xQTQBO0;{2p_PXR^iK5 z^Wo7L^H`QEGU4?aY)sa{lX(--hi_PHnb5GM-M1$jtkhAxnWed;^+Lz$vgIeaP0Ma$ z!DFf;7Fw(VqwMLtqx{}M*+;Ee2lzvtzooVlI)N!Z6}=LgazmnvoGH6PB!f}lB+U)y zj=#KS#=-|O51fyj<9M&v9#@~qbm4e9gY16fJ0=c0twZ(EI!7u`B#5o6Mb^jPDQaBf zQ_A<`%u=@xwRD2F+xLJRH?M$@Qgm52!mjjHTF;6q;f5TTVxLGJ*ZJs5W-43$o-7zV zz)-V67VJ3B@WfxTJ{nnt$zmK3MFg9gGt@7I$&@c6viZuBJesBbMZ99;NMFn;9cjOoBTabeSOk^dO}eLt^cD0L zj6$4v_Jxbt27mZ?(s2Xs=eaZO*#&Tod@58xq)`=l>D-r2J!eFJMZ%Y7f<>H{;ORFg zdNgLsi5CZR8OwMSyAy{v1PkEAM|apuT4j<=&1%}-P$lVJCM0jvWiNg{R3E9$Uw$P8 zkFJP@UMGgq3TDv5?GxW#tX#x-NDwv2Q6V@?&NwP-t)-h6a!Wr1#p29v1%JH3qG{+OJ%Ok}tBPA)|U?q9po z%yQ`wrAq_#SqhgtnkAYF8rb-?V=x(aX`Y*jOHV#pdsDo@pz_q{*XdS99JoXqy^Bpv z5+Resp}$P3SA7)`T`3XZ+UN9Hhn=Dh$7EFn1^K{hzu>G}avDt+UUfzlsuc7Z5aFmEsaFFa75Pl=L(sPr0pm=$$X))JB4u&;eG+xiTzrKF; zuThRmD@i&N&09TT9y#?vV%YA~AdI>AqAjPA=NA!zXA9@6k}{w7-GX5n?xGU(7oUw- z5SkM+nzvtxR}611bqm(!=l2hfe=2a#%&w+-Py!15bS5TxzG9g zACU{P$@EZY7tl8KTT!3 z%5f!DvAsWpf)af(S_u94kh0NOi+ir>qNO%mH;_ULCWj25Q%Ev|r37knG?iTEUs9E8 zR(yLXhsDjCd5a)wqU7_s3dIuIBWaC4`06d*w6y#6$@s!W!Qz*NEh~b#X&j5J;jQW_ z{&t(Pr527(_S(LMae)SprMPa<{Uw;&X5a16sEI(Z5#z1mCERZ{TfUdazO86!J^WR4 z%zU4j`K@~`*s9PAX96$>h5c8qGQJOgU)+bSqcD7GBwwraP$3zXT71iH?Z7k3PgIW3&t zvW&&@f%?NGg85RVn|ev+V{)IAt%kp<|{bMrK3L1CGex8-g&tmg!#~Sp_YEeKKvCT z)~YdYA2KY74f!9e;981)D(r4z>Y~6dMm0ND9ec=ar2Hy>?}SBTp}{R)ySzD@mLlWT z)ytz5t`n-C-b0!rW88d}J9RI48$F;X))PQcO>Zt{v$sFXElg-SLY2A}8BqNsE@1dt zAImMHQ`_^;UmTvN=E~>@fjafi-F~NDxMbO+?TvAzGX&jeh76Gr`1>Zit{A+MhAvLV zyX9LORxgN*jqKc2A`8;KomQ8(PTwx}Iw&XBZhTciQJ6u?*lo2Y?a`qMQ^Dq?rFKEi znXPOeepcmDWaa792U8x&Z#6Xb~YsqRIbC`c$ocih+&eH=SO|(?ZK%wYVk#5 zuVin#p0j@P5(m#vt?Fetu2C;$nIbe|dSx6>HkfN4NqDOslJvT0L*eLaYwOO=+Y*D|< z+Psb!tfnsL0*>H{cXk4jS3!gz6lB7SHQ@t}Xg55-M_@1o6bd2?gIEEL+HM#E(HjJ8 zkM8cZdJ_3)!Lq>aH$YTCys9`aCubrKP_WwsAj*-pIeFzoVGtM`3V|VEP$aNtZUljx z1^xgzb!V$Ti+s0J>q)>_1N;SsqJ{kP0SO_Ia3shY^sNm60azR919JM-hCl$E=BG9Y zgj68+={UKuN%5}ZyXRWz&{F*YL&3PiXwQo z92v5B4P!BKv7UQ7K6iKL?mhdx5yr~btMAx TV>eQTp%4@;H@Cc|0`30+Y(4+g literal 0 HcmV?d00001 diff --git a/tests/test_utils.py b/tests/test_utils.py index cbd65a4..7898bcc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -64,6 +64,12 @@ class Test(unittest.TestCase): a = [PSLiteral("test"), "test_2"] assert utils.decode_psl_list(a) == ["test", "test_2"] + def test_x_tolerance_ratio(self): + pdf = pdfplumber.open(os.path.join(HERE, "pdfs/issue-987-test.pdf")) + assert pdf.pages[0].extract_text() == 'Big Te xt\nSmall Text' + assert pdf.pages[0].extract_text(x_tolerance=4) == 'Big Te xt\nSmallText' + assert pdf.pages[0].extract_text(x_tolerance_ratio=0.15) == 'Big Text\nSmall Text' + def test_extract_words(self): path = os.path.join(HERE, "pdfs/issue-192-example.pdf") with pdfplumber.open(path) as pdf: