Ng/m UddlZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z mZmZmZmZddlmZmZmZmZmZmZddlmZddlmZdd lmZeje Z!d Z"d Z#d Z$d Z%d Z&ee'd<dZ(ee'd<ddddddddZ)dede egeffdZ*dede egeeefffdZ+eded ededdZ,ededed ed!dZ-d>deded#e.ddfd$Z/Gd%d&Z0Gd'd(Z1Gd)d*Z2 d?d,ed-e3d.edefd/Z4ej5e1j6j78Z9ej5e2j78Z:d,ed.ede0fd0Z; d@d,ed1eed2eed.ede.f d3Z dAd,ed5ed;eee.dz&get_line_cluster_key../s 5c|d S)Nbottomr*r+s r-r.z&get_line_cluster_key..0s !H+r/c|dSNx0r*r+s r-r.z&get_line_cluster_key..1s 4r/c|d S)Nx1r*r+s r-r.z&get_line_cluster_key..2s !D'r/rbttrrtlr*)r%s r-get_line_cluster_keyr:-s3!!%%  !!      r/char_dirc$ddddd|S)Nc"|d|dfS)Nr)r1r*r+s r-r.z#get_char_sort_key..8s!E(AhK0r/c8|d|dz |d fS)Nr)heightr*r+s r-r.z#get_char_sort_key..9s!AeHq{23ahY?r/c"|d|dfSr3r*r+s r-r.z#get_char_sort_key..:s!D'1T7+r/c&|d |d fS)Nr6r4r*r+s r-r.z#get_char_sort_key..;s1T7(QtWH-r/r7r*)r;s r-get_char_sort_keyrB6s300??++--      r/r7r)r1r4r6suffixc 8tt}||vrtd|d|d|||vrtd|d|d|t|t|krtd|d|d|d|dS)Nr%z must be one of z, not r;=z is incompatible with char_dir)set POSITION_KEYSkeys ValueError)r%r;rD valid_dirss r-validate_directionsrLNs]''))**Jz!! Kv K Kz K K K K   z!! Kv K Kz K K K K    8}}H %% 0v 0 0 0 0" 0 0%- 0 0   &%r/c:eZdZdZdeeeeefde de ddfdZ defdZ dd e ed e d ededeeeff dZ ddeeeefdeded eded e deeeeffdZ ddededeeeeffdZdS)TextMapz A TextMap maps each unicode character in the text to an individual `char` object (or, in the case of layout-implied whitespace, `None`). tuplesline_dir_renderchar_dir_renderr&Nct||d||_||_||_||_dS)N_render)rLrOrPrQ to_string as_string)selfrOrPrQs r-__init__zTextMap.__init__esB O_iHHH ..))r/cN|j}|j}dtt d|j}|dkr|dkr|S|d|dvrtt|dkr dD|d vrsttt|d krfd Dnfd Ddfd tDSdS)NrCrrr r8r9r9cRg|]$}dt|%SrC)joinreversed.0lines r- z%TextMap.to_string..s*CCCT$00CCCr/)r9rr8c@g|]}dt|z z|zS lenr`ramax_line_lengths r-rbz%TextMap.to_string..s;IM#d)) ;<Dr/c@g|]}|dt|z zzSrdrfrhs r-rbz%TextMap.to_string..s;IMT'B CDr/c3\K|]%dfdDV&dS)rCc3(K|] }|V dSNr*)r`rais r- z.TextMap.to_string...s'66DG666666r/Nr])r`rnliness @r-roz$TextMap.to_string..sV!!;.s<<.s'D)<)<)B)B)D)D$E$Er/cBg|]}|S))r|r}r{)r)r`rzr{r}r|rVs r-rbz"TextMap.search..sL      +)%      r/) isinstancer rJreescapeIcompilefinditerrUfilter) rVrrrr|r}r{compiledflagsgenfiltereds ` ``` r-searchzTextMap.searchs gw ' ' 2~~ Wu}} VHH~~)G,, EMMBDDqEz'511Hk(DN33EEEEsKK           r/rcB|rd}nd}||d|dS)af `strip` is analogous to Python's `str.strip()` method, and returns `text` attributes without their surrounding whitespace. Only relevant when the relevant TextMap is created with `layout` = True Setting `return_chars` to False will exclude the individual character objects from the returned text-line dicts. z *([^\n]+?) *(\n|$)z([^\n]+)rF)r{r}r|)r)rVrr}pats r-extract_text_lineszTextMap.extract_text_liness<  (CCC{{ AL   r/)rTT)TTTTr)TT)__name__ __module__ __qualname____doc__rr strr rrrWrTr intrrrrr r rrr*r/r-rNrN_s *U3/01 * * *  * * * *(3((((F"!  :    c3h <"!& & sGCL()& &  &  &  & &  d38n & & & & R8<   04 d38n       r/rNc+eZdZdZdeeeefddfdZdddddde e dde e e ddddddd fd ed ed ed ededededededededededeedeedeedeededededef*dZdS)WordMapz& A WordMap maps words->chars. rOr&Nc||_dSrm)rO)rVrOs r-rWzWordMap.__init__s  r/Fr)rrrrTlayout layout_width layout_heightlayout_width_charslayout_height_chars layout_bbox x_density y_densityx_shifty_shift y_tolerancer%r;line_dir_rotatedchar_dir_rotatedrQrP use_text_flow presortedexpand_ligaturesc23g}t|jst||p| |p| S|rtni}|rm|r|rt dnt t ||z }|r|rt dnt t ||z }dg|z}ng}d}t| 3t| 2t| }t| }t| |}t| |}|s|r|jnt|j3fd}t|3fd| |p| }t|D]\}} |r0|| dd}!|!|| zz }"| d vrd nd }#|"|#z|z }$nd}$tt |dkt |$|z }%t|%D]=}t|r|d dd kr||z }|d>||%z }d}&|s|r| nt| 2fd}'|'D]\}(})|r$||(}*|*|| zz }+| d vrd nd }#|+|#z|z },nd},tt#d |&t |,|&z }-|dg|-zz }|&|-z }&|)D]E}.||.d|.d}/|/D]}0||0|.f|&d z }&F|r |dg||&z zz }|rP||d zz }1t|1D]"}|dkr||z }|d#|d dkr |dd }t||p| |p| S)a Given a list of (word, chars) tuples (i.e., a WordMap), return a list of (char-text, char) tuples (i.e., a TextMap) that can be used to mimic the structural layout of the text on the page(s), using the following approach for top-to-bottom, left-to-right text: - Sort the words by (top, x0) if not already sorted. - Cluster the words by top (taking `y_tolerance` into account), and iterate through them. - For each cluster, divide (top - y_shift) by `y_density` to calculate the minimum number of newlines that should come before this cluster. Append that number of newlines *minus* the number of newlines already appended, with a minimum of one. - Then for each cluster, iterate through each word in it. Divide each word's x0, minus `x_shift`, by `x_density` to calculate the minimum number of characters that should come before this cluster. Append that number of spaces *minus* the number of characters and spaces already appended, with a minimum of one. Then append the word's text. - At the termination of each line, add more spaces if necessary to mimic `layout_width`. - Finally, add newlines to the end if necessary to mimic to `layout_height`. For other line/character directions (e.g., bottom-to-top, right-to-left), these steps are adjusted. rPrQz;`layout_width` and `layout_width_chars` cannot both be set.z=`layout_height` and `layout_height_chars` cannot both be set.)reNrc&|dSNrr*r,line_cluster_keys r-r.z$WordMap.to_textmap..Us3C3CAaD3I3Ir/keyc&|dSrr*rs r-r.z$WordMap.to_textmap..Zs&&qt,,r/)preserve_orderrZrrY)rYNc&|dSrr*)r, char_sort_keys r-r.z$WordMap.to_textmap..zs}}QqT7J7Jr/rN)rgrOrN LIGATURESrJrroundr:rBrHBBOX_ORIGIN_KEYSsortedr enumeraterurvappendminget)4rVrrrrrrrrrrrr%r;rrrQrPrrr_textmap expansions blank_line num_newlinesline_position_keychar_position_keyy_originx_originwords_sorted_line_dirtuples_by_linern line_tuples line_position y_dist_rawadjy_distnum_newlines_prependline_lenline_tuples_sortedwordr char_position x_dist_rawx_distnum_spaces_prependrlettersletternum_newlines_appendrrs4 @@r- to_textmapzWordMap.to_textmapsl794;  / ;8 / ;8  #3:YY  ! J$U &)|i/G)H)H%I%I"" L $W '*% 0I*J*J&K&K#%);;JJJ /99)(33 )(3)(3#H-k::#H-k:: K) KDKK )I)I)I)IJJJ  ) ! , , , , $5    (773 L3 LNA{  1 1+a.2C D D *h.@A $66bbA#c)I5#&AE f , $$ /00 . .8}}+ Q4(?(? *H ---- 0 0LHL -L K-J-J-J-JKKK  2 & & e$5$5d$;$;M!.(W2D!EJ (N : :""C'#- 9FFF%(Q)9)95==8;S%T%T"[M,>>>..&&A(nnQvY& BBG")&&  444 A && L[M-?(-JKK  )"59I"J .// . .q55 *H ----||++#CRC= +7x+7x    r/)rrrrrr rrrWDEFAULT_X_DENSITYDEFAULT_Y_DENSITYDEFAULT_Y_TOLERANCErrrrrrrr rNrr*r/r-rrstE%*;$<=$  "##$*,,0**,0,0+/+/#!%+s s s s  s  s ! s s s s s s s s s #5/s #5/!s ""%#s $"%%s &'s ()s *+s , -s s s s s s r/rceZdZeeddddddeedddddfdededee e dfdee e dfde d e d e d e d e d e de e de e de eedee efde fdZde de fdZdedefdZdedede dedede f dZdede deeddffdZdedeeee fddffdZdedeeeefddffd Zdedefd!Z d$ded"e defd#ZdS)% WordExtractorNFT x_tolerancerx_tolerance_ratioy_tolerance_ratiokeep_blank_charsr vertical_ttbhorizontal_ltrr%r;rr extra_attrssplit_at_punctuationrc||_||_||_||_||_||_||_||_|durt d|durt d| |_ | |_ | p| |_ | p| |_ t|j |j t|j |j d| gn| |_|dur t jn|pd|_|rt&ni|_dS)NFzNvertical_ttb is deprecated and will be removed; use line_dir/char_dir instead.zPhorizontal_ltr is deprecated and will be removed; use line_dir/char_dir instead._rotatedTrC)rrrrrrrrloggerwarningr%r;rrrLrstring punctuationrrr)rVrrrrrrrrr%r;rrrrrs r-rWzWordExtractor.__init__s7$'&!2!2 0*,( 5 NN2    U " " NN2   !   0 .sN>?##AfIqy99r/) rr4r6r)rr1rr?width direction)rrr]r) rVrr4r)r6r1 doctop_adjrr;rrs ` r- merge_charszWordExtractor.merge_charss-m<<CV"1%h/-2B52II "9-$$W--GGCPJ&sl"W!   # . .C%a(-DII r/ prev_char curr_charrc|dvrO|}|}|d}|d} |dkr|d} |d} |d} nj|d } |d } |d } nN|}|}|d}|d} |dkr|d} |d} |d} n|d } |d } |d } t| | kp| | |zkp| ||zkS)aThis method takes several factors into account to determine if `curr_char` represents the beginning of a new word: - Whether the text is "upright" (i.e., non-rotated) - Whether the user has specified that horizontal text runs left-to-right (default) or right-to-left, as represented by self.horizontal_ltr - Whether the user has specified that vertical text the text runs top-to-bottom (default) or bottom-to-top, as represented by self.vertical_ttb - The x0, top, x1, and bottom attributes of prev_char and curr_char - The self.x_tolerance and self.y_tolerance settings. Note: In this case, x/y refer to those directions for non-rotated text. For vertical text, they are flipped. A more accurate terminology might be "*intra*line character distance tolerance" and "*inter*line character distance tolerance" An important note: The *intra*line distance is measured from the *end* of the previous character to the *beginning* of the current character, while the *inter*line distance is measured from the *top* of the previous character to the *top* of the next character. The reasons for this are partly repository-historical, and partly logical, as successive text lines' bounding boxes often overlap slightly (and we don't want that overlap to be interpreted as the two lines being the same line). The upright-ness of the character determines the attributes to compare, while horizontal_ltr/vertical_ttb determine the direction of the comparison. )rr9r)rr4r6rr1)r) rVr r rrrr,yaycyaxbxcxs r-char_begins_new_wordz"WordExtractor.char_begins_new_words'R  & &AA5!B5!BE!!t_t_t_o%o%o%AA4B4BE!!u%x(u%))&&)) "W R!V  R!V    r/c #\ Kg dttdttddff fd }|j}|j}|j}|j}|D]}|d} |js&| r|dEd{V7| |j vr#||Ed{V|dEd{Vc rX| d||||n| ddz||n| ddzr||Ed{V |Ӊ r VdSdS)Nnew_charr&c3&KrV|gn|gdSrmr*)r current_words r-start_next_wordz:WordExtractor.iter_chars_to_words..start_next_wordXs1  #""""!)!122zLLLr/rrsize)rr) r rrrrrrrrisspacerrr) rVrrrxtxtrytytrcharrrs @r-iter_chars_to_wordsz!WordExtractor.iter_chars_to_wordsQs $&  Buo B z4- . B B B B B B $  $! * *D??? !)%>> #0 E E GZ%:*dm,----j99  *5 E E% I"&":"::y"Q"QEEJ++J77DDDDDE E  E Er/c`tt||Srm)rrtr0)rVrs r-extract_wordmapzWordExtractor.extract_wordmaps&tD44U;;<<===r/r}c|r,td||DStd||DS)Nc3,K|]\}}i|d|iVdS)rNr*r`rr/s r-roz.WordExtractor.extract_words..sI$D*.4-*--r/c3 K|] \}}|V dSrmr*r5s r-roz.WordExtractor.extract_words..s'UU!1zUUUUUUr/)rtr0)rVrr}s r- extract_wordszWordExtractor.extract_wordssv  V(,(@(@(G(G  UUT5M5Me5T5TUUUUU Ur/F)rrrDEFAULT_X_TOLERANCErrrrr rfloatrrr rrrWrrrrrrrrr r%r0rr2r7r*r/r-rrs105959!&#!#**,0,0+/16!%!4@4@4@4@!eT!12 4@ !eT!12 4@  4@4@4@4@4@4@#5/4@#5/4@d3i(4@$D#I.4@ !4@4@4@4@l CC CE C C C C4K K K  K  K  K  K K K K Z.!.. :tT) * ....`++ 5U*+T47 8++++.EE 5 *+T47 8EEEE >Z>G>>>>7< V V V/3 V  V V V V V Vr/rFrr}kwargsc Btdi|||S)Nr*)rr7)rr}r;s r-r7r7s(  " "6 " " 0 0 E EEr/c ddpt|dtdifdtD}||}|jdifdtD}|S)NTr)rrc*i|]}|v||Sr*r*r`kr;s r- z$chars_to_textmap..s$ F F FA!v++1fQi+++r/c*i|]}|v||Sr*r*r?s r-rAz$chars_to_textmap..s$ ? ? ?A1;;1fQi;;;r/r*)updaterrrWORD_EXTRACTOR_KWARGSr2rTEXTMAP_KWARGS)rr; extractorwordmaptextmaps ` r-chars_to_textmaprIs MM!::m44N8N8N   F F F F!6 F F FI''..G g  ? ? ? ? ? ? ?G Nr/rPrQc rt|}t|dkrdSdri||d}t|fi|jSt d ifdt D}||}|p|j}|p|j }t|j}dt}dt} t|||dvr| n|} td d d | DD||jS) NrrCrrc*i|]}|v||Sr*r*r?s r-rAz extract_text..s$JJJa6kkq&)kkkr/rrr!cg|]}|dfSrmr*)r`rs r-rbz extract_text..s,   t    r/rYc3TK|]#}dd|DV$dS)rec3&K|] }|dV dSrr*)r`rs r-roz)extract_text...s&&E&EtF|&E&E&E&E&E&Er/Nrpr_s r-rozextract_text..s=XX$chh&E&E&E&E&EEEXXXXXXr/r*)rrgrrIrUrrDr7r%r;r:r9rrrNr]) rrPrQr;textmap_kwargsrFwordsrrrrqs ` r- extract_textrQs ENNE 5zzQr zz(#  "1oVV  8888BB!  JJJJ%:JJJ  ''..)?Y-?)?Y-?/ 0BCCjj0CDD jj0CDD   *n<.s-EEa\![11EEEEEEr/)rrr])rrr clustereds ` r-extract_text_simplerZsE  z(';';[III 99EEEE9EEE E EEr/fontnamerr.ctgd|p tRtdddtdttddfffd }||}t ||jS) u Removes duplicate chars — those sharing the same text and positioning (within `tolerance`) as other characters in the set. Use extra_args to be more restrictive with the properties shared by the matching chars. )rrrr4rr&Nc36Kt|}tj|D]m\}}tt |t dD]<}t|t dD]}t|dV=ndS)Nrrr4r)rr'r(rrtr) r sorted_charsgrp grp_chars y_cluster x_clusterrpos_keyrRs r-yield_unique_charsz(dedupe_chars..yield_unique_chars'se--- '/ #FFF < rvs<                         JIIIIIIIIIIIIIII''''''%%%%%%  8 $ $%%          5Xugun-E(E7E%,