Ng]ddlZddlZddlmZddlmZmZmZmZddl m Z dZ ej dddgd Zej d gd d Zd Zej dddddddddfddddddddfddddddddddfddddddddddddd fdiddddddddddd dd!dd"dd#dd$dd%dd&dd'dd(dd)dd*dfgd+Zej dgd,d-Zej d.gd/d0Zej d1ejddddd2d3ddddd4d3gddd2d5ddd4d5gd67ejddddd2d3ddddd8d3ddddd9d3ddddd:d3ddddd;d3dddddd3g ddd2d5ddd8d5ddd9d5ddd:d5ddd;d5dddd5g d?7ejddddd@dAdddddBdAddCdddDdAdddddEdAdddddFdAdddddGdAdddddHdAdddddIdAdCddddJdAg ddd@d5dddBd5ddCdDd5dddEd5dddFd5dddGd5dddHd5dddId5dCddJd5g dK7gdLZej dMejdNddd2d5ddd4d5gd67ejdOddd2d5ddd8d5ddd9d5ddd:d5ddd;d5dddd5g d?7ejdPddd@d5dddBd5ddCdDd5dddEd5dddFd5dddGd5dddHd5dddId5dCddJd5g dK7gdQZdRZdSedTefdUZej dVgdWdXZdS)YN)text_extraction)deckerd_table_to_html!extract_cells_from_table_as_cellsextract_cells_from_text_as_htmlhtml_table_to_deckerd) partitioncLd}d}tjddd|}|dd}d}d}d}d }t t j||d d d ksJt t j||d d dksJt t j||d d dksJt t j||d d dksJt t j||d d dksJt t j||d d dksJt t j||d d dksJt t j||d d dksJdS)NzI like pizza. I like bagels.z!I like p i z z a . I like bagles.z\s+ z I like pizza.zI like pizza. I like .zI like pizza. I like beagles.z"I like pizza pizza. I like bagels.score) return_as?g?g(\?g{Gz?gg= ףp=?g{Gz?gHzG?)resubjoinreplaceroundrcalculate_edit_distance) source_cctsource_cct_word_spacesource_cct_spacessource_cct_no_spacesource_cct_one_sentencesource_cct_missing_wordsource_cct_addn_charsource_cct_dup_words j/var/www/html/ai-engine/env/lib/python3.11/site-packages/test_unstructured/metrics/test_text_extraction.pytest_calculate_edit_distancers/J?vsCHHZ,@,@AA$,,S"55-6:> o5j*X_```bcdd        3%!               3!!               3#!               3'!               3'!               3$!               3#!              )filenamestandardize_whitespacesexpected_scoreexpected_distance) fake-text.txtFg(\?&)r%Tgq= ףp? ctd5}|}dddn #1swxYwYtd|}dd|D}t j||d|}t j||d|} |d ksJ|d ksJ| d ksJt |d |ksJ| |ksJdS) Nzexample-docs/fake-text.txtz example-docs/)r! c,g|]}t|S)str).0els r z>test_calculate_edit_distance_with_filename..zs777CGG777r r r r"distancerrr)openreadrrrrr) r!r"r#r$frelements output_cctr r1s r*test_calculate_edit_distance_with_filenamer7lsM * + +qVVXX "<("<"<===H77h77788J  3J'Kb   E6 7 H A:::: C<<<< q==== ??n , , , , ( ( ( ( ( ( (s 155text1text2))z7The dog loved the cat, but the cat loved the cow0The dog loved the cat, but the cat loved the cow)z3Hello my name is H a r p e r, what's your name?/Hello my name is H a r p e r, what's your name?z+I have a dog and a cat, I love my dog.&I have a dog and a cat, I love my dog.)z Name Age City Occupation Alice 30 New York Engineer Bob 25 Los Angeles Designer Charlie 35 Chicago Teacher David 40 San Francisco Developer Name Age City Occupation Alice 30 New York Engineer Bob 25 Los Angeles Designer Charlie 35 Chicago Teacher David 40 San Francisco Developer )r?zName Age City Occupation Alice 30 New York Engineer Bob 25 Los Angeles Designer Charlie 35 Chicago Teacher David 40 San Francisco Developerctj||dddksJtj||dddksJtj||dddksJtj||dddksJdS)Nr Tr0rr1rFrrr8s r6test_calculate_edit_distance_with_various_whitespace_1rBsZ / 5GT         / 5J        / 5GU         / 5J          r cd}d}tj||ddtj||ddksJtj||ddtj||ddksJdS)Nr?a | Name | Age | City | Occupation | |---------|-----|--------------|----------------| | Alice | 30 | New York | Engineer | | Bob | 25 | Los Angeles | Designer | | Charlie | 35 | Chicago | Teacher | | David | 40 | San Francisco| Developer | r Tr0Fr1rA)source_cct_tabssource_cct_with_borderss r6test_calculate_edit_distance_with_various_whitespace_2rFsO   20G]a   /0G]b         20J`d   / %          r textexpectedr;r)thecatloveddogbutcowr<)hellomynameiszwhat'syourr>)ihavearOandrMloverSz4My dog's hair is red, but the dogs' houses are blue.) rSzdog'shairrUredrPrLzdogs'housesarebluezwSometimes sentences have a dash - like this one! A hyphen connects 2 words with no gap: easy-peasy. sometimes sentencesrXrYdashlikethisonehyphenconnects2wordswithnogapz easy-peasyc:tj||ksJdSN)r bag_of_wordsrGs rtest_bag_of_wordsrqs'p  ' - - 9 9 9 9 9 9r ))z9The dog loved the cat, but the cat loved the cow r;)z5 Hello my name is H a r p e r, what's your name?r<r=)zL is for the way you look at me O is for the only one I see V is very, very extraordinary E is even more than anyone that you adore canzL is for the way you look at me O is for the only one I see V is very, very extraordinary E is even more than anyone that you adore can)a | Name | Age | City | Occupation | |---------|-----|--------------|----------------| | Alice | 30 | New York | Engineer | | Bob | 25 | Los Angeles | Designer | | Charlie | 35 | Chicago | Teacher | | David | 40 | San Francisco| Developer | z| Name | Age | City | Occupation | |---------|-----|--------------|----------------| | Alice | 30 | New York | Engineer | | Bob | 25 | Los Angeles | Designer | | Charlie | 35 | Chicago | Teacher | | David | 40 | San Francisco| Developer |crtj|d|ksJtj||ksJdS)NT)r")r prepare_strrGs rtest_prepare_stringrt/sHH  &tT J J Jh V V V V  &t , , 4 4 4 4 4 4r  output_text source_textexpected_percentage))extrar r)r zSource text has a sentence.rK)z'The original s e n t e n c e is normal.z"The original sentence is normal...g?)z'We saw 23% improvement in this quarter.z-We saw 23% improvement in sales this quarter.g?)rlz4Is it possible to have more than everything missing?rKc<tj|||ksJdSro)rcalculate_percent_missing_textrus r#test_calculate_percent_missing_textr|Ws6@ 6{KPP       r )table_as_cellsexpected_extractionzMonth A.)xywhcontent22) row_index col_indexrz0Simple table, 1 head cell, 1 body cell, no spans)idzMonth B.zMonth C.1112132123z0Simple table, 3 head cell, 5 body cell, no spansh12col1)rrrrrh1col23h1col4h2col2h2col34r3col1r3col2r34col34r4col12z various spans, with 2 row headerc>dd|id}t||ksJdS)NTabler}typemetadata)r)r}r~example_elements r+test_cells_table_extraction_from_predictionr|s>N%~6O -_ = =AT T T T T T Tr ) text_as_htmlr~z
Month A.
22
" a
Month A. Month B. Month C.
11 12 13
21 22 23
" 
h12col1 h1col23 h1col4
h2col2 h2col34
r3col1 r3col2 r34col34
r4col12
c>dd|id}t||ksJdS)Nrrr)r)rr~rs r*test_html_table_extraction_from_predictionr)s@j L O +? ; ;?R R R R R R Rr cZddgdd}t|Jt|JdS)Nrr )rr}r)rr)rs r=test_cells_extraction_from_prediction_when_missing_predictionrsF&R[]4^4^__O *? ; ; C C C ,_ = = E E E E Er htmlreturncjd|dD}d|S)Nc:g|]}||Sr+)strip)r-lines rr/z_trim_html..s%DDD4tD$**,,DDDr r)r )splitr)r html_liness r _trim_htmlrs3DD4::d+;+;DDDJ 77:  r html_to_test)z
Month A.
22
a
Month A. Month B. Month C.
11 12 13
21 22 23
rclt|}t|}t||ksJdSro)rrr)r deckerd_table html_tables rtest_deckerd_html_converterrs>H*,77M&}55J l # #z 1 1 1 1 1 1r )rpytestunstructured.metricsr+unstructured.metrics.table.table_extractionrrrrunstructured.partition.autorrmark parametrizer7rBrFrqrtr|paramrrrr,rrr+r rrs  000000 211111ZZZzR*))))4'''**VW**V6B ?a!Aa J J >q!11 M M 5QQq1aWX Y Y C   " J Q Q  Q         q ! A Q   a q  a!  ;366n::o66n:""F55G""F5 9<=<- aaaJGGaaaDAA   aJGGaDAA B   aaaJGGaaaJGGaaaJGGaaaDAAaaaDAAaaaDAAaaaDAAaaaDAAaaaDAA  aJGGaJGGaJGGaDAAaDAAaDAAaDAAaDAAaDAA B/   D  ( ( ' ' ( ' ' ) ( s@ F"#!"( "#!"( "#!"' "#!"' "#!"( "#!"' "#!"' "#!") "#!"(S. ^2cr r r ]addJUUKddJU+   aJGGaDAA B'   *   0 aJGGaJGGaJGGaDAAaDAAaDAAaDAAaDAAaDAA BG$ $ $ \   6"#!"( "#!"( "#!"' "#!"' "#!"( "#!"' "#!"' "#!") "#!"(S. ^2SJ J J IORRfSSgRRfSFFF SS ???BBF22GBBF222r