Ng69ddlmZddlZddlZddlZddlZddlmZmZddl Z ddl m Z ddl mZmZmZmZmZmZmZd=dZd>d Zd=d Zd=d Zd?dZeefd@dZefdAdZefdBdZeddfdCdZd>dZe de!ej"DZ#dDd!Z$dEd$Z%d>d%Z&d>d&Z'd>d'Z(dFdGd*Z)dHdId1Z*dHdId2Z+ dJdKd8Z,dFdLd9Z-dMd;Z.dNd<Z/dS)O) annotationsN)OptionalTuple)format_encoding_str)DOUBLE_PARAGRAPH_PATTERN_REE_BULLET_PATTERN LINE_BREAK_REPARAGRAPH_PATTERNPARAGRAPH_PATTERN_REUNICODE_BULLETS_REUNICODE_BULLETS_RE_0WreturnstrcV|dd}|S)uCleans non-ascii characters from unicode string. Example ------- ˆThis text contains non-ascii characters!ˆ -> This text contains non-ascii characters! asciiignore)encodedecode)textens V/var/www/html/ai-engine/env/lib/python3.11/site-packages/unstructured/cleaners/core.pyclean_non_ascii_charsrs% Wh ' 'B 99;;rctj|}||Stjd|d}|S)uCleans unicode bullets from a section of text. Example ------- ● This is an excellent point! -> This is an excellent point! N)r matchsubstrip)rsearch cleaned_texts r clean_bulletsr"%sD %d + +F ~ %)"dA66L     rc2|}d|dd}td|dvd|dvgr|Stjd|d}|d s|d =t |dd kr|S|S) u/Cleans the start of bulleted text sections up to three “sub-section” bullets accounting numeric and alphanumeric types. Example ------- 1.1 This is a very important point -> This is a very important point a.b This is a very important point -> This is a very important point  rN.rz..z[\.])patternstring)splitjoinanyrelen)rtext_sptext_clbullets rclean_ordered_bulletsr24sjjllGhhwqrr{##G Cwqz !471:#5 677 Xggaj 9 9 9F ": 2J 6!9~~ Nrcddddddddd d d d d dd}|}|D]\}}|||}|S)uReplaces ligatures with their most likely equivalent characters. Example ------- The benefits -> The benefits High quality financial -> High quality financial aeAEfffiflffifflftlsoeOEqpstts)æÆuffufiufluffiuffluſtuʪuœuŒuȹustuʦ)itemsreplace)r ligatures_mapr!kvs rclean_ligaturesrILs}M L##%%221#++Aq11 r paragraphlistcg}tjtd|}tjt |}|D]2}|r.|tjtd|3|S)ukGroups paragraphs with bullets that have line breaks for visual/formatting purposes. For example: '''○ The big red fox is walking down the lane. ○ At the end of the lane the fox met a friendly bear.''' Gets converted to '''○ The big red fox is walking down the lane. ○ At the end of the land the fox met a bear.''' ·r$)r-rrrr*r appendr )rJclean_paragraphs bullet_parasr1s rgroup_bullet_paragraphrQks($ ::AACCI819==LLL  L  # #BF+z*group_broken_paragraphs..sEVV4c$**,,"4"4S"9"9::Q>VVVVVVrc:g|]}||S)rrXs r z+group_broken_paragraphs..s%$Q$Q$QdDJJLL$QT$Q$Q$Qrr$ ) r*rallr rrextendrQrNr-rr r+)rrRrT paragraphsrOrJ para_splitall_lines_shorts rgroup_broken_paragraphsres;&!&&t,,JOO       %%i00 VV:VVVVV  #IOO$5$5 6 6 O:J:PQZQ`Q`QbQb:c:c O  # #$:9$E$E F F F F  O  # #$Q$Qj$Q$Q$Q R R R R  # #BF+ “What a lovely quote!” ‘u‘’u’“u“”u”z''u’u–u—u—u–u‘u…u…u’u“uâ€?u※u‶u
u″u″u‮ru u u​uâ€uâ€s')rErs rreplace_unicode_quotesr}s << & &D << & &D << & &D << & &D <<# & &D << c * *D << U + +D << U + +D << E * *D << E * *D << U + +D << E * *D << 5 ) )D << E * *D << E * *D << E * *D << E * *D << B ' 'D << R ( (D << R ( (D << R ( (D <<" % %D << B ' 'D Krc#K|]:}tjt|d6|V;dS)PN) unicodedatacategorychr startswith)rYis rr[r[(s[ ( ITEM 1. BUSINESS [\xa0\n]r$ ([ ]{2,})r-rr)rr!s rclean_extra_whitespacer<s<6+sD11L6,\::L     rcRtjdd|S)zhCleans dash characters in text. Example ------- ITEM 1. -BUSINESS -> ITEM 1. BUSINESS z [-\u2013]r$rr|s r clean_dashesrHs$ 6,T * * 0 0 2 22rcP|dS)zvClean all trailing punctuation in text Example ------- ITEM 1. BUSINESS. -> ITEM 1. BUSINESS z.,:;)rrstripr|s rclean_trailing_punctuationrSs  ::<<  v & &&rutf-8encodingct|}tj|||S)uReplaces MIME encodings with their equivalent characters in the specified encoding. Example ------- 5 w=E2=80-99s -> 5 w’s )rquopri decodestringrr)rrformatted_encodings rreplace_mime_encodingsr]s?-X66  t{{+=>> ? ? F FGY Z ZZrFTr& ignore_caseboolrc|r tjnd}tjd|d||}|r|n|}|S)aRemoves prefixes from a string according to the specified pattern. Strips leading whitespace if the strip parameter is set to True. Input ----- text: The text to clean pattern: The pattern for the prefix. Can be a simple string or a regex pattern ignore_case: If True, ignores case in the pattern strip: If True, removes leading whitespace from the cleaned string. r^rflags)r- IGNORECASErlstriprr&rrr clean_texts r clean_prefixrhsS) /BMMaEWD>>>J(-=""$$$:J rc|r tjnd}tj|dd||}|r|n|}|S)aRemoves postfixes from a string according to the specified pattern. Strips trailing whitespace if the strip parameters is set to True. Input ----- text: The text to clean pattern: The pattern for the postfix. Can be a simple string or a regex pattern ignore_case: If True, ignores case in the pattern strip: If True, removes trailing whitespace from the cleaned string. r$rr)r-rrrrs r clean_postfixrysS) /BMMaEGD>>>J(-=""$$$:J rextra_whitespacedashesbulletstrailing_punctuation lowercasec|r|n|}|rt|n|}|rt|n|}|rt|n|}|rt |n|}|S)alCleans text. Input ----- extra_whitespace: Whether to clean extra whitespace characters in text. dashes: Whether to clean dash characters in text. bullets: Whether to clean unicode bullets from a section of text. trailing_punctuation: Whether to clean all trailing punctuation in text. lowercase: Whether to return lowercase text. )lowerrrrr"r)rrrrrrr!s rcleanrs&$-64::<<<$L4HZ"<000l28I< ---\L;K]),777Q]L29K=...|L     rc|td|D}t|}||S)zgConverts a string representation of a byte string to a regular string using the specified encoding.c,g|]}t|Sr])r)rYchars rr^z*bytes_string_to_string..s333dD 333r)bytesrr)rr text_bytesrs rbytes_string_to_stringrsC33d33344J,X66   / 0 00rTuple[str, np.ndarray]c4tjdd|}tjdd|}|}tjt |}d\}}}|t |kr||||ksPt tjd||r8t tjd||r|||<|dz }|dz }|dz }|||<|dz }|t |k|||d<||fS)aCleans extra whitespace characters that appear between words. Calculate distance between characters of original text and cleaned text. Returns cleaned text along with array of indices it has moved from original. Example ------- ITEM 1. BUSINESS -> ITEM 1. BUSINESS array([0., 0., 0., 0., 0., 0., 0., 0., 4., 4., 4., 4., 4., 4., 4., 4., 4., 4., 4., 4.])) rr$r)rrru[  ]rN)r-rrnpzerosr.rr)rr! moved_indicesdistanceoriginal_index cleaned_indexs r%clean_extra_whitespace_with_index_runrs66+sD11L6,\::L%%''LHSYY''M.5+Hnm #l++ + +  < #> > > *d>&:;; < < ?RXc< #>??@@ ?,4M- ( a N Q M A '/ m$! #l++ + +%-M-..!  &&rc2t|||z Sr)rl)indexrs r-index_adjustment_after_clean_extra_whitespacers u}U++ , ,,r)rr)rrrr)rJrrrK)rrrRrSrTrSrr)rrrTrSrr)rrrTrhrr) rrrRrSrkrlrmrnrr)rrrr)rrrrrr)r)rrrrrr)FT) rrr&rrrrrrr)FFFFF)rrrrrrrrrrrrrr)rrrr)rrrr)rrl)0 __future__rrr-sysrtypingrrnumpyr unstructured.file_utils.encodingrunstructured.nlp.patternsrrr r r r r rr"r2rIrQrergrjrur}dictfromkeysrange maxunicoderrrrrrrrrrrrrr]rrrs="""""" """"""""         0><#7'B')')')')')X(5)))))8#>))))).#0 (((((J    F mmuS^$$              3333''''[[[[["&#!&      :11111$'$'$'$'N------r