Ng\<ddlmZddlZddlZddlmZmZmZddlm Z ddl m Z ddl m Z mZmZmZmZmZddlZddlZddlmZmZddlmZmZmZmZmZmZmZm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&er dd l'm(Z(dd l)m*Z* dSdTdZ+dUd!Z, dVdWd/Z-dXd1Z.d2Z/ dYdZd<Z0d[d>Z1ed?Z2d\dCZ3d]dFZ4d^dHZ5d_dLZ6 d`dadRZ7dS)b) annotationsN)BufferedReaderBytesIO TextIOWrapper)SpooledTemporaryFile)sleep)IO TYPE_CHECKINGAnyOptionalTypeVarcast)CoordinateSystem PixelSpace) TYPE_TO_TEXT_ELEMENT_MAPCheckBoxCoordinatesMetadataElementElementMetadata ElementTypeListItem PageBreakText)logger)ENUMERATED_BULLETS_REUNICODE_BULLETS_RE) PageLayout) LayoutElementThtmllayout_element(LayoutElement | Element | dict[str, Any]coordinate_systemOptional[CoordinateSystem]infer_list_itemsbool source_format Optional[str]returnElement | list[Element]ct|tr|dkr|St|trtdSt|ts|}n|}|dd}|d}|d}|d}|dd } d } | r| j} |rKt|ttttj frtt| } nt} ||| | d } |tjkr|r t|fi| St!dd|i| S|t"vrit|tsJt"|} | dd|i| } |tjkr d | j_n|tjkr d | j_| S|tjtjtjtjtjtjfvr3|tjtjtjfv}t9dd|i| St;dd|i| S)zSConverts an unstructured_inference LayoutElement object to an unstructured Element.rtextr- coordinatestypeprobsourceN)detection_class_prob)r.r"metadatadetection_originchecked) isinstancerrdictto_dictgetvalueintstrfloatnumbersNumberrrLISTlayout_list_to_list_itemsrrHEADLINEr3category_depth SUB_HEADLINECHECK_BOX_CHECKEDCHECK_BOX_UNCHECKEDRADIO_BUTTON_CHECKEDRADIO_BUTTON_UNCHECKEDCHECKED UNCHECKEDrr)r r"r$r& layout_dictr-r. element_typer0 aux_originoriginclass_prob_metadata common_kwargs_element_classr7s `/var/www/html/ai-engine/env/lib/python3.11/site-packages/unstructured/partition/common/common.pynormalize_layout_elementrV!s.'**}/F/F.),,"b!!!! nd + +%$,,.. $ ??62 & &D//-00K??6**L ??6 " "D400J F"! 0 4#sE7>!BCC0-5;;OOO-//".'" M {'''  ,    1 1 1,,,,,,1,?'      ;/ / /56N # 2 2 [5 5 556N # 2 %'(*     )  ,  #               r-r.)Optional[tuple[tuple[float, float], ...]]r3Optional[ElementMetadata]r4 list[Element]cZ|rtj|ng}t|dkr|rtj|ng}g}|D]b}t|dkr;t |||||}||c|S)z=Converts a list LayoutElement to a list of ListItem elements.r5r)r-r.r"r3r4)rsplitlenrstriprappend) r-r.r"r3r4 split_items list_items text_segmentitems rUrDrDys8<C'-d333K ;18<D(.t444" "J# $ $ |!!## $ $q ( (!''))'"3!!1 D   d # # # rWelementrfilenamefiletype page_number Optional[int]url text_as_html image_path languagesOptional[list[str]]kwargsr c  V||t||nd} t|drt|jdkr|jnd} | r d| Dnd}| r d| Dnd}| r d| Dnd}t|drt|jdkr|jnd}|r d |Dnd}|r d |Dnd}|jjr |jjnd}t| ||||||||||||| }|j|| | |j_ |S) zAdds document metadata to the document element. Document metadata includes information like the filename, source url, and page number. N)pointssystemlinksrc8g|]}|dS)rir<.0links rU z(add_element_metadata..s"333T%333rWc8g|]}|dSr,rtrus rUrxz(add_element_metadata..s$555t$((6""555rWc8g|]}|dS) start_indexrtrus rUrxz(add_element_metadata..s$DDDd$((=11DDDrWemphasized_textsc8g|]}|dSr,rtrvemphasized_texts rUrxz(add_element_metadata..s&MMM  V $ $MMMrWc8g|]}|dS)tagrtr~s rUrxz(add_element_metadata..s&LLL  U # #LLLrW)r.rerfrgrirj link_urls link_textslink_start_indexesemphasized_text_contentsemphasized_text_tagsrFrkrl) rhasattrr]rrr|r3rFrupdater4)rdrerfrgrirjr.r"rkr4rlrncoordinates_metadatarrrrrr|rrdepthr3s rUadd_element_metadatars0  "'8'D $     %Wg66 [3w};M;MPQ;Q;QGMMW[E7<F33U3333$I9>H55u5555DJHMWDDeDDDDSW 7. / / 478P4Q4QTU4U4U     MM:: gt $ $ %# - -$,!! OOG $ $ $ $'G  OOG $ $ $ $ OrWctjD]Z} d|vrdS/#tjtjtjf$rYWwxYwdS)NsofficeTF)psutil process_iternamelower NoSuchProcess AccessDenied ZombieProcess)procs rU_is_soffice_runningrs#%% DIIKK--////tt0$f&96;OP    D  5s(A&A,+A,docx input_filenamer?output_directory target_format target_filterwait_for_soffice_ready_time_outr>c||d|}ddd|d||g} d}d}tj|d }|j} ||krv| d krp||z }t rt |nAtj|d }|j} ||kr| d kpn#t$rtd wxYwtj | |j dks| d krZtj d ||j tj |j dSdS)aConverts a .doc/.ppt file to a .docx/.pptx file using the libreoffice CLI. Parameters ---------- input_filename: str The name of the .doc file to convert to .docx output_directory: str The output directory for the convert .docx file target_format: str The desired output format target_filter: str The output filter name to use when converting. See references below for details. wait_for_soffice_ready_time_out: int The max wait time in seconds for soffice to become available to run References ---------- https://stackoverflow.com/questions/52277264/convert-doc-to-docx-using-soffice-not-working https://git.libreoffice.org/core/+/refs/heads/master/filter/source/config/fragments/filters N:rz --headlessz --convert-toz--outdirrg?T)capture_outputr+asoffice command was not found. Please install libreoffice on your system and try again. - Install instructions: https://www.libreoffice.org/get-help/install-howto/ - Mac: https://formulae.brew.sh/cask/libreoffice - Debian: https://wiki.debian.org/LibreOfficez3soffice failed to convert to format %s with code %i) subprocessrunstdoutdecoder^rrFileNotFoundErrorrinfo returncodeerrorstderr) rrrrrcommand wait_time sleep_timeoutputmessages rUconvert_office_docrs: (::=:: G   ===-&&((..00:::B  #I"$$ 9j!!!!#EEE -..006688 :::B     1     K AB A=RXRc     V]))++113344444 "/s CCC0Nonec Vtd|Ddkryt|}t |dkr+dd|ddd|dd}n |d d}t |dS) z Verify arguments; exactly one of all keyword arguments must not be None. Example: >>> exactly_one(filename=filename, file=file, text=text, url=url) c"g|] }|duo|dk S)Nr+r8)rvargs rUrxzexactly_one..Os% G G GS_ * G G GrWr5zExactly one of z, Nz and z must be specified.r)sumvaluesrkeysr]join ValueError)rnnamesrs rU exactly_onerHs G Gv}} G G GHHAMMV[[]]## u::>>b %*(=(=bbE"IbbbGGq666G!!! NMrW_Tfile _T | SpooledTemporaryFile[bytes] _T | BytesIOct|trI|dtt t |S|S)aConvert `file` to `BytesIO` when it is a `SpooledTemporaryFile`. Note that `file` does not need to be IO[bytes]. It can be `None` or `bytes` and this function will not complain. In Python <3.11, `SpooledTemporaryFile` does not implement `.readable()` or `.seekable()` which triggers an exception when the file is loaded by certain packages. In particular, the stdlib `zipfile.Zipfile` raises on opening a `SpooledTemporaryFile` as does `Pandas.read_csv()`. r)r9rseekrrbytesread)rs rUspooled_to_bytes_io_if_neededr[sM$,--1 ! tE499;;//000 KrWbytes | IO[bytes]rct|tr|St|tr@|d|}|d|St|t r|St|ttfrAt|j d5}|cdddS#1swxYwYtd)zExtract the bytes from `file` without preventing it from being read again later. As a convenience to simplify client code, also returns `file` unchanged if it is already bytes. rrbNzInvalid file-like object type) r9rrrrrgetvaluerropenrr)rf_bytesfs rUconvert_to_bytesrms $ $,-- ! ))++ ! $  }}$788 $)T " " a6688                 4 5 55sC((C,/C,scDttj|S)z Check if the input string contains any emoji characters. Parameters: - s (str): The input string to check. Returns: - bool: True if the string contains any emoji, False otherwise. )r%emoji emoji_count)rs rUcontains_emojirs !!$$ % %%rWpagerdict[str, Any]ct|dd}t|dd}|r|j}|j}|j}nH|r@|d}|d}|d}nd}d}d}|||dS)z:Retrieve image metadata and coordinate system from a page.imageNimage_metadataformatwidthheight)rrr)getattrrrrr<)rrr image_format image_width image_heights rUget_page_image_metadatars D'4 ( (ET#3T::N | k | %))(33 $((11 %))(33      rWocr_datalist['LayoutElement'] image_sizetuple[int | float, int | float]common_metadatac|\}}t||}g}|D]J} t| |||r|nd} |r| j||| K|S)zNConvert OCR layout data into `unstructured` elements with associated metadata.)rrr)r"r$r&)rrVr3rr_) rrrr$r&rrr"rr rds rUocr_data_to_elementsrs!+K"\JJJ H" ! !* /-+8D--f      5   # #O 4 4 4    OrW)NTr) r r!r"r#r$r%r&r'r(r)) r-r'r.rXr"r#r3rYr4r'r(rZ) NNNNNNNNNN)rdrrer'rfr'rgrhrir'rjr'r.rXr"r#rkr'r4r'rlrmrnr r(r)rrZr(rZ)rNr) rr?rr?rr?rr'rr>)rnr r(r)rrr(r)rrr(r)rr?r(r%)rrr(r)NTN) rrrrrrYr$r%r&r'r(rZ)8 __future__rrAriorrrtempfilertimertypingr r r r r rrr"unstructured.documents.coordinatesrrunstructured.documents.elementsrrrrrrrrrunstructured.loggerrunstructured.nlp.patternsrr'unstructured_inference.inference.layoutr.unstructured_inference.inference.layoutelementrrVrDrrrrrrrrrrrr8rWrUrs""""""5555555555))))))BBBBBBBBBBBBBBBB KKKKKKKK                      '&&&&&OOOOOOOOMBBBBBBLLLLLL 59!#) U U U U U p@#"!%"&=A48 $&*%)CCCCCL& #'+- K5K5K5K5K5\ " " " "  WT]]$66660 & & & &:26!#' rW