
    Ng&                       d Z ddlmZ ddlZddlmZmZ ddlZddlm	Z	 ddl
mZmZmZmZmZmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZ dd	lmZ dd
lm Z  d4dZ!d5dZ"d5dZ#d5dZ$d Z%d Z&d6dZ'd Z(d Z)d Z*d Z+d Z,d7d!Z-d7d"Z.d# Z/d$ Z0ej1        2                    d%i d&fd'did&fd'd(id(fg          d8d-            Z3d. Z4d/ Z5d0 Z6d1 Z7 ej8                    d9d3            Z9dS ):z3Test suite for `unstructured.partition.doc` module.    )annotationsN)AnyIterator)MockFixture)ANYCaptureFixtureFixtureRequestassert_round_trips_through_JSONexample_doc_pathmethod_mock)chunk_elements)	AddressCompositeElementElementListItemNarrativeTextTable
TableChunkTextTitle)partition_doc)partition_docxrequestr	   c                    t          d          }t          d          }t          |          t          |          k    sJ d S )N
simple.doczsimple.docx)r   r   r   )r   doc_file_pathdocx_file_paths      `/var/www/html/ai-engine/env/lib/python3.11/site-packages/test_unstructured/partition/test_doc.py)test_partition_doc_matches_partition_docxr   %   sC    $\22M%m44N''>.+I+IIIIIII    expected_elementslist[Element]capsysCaptureFixture[str]c                    t          t          d                    }|| k    sJ t          d |D                       sJ |                                j        dk    sJ |                                j        dk    sJ d S )Nr   c              3  P   K   | ]!}|j         j        t          d           k    V  "dS ) N)metadatafile_directoryr   .0es     r   	<genexpr>z3test_partition_doc_from_filename.<locals>.<genexpr>3   s5      SSQqz(,<R,@,@@SSSSSSr    r'   )r   r   all
readouterrouterr)r!   r#   elementss      r    test_partition_doc_from_filenamer3   /   s    -l;;<<H(((((SS(SSSSSSSS"b(((("b((((((r    c                   t          t          d          d          5 }t          |d          }d d d            n# 1 swxY w Y   || k    sJ |                                j        dk    sJ |                                j        dk    sJ d S )Nr   rbzMS Word 2007 XMLfilelibre_office_filterr'   )openr   r   r/   r0   r1   r!   r#   fr2   s       r   5test_partition_doc_from_file_with_libre_office_filterr<   8   s     
|,,d	3	3 Qq a=OPPPQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q ((((("b(((("b((((((   <A A c                T   t          t          d          d          5 }t          |d           }d d d            n# 1 swxY w Y   || k    sJ |                                j        dk    sJ |                                j        dk    sJ t          d |D                       sJ d S )Nr   r5   r6   r'   c              3  2   K   | ]}|j         j        d u V  d S Nr(   filenamer*   s     r   r-   zKtest_partition_doc_from_file_with_no_libre_office_filter.<locals>.<genexpr>L   ,      ==qqz"d*======r    )r9   r   r   r/   r0   r1   r.   r:   s       r   8test_partition_doc_from_file_with_no_libre_office_filterrD   C   s    
|,,d	3	3 Cq aTBBBC C C C C C C C C C C C C C C ((((("b(((("b((((==H==========r=   c                     t          d          } t          | d          5 }t          j        t          d          5  t          | |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r5   2Exactly one of filename and file must be specifiedmatch)rB   r7   )r   r9   pytestraises
ValueErrorr   )r   r;   s     r   Etest_partition_doc_raises_when_both_a_filename_and_file_are_specifiedrL   O   s    $\22M	mT	"	" :a]:-abbb 	: 	:=q9999	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:: : : : : : : : : : : : : : : : : :s4   A2AA2A	A2!A	"A22A69A6c                     t          j        t          d          5  t                       d d d            d S # 1 swxY w Y   d S )NrF   rG   )rI   rJ   rK   r    r    r   Vtest_partition_doc_raises_when_neither_a_file_path_nor_a_file_like_object_are_providedrO   W   s    	z)]	^	^	^                   s   8<<tmp_pathpathlib.Pathc                    t          | dz            }t          j        t          d          5  t	          |           d d d            d S # 1 swxY w Y   d S )Nzasdf.doczasdf.doc does not existrG   )rB   )strrI   rJ   rK   r   )rP   doc_filenames     r   &test_partition_raises_with_missing_docrU   \   s    x*,--L	z)B	C	C	C - -|,,,,- - - - - - - - - - - - - - - - - -s   AAAc                     t          t          d                    } t          |           dk    sJ t          d | D                       sJ d S )Nr   r   c              3  6   K   | ]}|j         j        d k    V  dS )r   NrA   r*   s     r   r-   zStest_partition_doc_from_filename_gets_filename_from_filename_arg.<locals>.<genexpr>j   s,      EEqqz"l2EEEEEEr    r   r   lenr.   r2   s    r   @test_partition_doc_from_filename_gets_filename_from_filename_argr[   f   sX    -l;;<<Hx==1EEHEEEEEEEEEEr    c                     t          t          d          d          5 } t          |           }d d d            n# 1 swxY w Y   t          |          dk    sJ t	          d |D                       sJ d S )Nr   r5   )r7   r   c              3  2   K   | ]}|j         j        d u V  d S r@   rA   r*   s     r   r-   zBtest_partition_doc_from_file_gets_filename_None.<locals>.<genexpr>r   rC   r    )r9   r   r   rY   r.   r;   r2   s     r   /test_partition_doc_from_file_gets_filename_Noner_   m   s    	|,,d	3	3 )q a((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) x==1==H==========s   ;??c                     t          t          d          d          } t          |           dk    sJ t          d | D                       sJ d S )Nr   test)metadata_filenamer   c              3  6   K   | ]}|j         j        d k    V  dS ra   NrA   r+   elements     r   r-   zMtest_partition_doc_from_filename_prefers_metadata_filename.<locals>.<genexpr>y   s-      KKww(F2KKKKKKr    rX   rZ   s    r   :test_partition_doc_from_filename_prefers_metadata_filenamerg   u   s]    -l;;vVVVHx==1KK(KKKKKKKKKKr    c                     t          t          d          d          5 } t          | d          }d d d            n# 1 swxY w Y   t          d |D                       sJ d S )Nr   r5   ra   )r7   rb   c              3  6   K   | ]}|j         j        d k    V  dS rd   rA   r*   s     r   r-   zItest_partition_doc_from_file_prefers_metadata_filename.<locals>.<genexpr>   s,      ??qz"f,??????r    )r9   r   r   r.   r^   s     r   6test_partition_doc_from_file_prefers_metadata_filenamerj   |   s    	|,,d	3	3 Cq a6BBBC C C C C C C C C C C C C C C ??h??????????r=   c                     dt          t          d                    } t          fd| D                       s-J d dt          | d         j        j                               d S )Nzapplication/mswordr   c              3  8   K   | ]}|j         j        k    V  d S r@   )r(   filetype)r+   r,   DOC_MIME_TYPEs     r   r-   zQtest_partition_doc_gets_the_DOC_MIME_type_in_metadata_filetype.<locals>.<genexpr>   s-      FFqz"m3FFFFFFr    zExpected all elements to have 'z' as their filetype, but got: r   )r   r   r.   reprr(   rm   )r2   rn   s    @r   >test_partition_doc_gets_the_DOC_MIME_type_in_metadata_filetyperp      s    (M-l;;<<HFFFFXFFFFF  	2- 	2 	2!%.//	2 	2    r    mockerr   c                    d|                      d           t          t          d                    }t          fd|D                       sJ d S )N2029-07-05T09:24:281unstructured.partition.doc.get_last_modified_datereturn_valuezfake.docc              3  8   K   | ]}|j         j        k    V  d S r@   r(   last_modified)r+   r,   filesystem_last_modifieds     r   r-   zItest_partition_doc_pulls_last_modified_from_filesystem.<locals>.<genexpr>   s.      VVqz'+CCVVVVVVr    patchr   r   r.   )rq   r2   rz   s     @r   6test_partition_doc_pulls_last_modified_from_filesystemr}      sq    4
LL;Jb     -j99::HVVVVXVVVVVVVVVVr    c                    d}d|                      d|           t          t          d                    }t          fd|D                       sJ d S )Nrs   z2020-07-05T09:24:28rt   ru   r   )metadata_last_modifiedc              3  8   K   | ]}|j         j        k    V  d S r@   rx   )r+   r,   r   s     r   r-   zRtest_partition_doc_prefers_metadata_last_modified_when_provided.<locals>.<genexpr>   s.      TTaqz'+AATTTTTTr    r{   )rq   rz   r2   r   s      @r   ?test_partition_doc_prefers_metadata_last_modified_when_providedr      s      52
LL;Jb     &&?U  H TTTT8TTTTTTTTTTr    c                 t    t          t          d                    } t          d | D                       sJ d S )Nr   c              3  8   K   | ]}|j         j        d gk    V  dS )engNr(   	languagesr*   s     r   r-   z=test_partition_doc_adds_languages_metadata.<locals>.<genexpr>   s.      AA1qz#w.AAAAAAr    )r   r   r.   rZ   s    r   *test_partition_doc_adds_languages_metadatar      sA    -l;;<<HAAAAAAAAAAAAr    c                 |    t          t          d          d          } d | D             dgddgdgdgdggk    sJ d S )Nzlanguage-docs/eng_spa_mult.docT)detect_language_per_elementc                &    g | ]}|j         j        S rN   r   r*   s     r   
<listcomp>zOtest_partition_doc_respects_detect_language_per_element_arg.<locals>.<listcomp>   s    333QAJ 333r    r   spa)r   r   rZ   s    r   ;test_partition_doc_respects_detect_language_per_element_argr      sv    9::X\  H 43(333					8      r    )kwargsexpected_valuehi_resstrategyautor   dict[str, Any]r   
str | Nonec                    ddl m} dd}t          | |d|	          }t          t	          d
          fi |\  }|                    t                     |j        d| k    sJ d S )Nr   )_DocxPartitionerselfr   returnIterator[Element]c              3  F   K   t          d| j        j                   V  d S )Nstrategy == )r   _optsr   )r   s    r   fake_iter_document_elementsz_test_partition_odt_forwards_strategy_arg_to_partition_docx.<locals>.fake_iter_document_elements   s.      7$*"5778888888r    _iter_document_elements)side_effectr   r   )r   r   r   r   )unstructured.partition.docxr   r   r   r   assert_called_once_withr   text)r   r   r   r   r   _iter_elements_rf   s          r   :test_partition_odt_forwards_strategy_arg_to_partition_docxr      s     =<<<<<9 9 9 9 "!/	  O /==HHHHJW++C000<:.::::::::r    c                    g d} g d}t          t          d                    }t          |d         t                    sJ |d         j        j        | k    sJ |d         j        j        |k    sJ |d         t          d          k    sJ |d         j        j        | k    sJ |d         j        j        |k    sJ |d         t          d          k    sJ |d         j        j        J |d         j        j        J d S )	N)bolditalicbold-italicr   )bir   r   zfake-doc-emphasized-text.docr      z$I am a bold italic bold-italic text.   zI am a normal text.)r   r   
isinstancer   r(   emphasized_text_contentsemphasized_text_tagsr   )!expected_emphasized_text_contentsexpected_emphasized_text_tagsr2   s      r   )test_partition_doc_grabs_emphasized_textsr      s)   (X(X(X%$8$8$8!-.LMMNNHhqk5)))))A;8<]]]]]A;48UUUUUA;-(NOOOOOOA;8<]]]]]A;48UUUUUA;-(=>>>>>>A;8@@@A;4<<<<<r    c                 X    t          t          t          d                               dS )zCElements produced can be serialized then deserialized without loss.r   N)r
   r   r   rN   r    r   +test_partition_doc_round_trips_through_jsonr      s'    #M2B<2P2P$Q$QRRRRRr    c                     t          d          } t          |           }t          | d          }t          d |D                       sJ |t          |          k    sJ d S )Nr   basic)chunking_strategyc              3  Z   K   | ]&}t          |t          t          t          f          V  'd S r@   )r   r   r   r   )r+   cs     r   r-   zYtest_partition_doc_chunks_elements_when_chunking_strategy_is_specified.<locals>.<genexpr>   s4      TTz!.zBCCTTTTTTr    )r   r   r.   r   )document_pathr2   chunkss      r   Ftest_partition_doc_chunks_elements_when_chunking_strategy_is_specifiedr      sr    $\22M]++H=GDDDF TTVTTTTTTTT^H--------r    c                     t          d          } d t          |           D             }d t          |           D             }||k    sJ t          |          t          t          |                    k    sJ d S )Nzduplicate-paragraphs.docc                    g | ]	}|j         
S rN   idre   s     r   r   zStest_partition_doc_assigns_deterministic_and_unique_element_ids.<locals>.<listcomp>  s    
B
B
B'7:
B
B
Br    c                    g | ]	}|j         
S rN   r   re   s     r   r   zStest_partition_doc_assigns_deterministic_and_unique_element_ids.<locals>.<listcomp>  s    DDDGWZDDDr    )r   r   rY   set)r   idsids_2s      r   ?test_partition_doc_assigns_deterministic_and_unique_element_idsr     s~    $%?@@M
B
B]=%A%A
B
B
BCDD}]'C'CDDDE %<<<<s88s3s88}}$$$$$$r    r   c            
         t          d          t          d          t          d          t          d          t          d          t          d          t          d          t	          d          gS )	Nz&These are a few of my favorite things:ParrotsHockeyAnalysisz4This is my first thought. This is my second thought.zThis is my third thought.2023zDOYLESTOWN, PA 18901)r   r   r   r   r   rN   r    r   r!   r!     sh     	677jLMM122V&''	 	r    )r   r	   )r!   r"   r#   r$   )rP   rQ   )rq   r   )r   r	   r   r   r   r   )r   r"   ):__doc__
__future__r   pathlibtypingr   r   rI   pytest_mockr   test_unstructured.unit_utilsr   r   r	   r
   r   r   unstructured.chunking.basicr   unstructured.documents.elementsr   r   r   r   r   r   r   r   r   unstructured.partition.docr   r   r   r   r3   r<   rD   rL   rO   rU   r[   r_   rg   rj   rp   r}   r   r   r   markparametrizer   r   r   r   r   fixturer!   rN   r    r   <module>r      sL   : 9 " " " " " "                   # # # # # #                7 6 6 6 6 6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 5 4 4 4 4 4 6 6 6 6 6 6J J J J) ) ) )) ) ) )	> 	> 	> 	>: : :  
- - - -F F F> > >L L L@ @ @  W W W WU U U U&B B B

 
 
   (^z4((3z66JF5ST ; ; ;	 ;*= = =&S S S
. . .	% 	% 	% 
 
 
 
 
 
r    