
    Ng*                    0   d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ  ej        e          j                                        Zej         !                    d          Z"g dZ#ej         !                    d          Z"d Z$ej%        &                    de#          d&d            Z'ej%        &                    de#          d&d            Z(ej%        &                    de#          d&d            Z)ej%        &                    de#          d&d            Z*ej%        &                    de#          d&d            Z+d Z,d Z-d Z.d Z/d'dZ0d Z1d  Z2d'd!Z3d" Z4d# Z5d$ Z6d% Z7dS )(z4Test-suite for `unstructured.partition.json` module.    )annotationsN)MockFixture)example_doc_path)CompositeElement)FileType)partition_email)partition_htmlpartition_json)partition_text)partition_xml)elements_to_jsonz/.dockerenv)fake-text.txtzfake-html.htmlzeml/fake-email.emlc                     t          ddd          } t          |           dk    sJ t          d | D                       sJ d S )N%example-docs/spring-weather.html.jsonbasici  )chunking_strategymax_characters
   c              3  @   K   | ]}t          |t                    V  d S N)
isinstancer   ).0chs     a/var/www/html/ai-engine/env/lib/python3.11/site-packages/test_unstructured/partition/test_json.py	<genexpr>zPtest_it_chunks_elements_when_a_chunking_strategy_is_specified.<locals>.<genexpr>)   s-      AABz".//AAAAAA    )r   lenall)chunkss    r   =test_it_chunks_elements_when_a_chunking_strategy_is_specifiedr!   #   s`    /7[_  F v;;"AA&AAAAAAAAAAr   filenamestrc                :   t          |           }g }t          j        t          j                            |          d                   }|t          j        k    rt          |          }|t          j        k    rt          |          }|t          j
        k    rt          |          }|t          j        k    rt          |          }t          j                    5 }t          j                            |           }t          j                            ||dz             }t%          ||d           t'          |          }d d d            n# 1 swxY w Y   t)          |          dk    sJ t)          t+          |d                             dk    sJ t)          |          t)          |          k    sJ t-          t)          |                    D ]G}||         ||         k    sJ ||         j        j        |                     d          d         k    sJ Hd S )	N   r"   .json   r"   indentr   /)r   r   from_extensionospathsplitextTXTr   HTMLr	   XMLr   EMLr   tempfileTemporaryDirectorybasenamejoinr   r   r   r#   rangemetadatar"   split)	r"   r/   elementsfiletypetmpdir	_filename	test_pathtest_elementsis	            r   !test_partition_json_from_filenamerC   ,   s3   H%%DH&rw'7'7'='=a'@AAH8<!40008=  !40008< $///8<"D111		$	&	& ;&G$$X..	GLLW)<==	Ia@@@@&	:::	; ; ; ; ; ; ; ; ; ; ; ; ; ; ; x==1s8A;  1$$$$x==C......3x==!! H H{mA.....{#,s0C0CB0GGGGGGH Hs   A%EEEc                f   t          |           }g }t          j        t          j                            |          d                   }|t          j        k    rt          |          }|t          j        k    rt          |          }|t          j
        k    rt          |          }|t          j        k    rt          |          }t          j                    5 }t          j                            |           }t          j                            ||dz             }t%          ||d           t'          |d          }d d d            n# 1 swxY w Y   t)          |          dk    sJ t)          t+          |d                             dk    sJ t-          d	 |D                       sJ d S )
Nr%   r&   r'   r(   r)   test)r"   metadata_filenamer   c              3  6   K   | ]}|j         j        d k    V  dS )rE   N)r:   r"   )r   elements     r   r   zKtest_partition_json_from_filename_with_metadata_filename.<locals>.<genexpr>_   s-      PPww(F2PPPPPPr   )r   r   r-   r.   r/   r0   r1   r   r2   r	   r3   r   r4   r   r5   r6   r7   r8   r   r   r   r#   r   )r"   r/   r<   r=   r>   r?   r@   rA   s           r   8test_partition_json_from_filename_with_metadata_filenamerI   I   s   H%%DH&rw'7'7'='=a'@AAH8<!40008=  !40008< $///8<"D111		$	&	& U&G$$X..	GLLW)<==	Ia@@@@&	VTTT	U U U U U U U U U U U U U U U }!!!!s=#$$%%))))PP-PPPPPPPPPPs   A&EEEc                   t          |           }g }t          j        t          j                            |          d                   }|t          j        k    rt          |          }|t          j        k    rt          |          }|t          j
        k    rt          |          }|t          j        k    rt          |          }t          j                    5 }t          j                            |           }t          j                            ||dz             }t%          ||d           t'          |d          5 }t)          |          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t+          |          dk    sJ t+          t-          |d                             dk    sJ t+          |          t+          |          k    sJ t/          t+          |                    D ]G}	||	         ||	         k    sJ ||	         j        j        |                     d	          d
         k    sJ Hd S )Nr%   r&   r'   r(   r)   rbfiler   r+   r,   )r   r   r-   r.   r/   r0   r1   r   r2   r	   r3   r   r4   r   r5   r6   r7   r8   r   openr   r   r#   r9   r:   r"   r;   
r"   r/   r<   r=   r>   r?   r@   frA   rB   s
             r   test_partition_json_from_filerQ   b   s   H%%DH&rw'7'7'='=a'@AAH8<!40008=  !40008< $///8<"D111		$	&	& 3&G$$X..	GLLW)<==	Ia@@@@)T"" 	3a*222M	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3	3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 x==1s8A;  1$$$$x==C......3x==!! H H{mA.....{#,s0C0CB0GGGGGGH Hs7   A%E5EE5E"	"E5%E"	&E55E9<E9c                t   t          |           }g }t          j        t          j                            |          d                   }|t          j        k    rt          |          }|t          j        k    rt          |          }|t          j
        k    rt          |          }|t          j        k    rt          |          }t          j                    5 }t          j                            |           }t          j                            ||dz             }t%          ||d           t'          |d          5 }t)          |d          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t+          t-          |                    D ]}	||	         j        j        dk    sJ d S )	Nr%   r&   r'   r(   r)   rK   rE   )rM   rF   )r   r   r-   r.   r/   r0   r1   r   r2   r	   r3   r   r4   r   r5   r6   r7   r8   r   rN   r   r9   r   r:   r"   rO   s
             r   4test_partition_json_from_file_with_metadata_filenamerS      s0   H%%DH&rw'7'7'='=a'@AAH8<!40008=  !40008< $///8<"D111		$	&	& M&G$$X..	GLLW)<==	Ia@@@@)T"" 	Ma*VLLLM	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M	M M M M M M M M M M M M M M M 3}%%&& < <Q(1V;;;;;< <s7   A%E6EE6E#	#E6&E#	'E66E:=E:c                   t          |           }g }t          j        t          j                            |          d                   }|t          j        k    rt          |          }|t          j        k    rt          |          }|t          j
        k    rt          |          }|t          j        k    rt          |          }t          j                    5 }t          j                            |           }t          j                            ||dz             }t%          ||d           t'          |          5 }|                                }d d d            n# 1 swxY w Y   t+          |          }	d d d            n# 1 swxY w Y   t-          |          dk    sJ t-          t/          |d                             dk    sJ t-          |          t-          |	          k    sJ t1          t-          |                    D ]G}
||
         |	|
         k    sJ ||
         j        j        |                     d          d	         k    sJ Hd S )
Nr%   r&   r'   r(   r)   textr   r+   r,   )r   r   r-   r.   r/   r0   r1   r   r2   r	   r3   r   r4   r   r5   r6   r7   r8   r   rN   readr   r   r#   r9   r:   r"   r;   )r"   r/   r<   r=   r>   r?   r@   rP   rV   rA   rB   s              r   test_partition_json_from_textrX      s   H%%DH&rw'7'7'='=a'@AAH8<!40008=  !40008< $///8<"D111		$	&	& 2&G$$X..	GLLW)<==	Ia@@@@)__ 	6688D	 	 	 	 	 	 	 	 	 	 	 	 	 	 	&D1112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 x==1s8A;  1$$$$x==C......3x==!! H H{mA.....{#,s0C0CB0GGGGGGH Hs7   A$F E!F!E%	%F(E%	)FFFc                     t          j        t                    5  t                       d d d            d S # 1 swxY w Y   d S r   pytestraises
ValueErrorr    r   r   .test_partition_json_raises_with_none_specifiedr_      s    	z	"	"                   s   6::c                 2    t          d          g k    sJ d S )N rU   r
   r^   r   r   +test_partition_json_works_with_empty_stringrb      s$    r"""b((((((r   c                 2    t          d          g k    sJ d S )Nz[]rU   r
   r^   r   r   )test_partition_json_works_with_empty_listrd      s$    t$$$******r   c                    t          d          } g }t          j        t          j                            |           d                   }|t          j        k    rt          |           }|t          j        k    rt          |           }|t          j
        k    rt          |           }|t          j        k    rt          |           }t          j                    5 }t          j                            |d          }t#          ||d           t%          |d          5 }|                                                    d          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t+          j        t.                    5  t1          ||	           d d d            n# 1 swxY w Y   t+          j        t.                    5  t1          ||
           d d d            n# 1 swxY w Y   t+          j        t.                    5  t1          ||           d d d            n# 1 swxY w Y   t+          j        t.                    5  t1          |||           d d d            d S # 1 swxY w Y   d S )Nr   r%   r&   zfake-text.txt.jsonr(   r)   rK   zutf-8)r"   rM   )r"   rV   )rM   rV   )r"   rM   rV   )r   r   r-   r.   r/   r0   r1   r   r2   r	   r3   r   r4   r   r5   r6   r8   r   rN   rW   decoder[   r\   r]   r   )r/   r<   r=   r>   r@   rP   rV   s          r   2test_partition_json_raises_with_too_many_specifiedrg      s   O,,DH&rw'7'7'='=a'@AAH8<!40008=  !40008< $///8<"D111		$	&	& ,&GLL)=>>	Ia@@@@)T"" 	,a6688??7++D	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,, , , , , , , , , , , , , , , 
z	"	" 3 3	22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 
z	"	" 6 6	55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 
z	"	" * *AD))))* * * * * * * * * * * * * * * 
z	"	" > >	====> > > > > > > > > > > > > > > > > >s   AE*(EE*E	E*E	E**E.1E.F,,F03F0G..G25G2H00H47H4I44I8;I8mockerr   c                    d|                      d           t          t          d                    }t          fd|D                       sJ d S )N2029-07-05T09:24:282unstructured.partition.json.get_last_modified_datereturn_valuespring-weather.html.jsonc              3  8   K   | ]}|j         j        k    V  d S r   r:   last_modified)r   efilesystem_last_modifieds     r   r   zXtest_partition_json_from_file_path_gets_last_modified_from_filesystem.<locals>.<genexpr>   s.      VVqz'+CCVVVVVVr   )patchr   r   r   )rh   r<   rs   s     @r   Etest_partition_json_from_file_path_gets_last_modified_from_filesystemru      sr    4
LL<Kc     ./IJJKKHVVVVXVVVVVVVVVVr   c                     t          dd          5 } t          |           }d d d            n# 1 swxY w Y   t          d |D                       sJ d S )Nr   rK   rL   c              3  2   K   | ]}|j         j        d u V  d S r   rp   r   rr   s     r   r   zHtest_partition_json_from_file_gets_last_modified_None.<locals>.<genexpr>   ,      BBAqz'4/BBBBBBr   )rN   r   r   )rP   r<   s     r   5test_partition_json_from_file_gets_last_modified_Nonerz      s    	5t	<	< *!q)))* * * * * * * * * * * * * * * BBBBBBBBBBBBs   .22c                     t          d          5 } |                                 }d d d            n# 1 swxY w Y   t          |          }t          d |D                       sJ d S )Nr   rU   c              3  2   K   | ]}|j         j        d u V  d S r   rp   rx   s     r   r   zHtest_partition_json_from_text_gets_last_modified_None.<locals>.<genexpr>   ry   r   rN   rW   r   r   )rP   rV   r<   s      r   5test_partition_json_from_text_gets_last_modified_Noner~      s    	5	6	6 !vvxx               4(((HBBBBBBBBBBBBs   155c                    d}d|                      d|           t          d          }t          fd|D                       sJ d S )Nrj   2020-07-05T09:24:28rk   rl   r   )metadata_last_modifiedc              3  8   K   | ]}|j         j        k    V  d S r   rp   r   rr   r   s     r   r   zTtest_partition_json_from_file_path_prefers_metadata_last_modified.<locals>.<genexpr>  .      TTaqz'+AATTTTTTr   )rt   r   r   )rh   rs   r<   r   s      @r   Atest_partition_json_from_file_path_prefers_metadata_last_modifiedr      s}    42
LL<Kc     /H^  H TTTT8TTTTTTTTTTr   c                     dt          t          d          d          5 } t          |           }d d d            n# 1 swxY w Y   t          fd|D                       sJ d S )Nr   rn   rK   )rM   r   c              3  8   K   | ]}|j         j        k    V  d S r   rp   r   s     r   r   zOtest_partition_json_from_file_prefers_metadata_last_modified.<locals>.<genexpr>  r   r   )rN   r   r   r   )rP   r<   r   s     @r   <test_partition_json_from_file_prefers_metadata_last_modifiedr     s    2	9::D	A	A YQ!qAWXXXY Y Y Y Y Y Y Y Y Y Y Y Y Y Y TTTT8TTTTTTTTTTs   ?AAc                     dt          d          5 } |                                 }d d d            n# 1 swxY w Y   t          |          }t          fd|D                       sJ d S )Nr   r   )rV   r   c              3  8   K   | ]}|j         j        k    V  d S r   rp   r   s     r   r   zOtest_partition_json_from_text_prefers_metadata_last_modified.<locals>.<genexpr>  r   r   r}   )rP   rV   r<   r   s      @r   <test_partition_json_from_text_prefers_metadata_last_modifiedr     s    2	5	6	6 !vvxx               4@VWWWHTTTT8TTTTTTTTTTs   488c                     d} t          j        t                    5  t          |            d d d            d S # 1 swxY w Y   d S )Nz{"hi": "there"}rU   rZ   rU   s    r   2test_partition_json_raises_with_unprocessable_jsonr   #  s     D	z	"	" " "D!!!!" " " " " " " " " " " " " " " " " "   :>>c                     d} t          j        t                    5  t          |            d d d            d S # 1 swxY w Y   d S )Nz[{"hi": "there"}]]rU   rZ   rU   s    r   ,test_partition_json_raises_with_invalid_jsonr   +  s    D	z	"	" " "D!!!!" " " " " " " " " " " " " " " " " "r   )r"   r#   )rh   r   )8__doc__
__future__r   r.   pathlibr5   r[   pytest_mockr   test_unstructured.unit_utilsr   unstructured.documents.elementsr   unstructured.file_utils.modelr   unstructured.partition.emailr   unstructured.partition.htmlr	   unstructured.partition.jsonr   unstructured.partition.textr   unstructured.partition.xmlr   unstructured.staging.baser   Path__file__parentresolve	DIRECTORYr/   existsis_in_docker
test_filesr!   markparametrizerC   rI   rQ   rS   rX   r_   rb   rd   rg   ru   rz   r~   r   r   r   r   r   r^   r   r   <module>r      sL   : : " " " " " " 				    # # # # # # 9 9 9 9 9 9 < < < < < < 2 2 2 2 2 2 8 8 8 8 8 8 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 4 4 4 4 4 4 6 6 6 6 6 6GL"")1133	w~~m,,  
 w~~m,,B B B Z00H H H 10H8 Z00Q Q Q 10Q0 Z00H H H 10H8 Z00< < < 10<. Z00H H H 10H:  
) ) )+ + +> > >FW W W WC C CC C CU U U UU U UU U U" " "" " " " "r   