
    Ng
                         d dl mZmZ d dlmZ d dlZd dlmZmZ d dl	m
Z
 d Zd Zdee         d	eeeef                  fd
Zd Zd Zd Zd Zd ZdS )    )ListTuple)patchN)mock_sent_tokenizemock_word_tokenize)tokenizec                  f   t           j                                         t          j        t
          dt                    5  t          j        t           d          5 } t          j                     d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                                  d S )Nfind)side_effectdownload_nltk_packages)r   &_download_nltk_packages_if_not_presentcache_clearr   objectnltkLookupErrorassert_called_oncemock_downloads    _/var/www/html/ai-engine/env/lib/python3.11/site-packages/test_unstructured/nlp/test_tokenize.py*test_nltk_packages_download_if_not_presentr   
   s   3??AAA	dF	<	<	< > >\($<== 	>;===	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>> > > > > > > > > > > > > > > $$&&&&&s6    BA;/B;A?	?BA?	BBBc                  X   t           j                                         t          j        t
          d          5  t          j        t
          d          5 } t          j                     d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                                  d S )Nr
   download)r   r   r   r   r   r   assert_not_calledr   s    r   %test_nltk_packages_do_not_download_ifr      s
   3??AAA	dF	#	# : :U\$
%C%C :}7999: : : : : : : : : : : : : : : : : : : : : : : : : : : : : : ##%%%%%s5   BA4(B4A8	8B;A8	<BBBtokensreturnc                     g }| D ]I}|                                 dk    r|                    |df           2|                    |df           J|S )NaskVB )lowerappend)r   pos_tagstokens      r   mock_pos_tagr%      s`    &(H ) );;==E!!OOUDM****OOUBK((((O    c                  B    t          j        d          } | g dk    sJ d S )NzITEM 2A. PROPERTIES))ITEMNNP)2ACD).r,   )
PROPERTIESNN)r   pos_tag)parts_of_speechs    r   test_pos_tagr1   %   sC    &'<==O         r&   c                    |                      t          dt                     |                      t          dt                     t          j                                         t          j                                        j        dk    sJ t          j        d           t          j                                        j        dk    sJ d S N_word_tokenize_pos_tagr   !Greetings! I am from outer space.   )setattrr   r   r%   word_tokenizer   
cache_infocurrsizemonkeypatchs    r   test_word_tokenize_cachesr>   /   s    "24FGGG*l;;;&&(((!,,..71<<<<>???!,,..71<<<<<<r&   c                    |                      t          dt                     |                      t          dt                     |                      t          dt                     t          j                                         t          j                                        j        dk    sJ t          j        d           t          j                                        j        dk    sJ d S )N_sent_tokenizer4   r5   r   r6   r7   )	r8   r   r   r   r%   sent_tokenizer   r:   r;   r<   s    r   test_sent_tokenize_cachesrB   8   s    "24FGGG"24FGGG*l;;;&&(((!,,..71<<<<>???!,,..71<<<<<<r&   c                    |                      t          dt                     |                      t          dt                     t          j                                         t          j                                        j        dk    sJ t          j        d           t          j                                        j        dk    sJ d S r3   )r8   r   r   r%   r/   r   r:   r;   r<   s    r   test_pos_tag_cachesrD   B   s    "24FGGG*l;;;  """&&((1Q66668999&&((1Q666666r&   c                      d} t          j        |            t          j        |            t          j        |            d S )Nz$I am a big brown bear. What are you?)r   rA   r9   r/   )sentences    r   test_tokenizers_functions_runrG   K   s@    5H8$$$8$$$Xr&   )typingr   r   unittest.mockr   r   test_unstructured.nlp.mock_nltkr   r   unstructured.nlpr   r   r   strr%   r1   r>   rB   rD   rG    r&   r   <module>rN      s                  R R R R R R R R % % % % % %' ' '& & &c tE#s(O'<      = = == = =7 7 7    r&   