
    Ng                     X    d dl Z d dlmZ d dlmZ d dlmZ  G d de j                  ZdS )    N)Counter)timeit)
Vocabularyc                       e Zd ZdZed             Zd Zd Zd Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z ej        d          d             ZdS )NgramModelVocabularyTestsztests Vocabulary Classc                 6    t          g dd          | _        d S N)zabcfdegr   r   r   r   w   
unk_cutoff)r   vocab)clss    ]/var/www/html/ai-engine/env/lib/python3.11/site-packages/nltk/test/unit/lm/test_vocabulary.py
setUpClassz$NgramModelVocabularyTests.setUpClass   s(    MMM
 
 
			    c                 :    |                      | j                   d S N)
assertTruer   selfs    r   test_truthinessz)NgramModelVocabularyTests.test_truthiness   s    
#####r   c                 F    |                      | j        j        d           d S )Nr   )assertEqualr   cutoffr   s    r   test_cutoff_value_set_correctlyz9NgramModelVocabularyTests.test_cutoff_value_set_correctly   s#    *A.....r   c                     |                      t                    5  d| j        _        d d d            d S # 1 swxY w Y   d S )N   )assertRaisesAttributeErrorr   r#   r   s    r   test_unable_to_change_cutoffz6NgramModelVocabularyTests.test_unable_to_change_cutoff   s    ~.. 	" 	" !DJ	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"s   599c                     |                      t                    5 }t          dd           d d d            n# 1 swxY w Y   d}|                     |t	          |j                             d S )Nabcr   r   z*Cutoff value cannot be less than 1. Got: 0)r'   
ValueErrorr   r"   str	exception)r   exc_infoexpected_error_msgs      r   test_cutoff_setter_checks_valuez9NgramModelVocabularyTests.test_cutoff_setter_checks_value#   s    z** 	,hu++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,I+S1C-D-DEEEEEs   9= =c                     |                      | j        j        d         d           |                      | j        j        d         d           |                      | j        j        d         d           d S )Nr   r   r   r      )r"   r   countsr   s    r   test_counts_set_correctlyz3NgramModelVocabularyTests.test_counts_set_correctly)   sh    *3/333*3/333*3/33333r   c                     |                      d| j        v            |                     d| j        v            |                     d| j        v            d S )Nr   r   r
   )r   r   assertFalser   s    r   %test_membership_check_respects_cutoffz?NgramModelVocabularyTests.test_membership_check_respects_cutoff.   sW    tz)***
*+++
*+++++r   c                 V    |                      dt          | j                             d S )N   )r"   lenr   r   s    r   test_vocab_len_respects_cutoffz8NgramModelVocabularyTests.test_vocab_len_respects_cutoff6   s(     	C
OO,,,,,r   c                     g d}g d}|                      |t          | j        j                                                             |                      |t          | j                             d S )N)	r   r   r   r   r   r   r   r   r
   )r   r   r   r   <UNK>)assertCountEquallistr   r4   keys)r   vocab_countsvocab_itemss      r   test_vocab_iter_respects_cutoffz9NgramModelVocabularyTests.test_vocab_iter_respects_cutoff;   sm    DDD333lD1B1G1G1I1I,J,JKKKk4
+;+;<<<<<r   c                 F   t          d          }|                     t          |          d           |                     |           |                     |j        |           |                    t          d                     |                     |j        |           d S )Nr   r   r   abcde)r   r"   r;   r7   assertIn	unk_labelupdater@   )r   emptys     r   test_update_empty_vocabz1NgramModelVocabularyTests.test_update_empty_vocabB   s    a(((UQ'''eou---T']]###eou-----r   c                     |                      | j                            d          d           |                      | j                            d          d           d S )Nr   r   r>   r"   r   lookupr   s    r   test_lookupz%NgramModelVocabularyTests.test_lookupK   sT    **3//555**3//99999r   c           
         |                      | j                            ddg          d           |                      | j                            d          d           |                      | j                            d          d           |                      | j                            t          t          t          d                              d           d S )Nr   r   r   r   )r   r   r   r>   r&   )r>   r>   r>   )r"   r   rN   mapr-   ranger   s    r   test_lookup_iterablesz/NgramModelVocabularyTests.test_lookup_iterablesO   s    **C:66
CCC**:66
CCC**:66GGGJc#uQxx00113N	
 	
 	
 	
 	
r   c           	         |                      | j                            d          d           |                      | j                            g           d           |                      | j                            t          g                     d           |                      | j                            d t	          dd          D                       d           d S )N c              3      K   | ]}|V  d S r   rW   ).0ns     r   	<genexpr>zHNgramModelVocabularyTests.test_lookup_empty_iterables.<locals>.<genexpr>[   s"      *B*B1*B*B*B*B*B*Br   r   )r"   r   rN   iterrT   r   s    r   test_lookup_empty_iterablesz5NgramModelVocabularyTests.test_lookup_empty_iterablesW   s    **2..333**2..333**48844b999***B*BeAqkk*B*B*BBBBGGGGGr   c                 :   |                      | j                            ddgddgg          d           |                      | j                            ddgdg          d           |                      | j                            ddggggg          d           d S )Nr   r   r   )rQ   rR   )rQ   r>   ))))rQ   rM   r   s    r   test_lookup_recursivez/NgramModelVocabularyTests.test_lookup_recursive]   s    JSzC:6779U	
 	
 	
 	**S#J+<==?TUUU**Szl^,<+=>>@VWWWWWr   c                 V   |                      t                    5  | j                            d            d d d            n# 1 swxY w Y   |                      t                    5  t	          | j                            d d g                     d d d            d S # 1 swxY w Y   d S r   r'   	TypeErrorr   rN   r@   r   s    r   test_lookup_Nonez*NgramModelVocabularyTests.test_lookup_Noned   s   y)) 	$ 	$Jd###	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$y)) 	2 	2""D$<00111	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2#   AA	A'*BB"%B"c                 V   |                      t                    5  | j                            d           d d d            n# 1 swxY w Y   |                      t                    5  t	          | j                            ddg                     d d d            d S # 1 swxY w Y   d S )Nr3   r   ra   r   s    r   test_lookup_intz)NgramModelVocabularyTests.test_lookup_intj   s   y)) 	! 	!Ja   	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!y)) 	, 	,""Aq6**+++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,rd   c                 b    |                      | j                            d          d           d S )N r>   rM   r   s    r   test_lookup_empty_strz/NgramModelVocabularyTests.test_lookup_empty_strp   s.    **2..88888r   c                 $   t          g dd          }t          g dd          }t          g ddd          }t          ddgd          }|                     ||           |                     ||           |                     ||           d S )N)r   r   r   r3   r   blah)r   rH   r   r   )r   r"   assertNotEqual)r   v1v2v3v4s        r   test_eqalityz&NgramModelVocabularyTests.test_eqalitys   s    A666A666AHHHc
q111R   B###B#####r   c                 V    |                      t          | j                  d           d S )Nz8<Vocabulary with cutoff=2 unk_label='<UNK>' and 5 items>)r"   r-   r   r   s    r   test_strz"NgramModelVocabularyTests.test_str}   s2    
OOW	
 	
 	
 	
 	
r   c           	      x    |                      | j        t          t          g d          d                     d S r	   )r"   r   r   r   r   s    r   test_creation_with_counterz4NgramModelVocabularyTests.test_creation_with_counter   sV    JUUU  	  	
 	
 	
 	
 	
r   z?Test is known to be flaky as it compares (runtime) performance.)reasonc                    t          d          }ddlm} t          |                                          }t	          dt                                }t	          dt                                }|                     ||d           d S )	NrF   r   )englishzlen(small_vocab))globalszlen(large_vocab)r3   )places)r   nltk.corpus.europarl_rawrx   wordsr   localsassertAlmostEqual)r   small_vocabrx   large_vocabsmall_vocab_len_timelarge_vocab_len_times         r   test_len_is_constantz.NgramModelVocabularyTests.test_len_is_constant   s    
 !))444444 11  &&8&((KKK%&8&((KKK 	35IRSTTTTTr   N)__name__
__module____qualname____doc__classmethodr   r    r$   r)   r1   r5   r8   r<   rD   rK   rO   rU   r]   r_   rc   rf   ri   rq   rs   ru   unittestskipr   rW   r   r   r   r      s         
 
 [
$ $ $/ / /" " "F F F4 4 4
, , ,- - -
= = =. . .: : :
 
 
H H HX X X2 2 2, , ,9 9 9$ $ $
 
 

	
 	
 	
 X]P  U U U U Ur   r   )r   collectionsr   r   nltk.lmr   TestCaser   rW   r   r   <module>r      s                      MU MU MU MU MU 1 MU MU MU MU MUr   