
    Ng(                         d dl Z d dlZddlmZ ddlmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ  G d de          Ze                                 dS )    N   )messages)unicode_blockUNICODE_BASIC_LATINUNICODE_LATIN_1_SUPPLEMENTUNICODE_LATIN_EXTENDED_BUNICODE_GENERAL_PUNCTUATIONUNICODE_ARABIC!UNICODE_LATIN_EXTENDED_ADDITIONALUNICODE_HIRAGANAUNICODE_KATAKANAUNICODE_BOPOMOFOUNICODE_BOPOMOFO_EXTENDEDUNICODE_CJK_UNIFIED_IDEOGRAPHSUNICODE_HANGUL_SYLLABLESc                   l   e Zd Z ej        d          ZdZd Zd Zd Z	e
d             Ze
d             Z ej        d           ej        d	           ej        d
           ej        d           ej        d          gZ ej        d          Z ej        d          Z ej        dez   dz   ez   dz   ej                  Zg  ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d            ej        d!           ej        d"           ej        d#           ej        d$           ej        d%           ej        d&           ej        d'           ej        d(           ej        d)           ej        d*           ej        d+           ej        d,           ej        d-           ej        d.           ej        d/           ej        d0           ej        d1           ej        d2           ej        d3           ej        d4           ej        d5           ej        d6           ej        d7           ej        d8           ej        d9           ej        d:           ej        d;           ej        d<           ej        d=           ej        d>           ej        d?           ej        d@           ej        dA           ej        dB           ej        dC           ej        dD           ej        dE           ej        dF           ej        dG           ej        dH           ej        dI           ej        dJ           ej        dK           ej        dL           ej        dM           ej        dN           ej        dO           ej        dP           ej        dQ           ej        dR           ej        dS           ej        dT           ej        dU           ej        dV           ej        dW           ej        dX           ej        dY           ej        dZ           ej        d[           ej        d\           ej        d]           ej        d^           ej        d_           ej        d`           ej        da           ej        db           ej        dc           ej        dd           ej        de           ej        df           ej        dg           ej        dh           ej        di           ej        dj           ej        dk           ej        dl           ej        dm           ej        dn           ej        do           ej        dp           ej        dq           ej        dr           ej        ds           ej        dt           ej        du           ej        dv           ej        dw           ej        dx           ej        dy           ej        dz           ej        d{           ej        d|           ej        d}           ej        d~           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d           ej        d          Zi Ze
d             ZdS )NGramzNGram.LATIN1_EXCLUDE   c                 "    d| _         d| _        d S )N F)gramscapitalword)selfs    R/var/www/html/ai-engine/env/lib/python3.11/site-packages/langdetect/utils/ngram.py__init__zNGram.__init__   s    
     c                 z   |                      |          }| j        d         }|dk    rd| _        d| _        |dk    rdS n1t          | j                  | j        k    r| j        dd         | _        | xj        |z  c_        |                                r|                                r	d| _        dS dS d| _        dS )z%Append a character into ngram buffer.r   FNr   T)	normalizer   r   lenN_GRAMisupper)r   ch	last_chars      r   add_charzNGram.add_char   s    ^^BJrN	DJ$DSyy __++ABBDJ

b

::<< 	%  "" (#'   ( (  %Dr   c                     | j         rdS |dk     s#|| j        k    st          | j                  |k     rdS |dk    r| j        d         }|dk    rdS |S | j        | d         S )zGet n-gram.Nr   r   r   )r   r!   r    r   )r   nr#   s      r   getz	NGram.get2   su     	Fq55AOOs4:':':F66BBSyyI:qbcc?"r   c                    t          |          }|t          k    r|dk     sd|cxk     rdk     sn d|k     rd}n|t          k    r"| j                            |          dk    rd}n|t
          k    rZ|t          j        d          k    rt          j        d          }|t          j        d	          k    rt          j        d
          }n*|t          k    rd}n|t          k    r-|t          j        d          k    rt          j        d          }n|t          k    r-|t          j        d          k    rt          j        d          }n|t          k    rt          j        d          }n|t          k    rt          j        d          }nk|t          t          fv rt          j        d          }nF|t          k    r| j                            ||          }n|t$          k    rt          j        d          }|S )NAZazr   r   u   șu   şu   țu   ţu   یu   يu   Ạu   ểu   あu   アu   ㄅu   가)r   r   r   LATIN1_EXCLUDEDfindr   sixur	   r
   r   r   r   r   r   r   CJK_MAPr(   r   )clsr#   blocks      r   r   zNGram.normalize@   s   b!!'''CxxC"NNNNsNNNNsRxx000"''++q00...SU8__$$U8__SU8__$$U8__111BBn$$SU8__$$U8__777SU8__$$U8__&&&xBB&&&xBB')BCCCxBB444R((BB...xB	r   c                 D      fd} j                             ||          S )zfNormalizer for Vietnamese.
        Normalize Alphabet + Diacritical Mark(U+03xx) into U+1Exx.
        c                     j                             |                     d                    }j                            |                     d                    }j        |         |         S )Nr      )TO_NORMALIZE_VI_CHARSr/   groupDMARK_CLASSNORMALIZED_VI_CHARS)malphabetdmarkr3   s      r   replz NGram.normalize_vi.<locals>.replh   sU    055aggajjAAHO((44E*51(;;r   )ALPHABET_WITH_DMARKsub)r3   textr?   s   `  r   normalize_vizNGram.normalize_vic   s7    
	< 	< 	< 	< 	< &**4666r   NORMALIZED_VI_CHARS_0300NORMALIZED_VI_CHARS_0301NORMALIZED_VI_CHARS_0303NORMALIZED_VI_CHARS_0309NORMALIZED_VI_CHARS_0323r8   r:   z([z])([z])zNGram.KANJI_1_0zNGram.KANJI_1_2zNGram.KANJI_1_4zNGram.KANJI_1_8zNGram.KANJI_1_11zNGram.KANJI_1_12zNGram.KANJI_1_13zNGram.KANJI_1_14zNGram.KANJI_1_16zNGram.KANJI_1_18zNGram.KANJI_1_22zNGram.KANJI_1_27zNGram.KANJI_1_29zNGram.KANJI_1_31zNGram.KANJI_1_35zNGram.KANJI_2_0zNGram.KANJI_2_1zNGram.KANJI_2_4zNGram.KANJI_2_9zNGram.KANJI_2_10zNGram.KANJI_2_11zNGram.KANJI_2_12zNGram.KANJI_2_13zNGram.KANJI_2_15zNGram.KANJI_2_16zNGram.KANJI_2_18zNGram.KANJI_2_21zNGram.KANJI_2_22zNGram.KANJI_2_23zNGram.KANJI_2_28zNGram.KANJI_2_29zNGram.KANJI_2_30zNGram.KANJI_2_31zNGram.KANJI_2_32zNGram.KANJI_2_35zNGram.KANJI_2_36zNGram.KANJI_2_37zNGram.KANJI_2_38zNGram.KANJI_3_1zNGram.KANJI_3_2zNGram.KANJI_3_3zNGram.KANJI_3_4zNGram.KANJI_3_5zNGram.KANJI_3_8zNGram.KANJI_3_9zNGram.KANJI_3_11zNGram.KANJI_3_12zNGram.KANJI_3_13zNGram.KANJI_3_15zNGram.KANJI_3_16zNGram.KANJI_3_18zNGram.KANJI_3_19zNGram.KANJI_3_22zNGram.KANJI_3_23zNGram.KANJI_3_27zNGram.KANJI_3_29zNGram.KANJI_3_30zNGram.KANJI_3_31zNGram.KANJI_3_32zNGram.KANJI_3_35zNGram.KANJI_3_36zNGram.KANJI_3_37zNGram.KANJI_3_38zNGram.KANJI_4_0zNGram.KANJI_4_9zNGram.KANJI_4_10zNGram.KANJI_4_16zNGram.KANJI_4_17zNGram.KANJI_4_18zNGram.KANJI_4_22zNGram.KANJI_4_24zNGram.KANJI_4_28zNGram.KANJI_4_34zNGram.KANJI_4_39zNGram.KANJI_5_10zNGram.KANJI_5_11zNGram.KANJI_5_12zNGram.KANJI_5_13zNGram.KANJI_5_14zNGram.KANJI_5_18zNGram.KANJI_5_26zNGram.KANJI_5_29zNGram.KANJI_5_34zNGram.KANJI_5_39zNGram.KANJI_6_0zNGram.KANJI_6_3zNGram.KANJI_6_9zNGram.KANJI_6_10zNGram.KANJI_6_11zNGram.KANJI_6_12zNGram.KANJI_6_16zNGram.KANJI_6_18zNGram.KANJI_6_20zNGram.KANJI_6_21zNGram.KANJI_6_22zNGram.KANJI_6_23zNGram.KANJI_6_25zNGram.KANJI_6_28zNGram.KANJI_6_29zNGram.KANJI_6_30zNGram.KANJI_6_32zNGram.KANJI_6_34zNGram.KANJI_6_35zNGram.KANJI_6_37zNGram.KANJI_6_39zNGram.KANJI_7_0zNGram.KANJI_7_3zNGram.KANJI_7_6zNGram.KANJI_7_7zNGram.KANJI_7_9zNGram.KANJI_7_11zNGram.KANJI_7_12zNGram.KANJI_7_13zNGram.KANJI_7_16zNGram.KANJI_7_18zNGram.KANJI_7_19zNGram.KANJI_7_20zNGram.KANJI_7_21zNGram.KANJI_7_23zNGram.KANJI_7_25zNGram.KANJI_7_28zNGram.KANJI_7_29zNGram.KANJI_7_32zNGram.KANJI_7_33zNGram.KANJI_7_35zNGram.KANJI_7_37c                 H    | j         D ]}|d         }|D ]}|| j        |<   d S )Nr   )	CJK_CLASSr2   )r3   cjk_listrepresentativer#   s       r   _init_cjk_mapzNGram._init_cjk_map   sG     	1 	1H%a[N 1 1"0B1	1 	1r   N)__name__
__module____qualname__r   
get_stringr.   r!   r   r%   r(   classmethodr   rC   r;   r8   r:   recompileUNICODEr@   rJ   r2   rM    r   r   r   r      sy       )h)*@AAOF! ! !% % %&# # #     [ D 7 7 [7 	6776776776776779 0H/0GHH%(%m44K$"*$$v-;dB

 
~1-..~1-..~1 	-..~1 	-..	~1
 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1 	.//~1  	-..!~1" 	-..#~1$ 	-..%~1& 	-..'~1( 	.//)~1* 	.//+~1, 	.//-~1. 	.///~10 	.//1~12 	.//3~14 	.//5~16 	.//7~18 	.//9~1: 	.//;~1< 	.//=~1> 	.//?~1@ 	.//A~1B 	.//C~1D 	.//E~1F 	.//G~1H 	.//I~1J 	.//K~1L 	.//M~1N 	-..O~1P 	-..Q~1R 	-..S~1T 	-..U~1V 	-..W~1X 	-..Y~1Z 	-..[~1\ 	.//]~1^ 	.//_~1` 	.//a~1b 	.//c~1d 	.//e~1f 	.//g~1h 	.//i~1j 	.//k~1l 	.//m~1n 	.//o~1p 	.//q~1r 	.//s~1t 	.//u~1v 	.//w~1x 	.//y~1z 	.//{~1| 	.//}~1~ 	.//~1@ 	-..A~1B 	-..C~1D 	.//E~1F 	.//G~1H 	.//I~1J 	.//K~1L 	.//M~1N 	.//O~1P 	.//Q~1R 	.//S~1T 	.//U~1V 	.//W~1X 	.//Y~1Z 	.//[~1\ 	.//]~1^ 	.//_~1` 	.//a~1b 	.//c~1d 	.//e~1f 	.//g~1h 	.//i~1j 	-..k~1l 	-..m~1n 	-..o~1p 	.//q~1r 	.//s~1t 	.//u~1v 	.//w~1x 	.//y~1z 	.//{~1| 	.//}~1~ 	.//~1@ 	.//A~1B 	.//C~1D 	.//E~1F 	.//G~1H 	.//I~1J 	.//K~1L 	.//M~1N 	.//O~1P 	.//Q~1R 	.//S~1T 	-..U~1V 	-..W~1X 	-..Y~1Z 	-..[~1\ 	-..]~1^ 	.//_~1` 	.//a~1b 	.//c~1d 	.//e~1f 	.//g~1h 	.//i~1j 	.//k~1l 	.//m~1n 	.//o~1p 	.//q~1r 	.//s~1t 	.//u~1v 	.//w~1x 	.//y~1z 	.//{~1| 	.//}~1I@ G1 1 [1 1 1r   r   )rS   r0    r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   objectr   rM   rV   r   r   <module>rY      s   				 



                                   "k1 k1 k1 k1 k1F k1 k1 k1Z        r   