
    g,                        d dl mZmZ  G d de          Z G d de          Z G d de          Z G d	 d
e          Z G d de          Z G d de          Z G d de          Z	 G d de          Z
 G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z G d d e          Z G d! d"e          Z G d# d$e          Z G d% d&e          Z G d' d(e          Z G d) d*e          Z G d+ d,e          Z G d- d.e          Z G d/ d0e          Z G d1 d2e          Z G d3 d4e          Z G d5 d6e          Z G d7 d8e          Z G d9 d:e          Z G d; d<e          Z G d= d>e          Z  G d? d@e          Z! G dA dBe          Z" G dC dDe          Z# G dE dFe          Z$ G dG dHe          Z% G dI dJe          Z& G dK dLe          Z' G dM dNe          Z( G dO dPe          Z) G dQ dRe          Z* G dS dTe          Z+ G dU dVe          Z, G dW dXe          Z- G dY dZe          Z. G d[ d\e          Z/ G d] d^e          Z0 G d_ d`e          Z1 G da dbe          Z2 G dc dde          Z3 G de dfe          Z4 G dg dhe          Z5 G di dje          Z6 G dk dle          Z7 G dm dne          Z8 G do dpe          Z9 G dq dre          Z: G ds dte          Z; G du dve          Z< G dw dxe          Z= G dy dze          Z> G d{ d|e          Z? G d} d~e          Z@ G d de          ZA G d de          ZB G d de          ZCdS )   )DummyObjectrequires_backendsc                       e Zd ZdgZd ZdS )AlbertTokenizerFast
tokenizersc                 (    t          | dg           d S Nr   r   selfargskwargss      g/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/utils/dummy_tokenizers_objects.py__init__zAlbertTokenizerFast.__init__       $/////    N__name__
__module____qualname__	_backendsr    r   r   r   r      *        I0 0 0 0 0r   r   )	metaclassc                       e Zd ZdgZd ZdS )BartTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBartTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )BarthezTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBarthezTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )BertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBertTokenizerFast.__init__   r   r   Nr   r   r   r   r"   r"      r   r   r"   c                       e Zd ZdgZd ZdS )BigBirdTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBigBirdTokenizerFast.__init__$   r   r   Nr   r   r   r   r%   r%   !   r   r   r%   c                       e Zd ZdgZd ZdS )BlenderbotTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z BlenderbotTokenizerFast.__init__+   r   r   Nr   r   r   r   r(   r(   (   r   r   r(   c                       e Zd ZdgZd ZdS )BlenderbotSmallTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z%BlenderbotSmallTokenizerFast.__init__2   r   r   Nr   r   r   r   r+   r+   /   r   r   r+   c                       e Zd ZdgZd ZdS )BloomTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBloomTokenizerFast.__init__9   r   r   Nr   r   r   r   r.   r.   6   r   r   r.   c                       e Zd ZdgZd ZdS )CamembertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCamembertTokenizerFast.__init__@   r   r   Nr   r   r   r   r1   r1   =   r   r   r1   c                       e Zd ZdgZd ZdS )CLIPTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCLIPTokenizerFast.__init__G   r   r   Nr   r   r   r   r4   r4   D   r   r   r4   c                       e Zd ZdgZd ZdS )CodeLlamaTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCodeLlamaTokenizerFast.__init__N   r   r   Nr   r   r   r   r7   r7   K   r   r   r7   c                       e Zd ZdgZd ZdS )CodeGenTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCodeGenTokenizerFast.__init__U   r   r   Nr   r   r   r   r:   r:   R   r   r   r:   c                       e Zd ZdgZd ZdS )CohereTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCohereTokenizerFast.__init__\   r   r   Nr   r   r   r   r=   r=   Y   r   r   r=   c                       e Zd ZdgZd ZdS )ConvBertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zConvBertTokenizerFast.__init__c   r   r   Nr   r   r   r   r@   r@   `   r   r   r@   c                       e Zd ZdgZd ZdS )CpmTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCpmTokenizerFast.__init__j   r   r   Nr   r   r   r   rC   rC   g   r   r   rC   c                       e Zd ZdgZd ZdS )DebertaTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zDebertaTokenizerFast.__init__q   r   r   Nr   r   r   r   rF   rF   n   r   r   rF   c                       e Zd ZdgZd ZdS )DebertaV2TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zDebertaV2TokenizerFast.__init__x   r   r   Nr   r   r   r   rI   rI   u   r   r   rI   c                       e Zd ZdgZd ZdS )RealmTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zRealmTokenizerFast.__init__   r   r   Nr   r   r   r   rL   rL   |   r   r   rL   c                       e Zd ZdgZd ZdS )RetriBertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zRetriBertTokenizerFast.__init__   r   r   Nr   r   r   r   rO   rO      r   r   rO   c                       e Zd ZdgZd ZdS )DistilBertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z DistilBertTokenizerFast.__init__   r   r   Nr   r   r   r   rR   rR      r   r   rR   c                       e Zd ZdgZd ZdS )DPRContextEncoderTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z'DPRContextEncoderTokenizerFast.__init__   r   r   Nr   r   r   r   rU   rU      r   r   rU   c                       e Zd ZdgZd ZdS )DPRQuestionEncoderTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z(DPRQuestionEncoderTokenizerFast.__init__   r   r   Nr   r   r   r   rX   rX      r   r   rX   c                       e Zd ZdgZd ZdS )DPRReaderTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zDPRReaderTokenizerFast.__init__   r   r   Nr   r   r   r   r[   r[      r   r   r[   c                       e Zd ZdgZd ZdS )ElectraTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zElectraTokenizerFast.__init__   r   r   Nr   r   r   r   r^   r^      r   r   r^   c                       e Zd ZdgZd ZdS )FNetTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zFNetTokenizerFast.__init__   r   r   Nr   r   r   r   ra   ra      r   r   ra   c                       e Zd ZdgZd ZdS )FunnelTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zFunnelTokenizerFast.__init__   r   r   Nr   r   r   r   rd   rd      r   r   rd   c                       e Zd ZdgZd ZdS )GemmaTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zGemmaTokenizerFast.__init__   r   r   Nr   r   r   r   rg   rg      r   r   rg   c                       e Zd ZdgZd ZdS )GPT2TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zGPT2TokenizerFast.__init__   r   r   Nr   r   r   r   rj   rj      r   r   rj   c                       e Zd ZdgZd ZdS )GPTNeoXTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zGPTNeoXTokenizerFast.__init__   r   r   Nr   r   r   r   rm   rm      r   r   rm   c                       e Zd ZdgZd ZdS )GPTNeoXJapaneseTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z!GPTNeoXJapaneseTokenizer.__init__   r   r   Nr   r   r   r   rp   rp      r   r   rp   c                       e Zd ZdgZd ZdS )HerbertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zHerbertTokenizerFast.__init__   r   r   Nr   r   r   r   rs   rs      r   r   rs   c                       e Zd ZdgZd ZdS )LayoutLMTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLayoutLMTokenizerFast.__init__   r   r   Nr   r   r   r   rv   rv      r   r   rv   c                       e Zd ZdgZd ZdS )LayoutLMv2TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z LayoutLMv2TokenizerFast.__init__   r   r   Nr   r   r   r   ry   ry      r   r   ry   c                       e Zd ZdgZd ZdS )LayoutLMv3TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z LayoutLMv3TokenizerFast.__init__   r   r   Nr   r   r   r   r|   r|      r   r   r|   c                       e Zd ZdgZd ZdS )LayoutXLMTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLayoutXLMTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )LEDTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLEDTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )LlamaTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLlamaTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )LongformerTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z LongformerTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )LxmertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLxmertTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )MarkupLMTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMarkupLMTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )MBartTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMBartTokenizerFast.__init__   r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )MBart50TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMBart50TokenizerFast.__init__'  r   r   Nr   r   r   r   r   r   $  r   r   r   c                       e Zd ZdgZd ZdS )MobileBertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z MobileBertTokenizerFast.__init__.  r   r   Nr   r   r   r   r   r   +  r   r   r   c                       e Zd ZdgZd ZdS )MPNetTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMPNetTokenizerFast.__init__5  r   r   Nr   r   r   r   r   r   2  r   r   r   c                       e Zd ZdgZd ZdS )MT5TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMT5TokenizerFast.__init__<  r   r   Nr   r   r   r   r   r   9  r   r   r   c                       e Zd ZdgZd ZdS )MvpTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMvpTokenizerFast.__init__C  r   r   Nr   r   r   r   r   r   @  r   r   r   c                       e Zd ZdgZd ZdS )NllbTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zNllbTokenizerFast.__init__J  r   r   Nr   r   r   r   r   r   G  r   r   r   c                       e Zd ZdgZd ZdS )NougatTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zNougatTokenizerFast.__init__Q  r   r   Nr   r   r   r   r   r   N  r   r   r   c                       e Zd ZdgZd ZdS )OpenAIGPTTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zOpenAIGPTTokenizerFast.__init__X  r   r   Nr   r   r   r   r   r   U  r   r   r   c                       e Zd ZdgZd ZdS )PegasusTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zPegasusTokenizerFast.__init___  r   r   Nr   r   r   r   r   r   \  r   r   r   c                       e Zd ZdgZd ZdS )Qwen2TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zQwen2TokenizerFast.__init__f  r   r   Nr   r   r   r   r   r   c  r   r   r   c                       e Zd ZdgZd ZdS )ReformerTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zReformerTokenizerFast.__init__m  r   r   Nr   r   r   r   r   r   j  r   r   r   c                       e Zd ZdgZd ZdS )RemBertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zRemBertTokenizerFast.__init__t  r   r   Nr   r   r   r   r   r   q  r   r   r   c                       e Zd ZdgZd ZdS )RobertaTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zRobertaTokenizerFast.__init__{  r   r   Nr   r   r   r   r   r   x  r   r   r   c                       e Zd ZdgZd ZdS )RoFormerTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zRoFormerTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )SeamlessM4TTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z!SeamlessM4TTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )SplinterTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zSplinterTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )SqueezeBertTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z!SqueezeBertTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )T5TokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zT5TokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )UdopTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zUdopTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )WhisperTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zWhisperTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )XGLMTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zXGLMTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )XLMRobertaTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z XLMRobertaTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )XLNetTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zXLNetTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd ZdS )PreTrainedTokenizerFastr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z PreTrainedTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   N)Dutilsr   r   r   r   r   r"   r%   r(   r+   r.   r1   r4   r7   r:   r=   r@   rC   rF   rI   rL   rO   rR   rU   rX   r[   r^   ra   rd   rg   rj   rm   rp   rs   rv   ry   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>r      s   2 2 2 2 2 2 2 20 0 0 0 0K 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0K 0 0 0 00 0 0 0 0k 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0K 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0k 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0K 0 0 0 00 0 0 0 0k 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0K 0 0 0 00 0 0 0 0{ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0k 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0k 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0k 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0[ 0 0 0 00 0 0 0 0+ 0 0 0 00 0 0 0 0 0 0 0 00 0 0 0 0; 0 0 0 00 0 0 0 0 0 0 0 0 0 0r   