
    g7                     N   d dl mZmZ  G d de          Z G d de          Z G d de          Z G d	 d
e          Z G d de          Z G d de          Z G d de          Z	 G d de          Z
 G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          Z G d d e          Z G d! d"e          Z G d# d$e          Z G d% d&e          Z G d' d(e          Z G d) d*e          Z G d+ d,e          Z G d- d.e          Z G d/ d0e          Z G d1 d2e          Z G d3 d4e          Z G d5 d6e          Z G d7 d8e          Z G d9 d:e          Z G d; d<e          Z G d= d>e          Z  G d? d@e          Z! G dA dBe          Z" G dC dDe          Z# G dE dFe          Z$ G dG dHe          Z% G dI dJe          Z&dKS )L   )DummyObjectrequires_backendsc                       e Zd ZdgZd ZdS )AlbertTokenizersentencepiecec                 (    t          | dg           d S Nr   r   selfargskwargss      j/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/utils/dummy_sentencepiece_objects.py__init__zAlbertTokenizer.__init__       $ 122222    N__name__
__module____qualname__	_backendsr    r   r   r   r      +         !I3 3 3 3 3r   r   )	metaclassc                       e Zd ZdgZd ZdS )BarthezTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBarthezTokenizer.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )BartphoTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBartphoTokenizer.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )BertGenerationTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   z BertGenerationTokenizer.__init__   r   r   Nr   r   r   r   r"   r"      r   r   r"   c                       e Zd ZdgZd ZdS )BigBirdTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zBigBirdTokenizer.__init__$   r   r   Nr   r   r   r   r%   r%   !   r   r   r%   c                       e Zd ZdgZd ZdS )CamembertTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCamembertTokenizer.__init__+   r   r   Nr   r   r   r   r(   r(   (   r   r   r(   c                       e Zd ZdgZd ZdS )CodeLlamaTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCodeLlamaTokenizer.__init__2   r   r   Nr   r   r   r   r+   r+   /   r   r   r+   c                       e Zd ZdgZd ZdS )CpmTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zCpmTokenizer.__init__9   r   r   Nr   r   r   r   r.   r.   6   r   r   r.   c                       e Zd ZdgZd ZdS )DebertaV2Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zDebertaV2Tokenizer.__init__@   r   r   Nr   r   r   r   r1   r1   =   r   r   r1   c                       e Zd ZdgZd ZdS )ErnieMTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zErnieMTokenizer.__init__G   r   r   Nr   r   r   r   r4   r4   D   r   r   r4   c                       e Zd ZdgZd ZdS )XLMProphetNetTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zXLMProphetNetTokenizer.__init__N   r   r   Nr   r   r   r   r7   r7   K   r   r   r7   c                       e Zd ZdgZd ZdS )FNetTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zFNetTokenizer.__init__U   r   r   Nr   r   r   r   r:   r:   R   r   r   r:   c                       e Zd ZdgZd ZdS )GemmaTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zGemmaTokenizer.__init__\   r   r   Nr   r   r   r   r=   r=   Y   r   r   r=   c                       e Zd ZdgZd ZdS )GPTSw3Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zGPTSw3Tokenizer.__init__c   r   r   Nr   r   r   r   r@   r@   `   r   r   r@   c                       e Zd ZdgZd ZdS )LayoutXLMTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLayoutXLMTokenizer.__init__j   r   r   Nr   r   r   r   rC   rC   g   r   r   rC   c                       e Zd ZdgZd ZdS )LlamaTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zLlamaTokenizer.__init__q   r   r   Nr   r   r   r   rF   rF   n   r   r   rF   c                       e Zd ZdgZd ZdS )M2M100Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zM2M100Tokenizer.__init__x   r   r   Nr   r   r   r   rI   rI   u   r   r   rI   c                       e Zd ZdgZd ZdS )MarianTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMarianTokenizer.__init__   r   r   Nr   r   r   r   rL   rL   |   r   r   rL   c                       e Zd ZdgZd ZdS )MBartTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMBartTokenizer.__init__   r   r   Nr   r   r   r   rO   rO      r   r   rO   c                       e Zd ZdgZd ZdS )MBart50Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMBart50Tokenizer.__init__   r   r   Nr   r   r   r   rR   rR      r   r   rR   c                       e Zd ZdgZd ZdS )MLukeTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMLukeTokenizer.__init__   r   r   Nr   r   r   r   rU   rU      r   r   rU   c                       e Zd ZdgZd ZdS )MT5Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zMT5Tokenizer.__init__   r   r   Nr   r   r   r   rX   rX      r   r   rX   c                       e Zd ZdgZd ZdS )NllbTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zNllbTokenizer.__init__   r   r   Nr   r   r   r   r[   r[      r   r   r[   c                       e Zd ZdgZd ZdS )PegasusTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zPegasusTokenizer.__init__   r   r   Nr   r   r   r   r^   r^      r   r   r^   c                       e Zd ZdgZd ZdS )PLBartTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zPLBartTokenizer.__init__   r   r   Nr   r   r   r   ra   ra      r   r   ra   c                       e Zd ZdgZd ZdS )ReformerTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zReformerTokenizer.__init__   r   r   Nr   r   r   r   rd   rd      r   r   rd   c                       e Zd ZdgZd ZdS )RemBertTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zRemBertTokenizer.__init__   r   r   Nr   r   r   r   rg   rg      r   r   rg   c                       e Zd ZdgZd ZdS )SeamlessM4TTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zSeamlessM4TTokenizer.__init__   r   r   Nr   r   r   r   rj   rj      r   r   rj   c                       e Zd ZdgZd ZdS )SiglipTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zSiglipTokenizer.__init__   r   r   Nr   r   r   r   rm   rm      r   r   rm   c                       e Zd ZdgZd ZdS )Speech2TextTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zSpeech2TextTokenizer.__init__   r   r   Nr   r   r   r   rp   rp      r   r   rp   c                       e Zd ZdgZd ZdS )SpeechT5Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zSpeechT5Tokenizer.__init__   r   r   Nr   r   r   r   rs   rs      r   r   rs   c                       e Zd ZdgZd ZdS )T5Tokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zT5Tokenizer.__init__   r   r   Nr   r   r   r   rv   rv      r   r   rv   c                       e Zd ZdgZd ZdS )UdopTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zUdopTokenizer.__init__   r   r   Nr   r   r   r   ry   ry      r   r   ry   c                       e Zd ZdgZd ZdS )XGLMTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zXGLMTokenizer.__init__   r   r   Nr   r   r   r   r|   r|      r   r   r|   c                       e Zd ZdgZd ZdS )XLMRobertaTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zXLMRobertaTokenizer.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd ZdS )XLNetTokenizerr   c                 (    t          | dg           d S r	   r
   r   s      r   r   zXLNetTokenizer.__init__   r   r   Nr   r   r   r   r   r      r   r   r   N)'utilsr   r   r   r   r   r"   r%   r(   r+   r.   r1   r4   r7   r:   r=   r@   rC   rF   rI   rL   rO   rR   rU   rX   r[   r^   ra   rd   rg   rj   rm   rp   rs   rv   ry   r|   r   r   r   r   r   <module>r      s   2 2 2 2 2 2 2 23 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3; 3 3 3 33 3 3 3 3; 3 3 3 33 3 3 3 3[ 3 3 3 33 3 3 3 3; 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3{ 3 3 3 33 3 3 3 3k 3 3 3 33 3 3 3 3{ 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3; 3 3 3 33 3 3 3 3{ 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3{ 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3{ 3 3 3 33 3 3 3 3[ 3 3 3 33 3 3 3 3k 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3+ 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3[ 3 3 3 33 3 3 3 3 3 3 3 33 3 3 3 3[ 3 3 3 33 3 3 3 3+ 3 3 3 33 3 3 3 3K 3 3 3 33 3 3 3 3k 3 3 3 33 3 3 3 3k 3 3 3 33 3 3 3 3K 3 3 3 33 3 3 3 3{ 3 3 3 3 3 3r   