gdZddlZddlmZmZddlmZddlmZej e Z ddiZ d Z Gd d eZdS) zTokenization classes for ESM.N)ListOptional)PreTrainedTokenizer)logging vocab_file vocab.txtct|d5}|}d|DcdddS#1swxYwYdS)Nrc6g|]}|S)strip).0ls d/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/esm/tokenization_esm.py z#load_vocab_file.. s )))a ))))openread splitlines)rfliness rload_vocab_filers j#  *!##%%))5)))******************s2AAAc $eZdZdZeZddgZ dfd Zd ed e fd Z d e d efdZ dZ dZ d e d efdZd ed e fdZ ddeedeeed eefdZ ddedeeded eefdZdZed efdZxZS) EsmTokenizerz& Constructs an ESM tokenizer. input_idsattention_maskc Lt||_tt|j|_dt|jD|_t jd|||||d||j|_| |jdS)Nci|]\}}|| Sr r )rindtoks r z)EsmTokenizer.__init__..7sQQQ(#sS#QQQr) unk_token cls_token pad_token mask_token eos_tokenr ) r all_tokensdict enumerate _id_to_token _token_to_idsuper__init__unique_no_split_tokens _update_trie) selfrr(r)r*r+r,kwargs __class__s rr3zEsmTokenizer.__init__+s**55 4?!;!;<<QQi6P6PQQQ !       '+o# $566666rindexreturncB|j||jSNr0getr(r6r9s r_convert_id_to_tokenz!EsmTokenizer._convert_id_to_tokenG $$UDN;;;rtokencr|j||j|jSr<r1r>r(r6rBs r_convert_token_to_idz!EsmTokenizer._convert_token_to_idJ. $$UD,=,A,A$.,Q,QRRRrc *|Sr<)split)r6textr7s r _tokenizezEsmTokenizer._tokenizeMszz||rcl|j}||j|Sr<)r1copyupdateadded_tokens_encoder)r6 base_vocabs r get_vocabzEsmTokenizer.get_vocabPs3&++-- $3444rcr|j||j|jSr<rDrEs r token_to_idzEsmTokenizer.token_to_idUrGrcB|j||jSr<r=r?s r id_to_tokenzEsmTokenizer.id_to_tokenXrArN token_ids_0 token_ids_1c|jg}|jg}||j||zS||z|zS|jtd||z|z|z|zS)Nz=Cannot tokenize multiple sequences when EOS token is not set!) cls_token_id eos_token_id ValueError)r6rVrWclsseps r build_inputs_with_special_tokensz-EsmTokenizer.build_inputs_with_special_tokens[sv ! !   ([(([(3..   &\]] ][ 3&4s::rFalready_has_special_tokensc|r|tdfd|DSdgdgt|zzdgz}||dgt|zdgzz }|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`List[int]`): List of ids of the first sequence. token_ids_1 (`List[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.c*g|]}|jvrdndS)r)all_special_ids)rrBr6s rrz8EsmTokenizer.get_special_tokens_mask..s*WWW%$"666AAAWWWrrbr)r[len)r6rVrWr_masks` rget_special_tokens_maskz$EsmTokenizer.get_special_tokens_maskis$ & X& R XWWW;WWW WsqcC ,,,-3  " QC#k***aS0 0D rctj||r|dznddz}t|d5}|d|jdddn #1swxYwY|fS)N-r w )ospathjoinrwriter-)r6save_directoryfilename_prefixrrs rsave_vocabularyzEsmTokenizer.save_vocabularysW\\.O3c?S3H3Hacgr2rss *c " " 0a GGDIIdo.. / / / 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0}s.A55A9<A9c*t|jSr<)rdr-)r6s r vocab_sizezEsmTokenizer.vocab_sizes4?###r)rrr r!r"r<)NF)__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESvocab_files_namesmodel_input_namesr3intstrr@rFrKrQrSrUrrr^boolrfrrpropertyrt __classcell__)r8s@rrr#s*$&67 7777778<#<#<<<<S#S#SSSS SSSSSS<<<<<<JN ; ;9 ;3;DI3F ; c ; ; ; ;in.6tnae c> $C$$$X$$$$$rr)rxrltypingrrtokenization_utilsrutilsr get_loggerruloggerryrrr rrrs$# !!!!!!!!555555  H % %!;/*** m$m$m$m$m$&m$m$m$m$m$r