gwdZddlZddlZddlZddlmZddlmZmZm Z m Z ddl Z ddl ZddlmZmZddlmZdd lmZmZd d d d dZddiZdZejeZdZidddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4id5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVidWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxidydzd{d|d}d~ddddddddddddddddddddddddddddidddddddddddddddddddddddddddddddddddddddddddddddddd̜Zid̈́eDdd)d+dddddEdEddddМ Z ddgZ!GdӄdeZ"dՄZ#ddքZ$dׄZ% ddee&de'de'de'fdބZ(dee&fd߄Z)dee&fdZ*dZ+dS)z!Tokenization classes for Whisper.N) lru_cache)ListOptionalTupleUnion) AddedTokenPreTrainedTokenizer)logging)BasicTextNormalizerEnglishTextNormalizerz vocab.jsonztokenizer.jsonz merges.txtznormalizer.json) vocab_filetokenizer_file merges_filenormalizer_filezopenai/whisper-baseic \tttdtddztttdtddzztttdtddzz}|dd}d }td D]8}||vr2|||d |z|dz }9d |D}t t ||S) a8 Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control characters the bpe code barfs on. The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 bytes and unicode strings. !~r ¡¬®ÿNrc,g|]}t|S)chr).0ns l/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/whisper/tokenization_whisper.py z$bytes_to_unicode..As   Q#a&&   )listrangeordappenddictzip)bscsrbs r bytes_to_unicoder,-s U3s88SXX\ * *++d5TCIIPQM3R3R.S.SSVZ[`adeiajajloptluluxyly[z[zV{V{{ AAAB A 4[[ B;; IIaLLL IIdQh    FA  "   B B  r"c~t}|d}|ddD]}|||f|}|S)z Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). rr N)setadd)wordpairs prev_charchars r get_pairsr4IsP EEEQIQRR 9d#$$$ Lr"enenglishzhchinesedegermanesspanishrurussiankokoreanfrfrenchjajapanesept portuguesetrturkishplpolishcacatalannldutchararabicsvswedishititalianid indonesianhihindififinnishvi vietnamesehehebrewuk ukrainianelgreekmsmalayr*czechroromaniandadanishhu hungariantatamilno norwegianththaiururduhrcroatianbg bulgarianlt lithuanianlalatinmimaoriml malayalamcywelshskslovaktetelugufapersianlvlatvianbnbengalisrserbianaz azerbaijanisl slovenianknkannadaetestonianmk macedonianbrbretoneubasqueis icelandichyarmeniannenepalimn mongolianr)bosniankkkazakhsqalbanianswswahiliglgalicianmrmarathipapunjabisisinhalakmkhmersnshonayoyorubasosomaliaf afrikaansococcitankageorgianbe belarusiantgtajiksdsindhigugujaratiamamharicyiyiddishlolaouzuzbekfofaroesehtzhaitian creolepspashtotkturkmennnnynorskmtmaltesesanskrit luxembourgishmyanmartibetantagalogmalagasyassamesetatarhawaiianlingalahausabashkirjavanese sundanese cantonese)salbmybotlmgastthawlnhabajwsuyueci|]\}}|| Srr)rcodelanguages r rs>>>.$x>>>r"rr) burmese valencianflemishhaitian letzeburgeschpushtopanjabi moldavianmoldovan sinhalese castilianmandarin translate transcribeceZdZdZeZddgZ d>fd Zed e fd Z d Z d Z d?d e de defdZed ee fdZd@d ee fdZ dAdee deee ded ee ffd ZdZdZdZdZdBdZdZedBdZdCd e fdZdDd ZedDd!Z dBd"efd#Z!d$Z" dEd"ed%ed&ed'e#d(ed)ed*ed+ed e ffd, Z$ dFd-e%e ee fd"ed)ed*ed+ed e f d.Z&d/Z'd@d0e d1ee d e(e fd2Z)dBd3Z*dGd5Z+d6Z,dHd8e fd9Z-d-ee d:e d;e fd<Z.ed=Z/xZ0S)IWhisperTokenizeraa Construct a Whisper tokenizer. This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to the superclass for more information regarding such methods. Args: vocab_file (`str`): Path to the vocabulary file. merges_file (`str`): Path to the merges file. normalizer_file (`str`, *optional*): Path to the normalizer_file file. errors (`str`, *optional*, defaults to `"replace"`): Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information. unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The beginning of sequence token. The `decoder_start_token_id` is used to set the first token as `"<|startoftranscript|>"` when generating. eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The end of sequence token. pad_token (`str`, *optional*): The token used for padding, for example when batching sequences of different lengths. add_prefix_space (`bool`, *optional*, defaults to `False`): Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. language (`str`, *optional*): The language of the transcription text. The corresponding language id token is appended to the start of the sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token `"<|es|>"` is appended to the start of sequence. This should be used for multilingual fine-tuning only. task (`str`, *optional*): Task identifier to append at the start of sequence (if any). This should be used for mulitlingual fine-tuning, with `"transcribe"` for speech recognition and `"translate"` for speech translation. predict_timestamps (`bool`, *optional*, defaults to `False`): Whether to omit the `<|notimestamps|>` token at the start of the sequence. input_idsattention_maskNreplace <|endoftext|>Fc zt|trt|ddddn|}t|trt|ddddn|}t|trt|ddddn|}t|trt|ddddn|}t|d5}t j||_dddn #1swxYwYd|jD|_||_ t|_ d|j D|_ t|d5}| dd d }dddn #1swxYwYd |D}tt!|t#t%||_i|_| |_|Ct|d5}t j||_dddn #1swxYwYnd|_t/jd |_t/jd |_| |_t9jd|||||| d| | |_| |_dS)NFT)lstriprstrip normalizedspecialutf-8encodingci|]\}}|| Srrrkvs r rz-WhisperTokenizer.__init__..%s>>>A1>>>r"ci|]\}}|| Srrrs r rz-WhisperTokenizer.__init__..(sHHHdaQHHHr" r cPg|]#}t|$Sr)tuplesplit)rmerges r r!z-WhisperTokenizer.__init__..+s(CCCueEKKMM**CCCr"zJ's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+z<\|(\d+\.\d+)\|>)errors unk_token bos_token eos_token pad_tokenadd_prefix_spacer) isinstancestrr openjsonloadencoderitemsdecoderr%r, byte_encoder byte_decoderreadr#r'r(r$len bpe_rankscacher*english_spelling_normalizerrecompilepat timestamp_patrsuper__init__taskpredict_timestamps)selfrrrr%r&r'r(r)r*rr@rAkwargs vocab_handle merges_handle bpe_merges __class__s r r?zWhisperTokenizer.__init__s$)S)) JyuX\ ] ] ] ] )S)) JyuX\ ] ] ] ] )S)) JyuX\ ] ] ] ] )S)) JyuX\ ] ] ] ]  *w / / / 3<9\22DL 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3>>););)=)=>>>  ,..HHd.?.E.E.G.GHHH + 0 0 0 @M&++--33D99!B$?J @ @ @ @ @ @ @ @ @ @ @ @ @ @ @CC CCC c*eC OO.D.DEEFF 0  &o888 KL379\3J3J0 K K K K K K K K K K K K K K K04D ,:oppZ(;<<   -       "4s6?C%%C),C)+0F''F+.F+IIIreturnc*t|jSN)r6r0)rBs r vocab_sizezWhisperTokenizer.vocab_sizeHs4<   r"c|fdtjD}|j|S)Nc<i|]}||Sr)convert_ids_to_tokens)rirBs r rz.WhisperTokenizer.get_vocab..Ms)RRRa++A..RRRr")r$rKupdateadded_tokens_encoder)rBvocabs` r get_vocabzWhisperTokenizer.get_vocabLs@RRRR5;Q;QRRR T./// r"cf|jvr j|St|}t|}|s|S t|fd}|jvrn8|\}}g}d}|t |kr |||} |||| | }n-#t$r |||dYnwxYw|||krC|t |dz kr-||dz|kr| ||z|dz }n | |||dz }|t |kt|}|}t |dkrnt|}Wd |}|j|<|S)NTcTj|tdS)Ninf)r7getfloat)pairrBs r z&WhisperTokenizer.bpe..\s 1C1CD%PU,,1W1Wr"keyrr  ) r8r"r4minr7r6indexextend ValueErrorr&join) rBtokenr0r1bigramfirstsecondnew_wordrOjs ` r bpezWhisperTokenizer.bpeRs DJ  :e$ $U||$ L ($W$W$W$WXXXFT^++"ME6HAc$ii-- 5!,,A OOD1I...AA "OODH---E7e##CIIM(9(9d1q5kV>S>SOOEFN333FAAOODG,,,FAc$ii-- XHD4yyA~~!$9 (:xx~~  5 s7B--'CCrr@rAcf||n|j|_||n|j|_||n|j|_dS)a Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to update the prefix tokens as required when fine-tuning. Example: ```python >>> # instantiate the tokenizer and set the prefix token to Spanish >>> tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish") >>> # now switch the prefix token from Spanish to French >>> tokenizer.set_prefix_tokens(language="french") ``` Args: language (`str`, *optional*, defaults to `None`): The language of the transcription text. task (`str`, *optional*, defaults to `None`): Task identifier to append at the start of sequence (if any). predict_timestamps (`bool`, *optional*, defaults to `None`): Whether to omit the `<|notimestamps|>` token at the start of the sequence. N)rr@rA)rBrr@rAs r set_prefix_tokensz"WhisperTokenizer.set_prefix_tokens|sE(%-$8dm  ,DD$) 8J8V"4"4\`\sr"c 6|d}|d}|d}|d}tt}|j|j|_|jt vrt |j}n|jt vr|j}nt|jdk}td|jd|r&tt n%tt d|j -|j tvrtd |j d t|g}|j.| |d z||z|j "| |j d kr|n||js| ||S) N<|startoftranscript|>z <|translate|>z<|transcribe|><|notimestamps|>r]zUnsupported language: z. Language should be one of: .zUnsupported task: z. Task should be in: r r )convert_tokens_to_idsr" LANGUAGESkeysrlowerTO_LANGUAGE_CODEvaluesr6rbr#r@TASK_IDSr&r`rA) rB bos_token_idtranslate_token_idtranscribe_token_idnotimestamps_token_idlangs language_idis_language_code bos_sequences r prefix_tokenszWhisperTokenizer.prefix_tokenss112IJJ !77HH"889IJJ $ : :;M N Ninn&&'' = $ M//11DM} 000.t}= "2"9"9";";;;"m #&t}#5#5#:  rT]rr;Kn-4466777QUVfVkVkVmVmQnQnrrr 9 y(( !`di!`!`V^!`!`aaa$~ = $    q 05;;{3K3K K L L L 9   tyL7P7P 3 3Vh i i i& 7    5 6 6 6r"cX||j|z|jgzS|j|z|z|jgzS)z=Build model inputs from a sequence by appending eos_token_id.)r eos_token_id)rB token_ids_0 token_ids_1s r build_inputs_with_special_tokensz1WhisperTokenizer.build_inputs_with_special_tokenss?  % 3t7H6II I!K/+=AR@SSSr"rralready_has_special_tokensc|r$t||dSdgt|jz}dg}||dgt|zz|zS|dgt|zzdgt|zz|zS)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rrrr Nr)r>get_special_tokens_maskr6r)rBrrr prefix_ones suffix_onesrGs r rz(WhisperTokenizer.get_special_tokens_masks& & 7722'[]a3 cC 2333 c  1#K(8(8"89KG GqcC $4$445!s;?O?O9OPS^^^r"c8g}tjj|D]{}dfd|dD}|d|dD||S)zTokenize a string.c32K|]}j|VdSrJ)r3)rr+rBs r z-WhisperTokenizer._tokenize..s<)*!!$r"rc3K|]}|VdSrJr)r bpe_tokens r rz-WhisperTokenizer._tokenize..s"TTIiTTTTTTr"r^)r:findallr<rcencoderarjr#)rBtext bpe_tokensrds` r _tokenizezWhisperTokenizer._tokenizes Z$// U UEGG.3ll7.C.CE   TT%9N9Ns9S9STTT T T T Tr"cr|j||j|jS)z0Converts a token (str) in an id using the vocab.)r0rWr&)rBrds r _convert_token_to_idz%WhisperTokenizer._convert_token_to_ids,|t|'7'7'G'GHHHr"c8|j|dS)z Converts an index (integer) in a token (str) using the vocab. Whisper's base tokenizer always decodes OOV tokens as "", thus we do not use the `unk_token` here. r)r2rW)rBr`s r _convert_id_to_tokenz%WhisperTokenizer._convert_id_to_tokens |r***r"cTtjd||S)NzThe private method `_normalize` is deprecated and will be removed in v5 of Transformers.You can normalize an input string using the Whisper English normalizer using the `normalize` method.)warningswarn normalize)rBrs r _normalizezWhisperTokenizer._normalizes0  s   ~~d###r"cXtjd|||S)NzThe private method `_basic_normalize` is deprecated and will be removed in v5 of Transformers.You can normalize an input string using the Whisper basic normalizer using the `basic_normalize` method.remove_diacritics)rrbasic_normalize)rBrrs r _basic_normalizez!WhisperTokenizer._basic_normalizes8  w   ##D". r r z<|z.2fz|>clg|]0}t|tr|n|1S)skip_special_tokens)r+r,decode)rsrBrs r r!z.,sJ   efAs## `AAQL_)`)`   r"r)all_special_idsrXr&rc) rB token_idsrtime_precisiontimestamp_beginoutputscur_max_timestampprev_segments_lenrd timestamps ` ` r _decode_with_timestampsz(WhisperTokenizer._decode_with_timestampss .r2Q6$ * *E''!5?#:n"LMM 000%)::%$-!KY1B%BKKKKLLLr"""" ""5))))     jq   wwwr"cg}dtt|vr8t|dr(t|jr|}t j|}|jddkr't|jdkrtd|j ddz}||k}t j |dd|ddzddz}|jddkr| dkrgSt j |dddz|vr6t j |t j |dddz}t j |dd}d}d} |D]} ||| } t| dkr| d|z } | d|z } | |kr| |z } | }|| } || }||}| || | z|z| | z|zfd| }|S) a~ Compute offsets for a given tokenized input Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. time_precision (`float`, *optional*, defaults to 0.02): The time ratio to convert from token to time. torchcpurr z)Can only process a single input at a timer Nrr)r,typehasattrcallablernparrayshaper6rbrwheresumr&item_preprocess_token_ids_decode_filter_timestamp_ids)rBrroffsetsrtimestamp_tokens consecutive last_slicerr current_slice sliced_tokensstart_timestamp_positionend_timestamp_positionrs r _compute_offsetsz!WhisperTokenizer._compute_offsets1s c$y//** * * 50I0I *hW`WdNeNe *! IHY'' ?1  ! !c)/&:&:Q&>&>HII I.r2Q6$7h/47G7KKLLQORSS  Q 1 $ $)9)=)=)?)?1)D)DI X& ' ' *2 . 2+ E E)K:J1K1KA1Nr1RUV1VWWKX.//215 ( ' 'M%j&>?M=!!A%%+8+;+@+@+B+B_+T()6r):)?)?)A)AO)S&+.???%)::%$:!!% : := I I ||M2211$77 $58II^[36GG>Y&'JJr"c`|fdtdDS)a Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache. Args: time_precision (`float`, *optional*, defaults to 0.02): The time ratio to convert from token to time. c g|] }d|zz S)z<|%.2f|>r)rrOrs r r!z2WhisperTokenizer.timestamp_ids..vs$*h*h*hSTJ!n:L,M*h*h*hr"i)rqr$)rBrs `r timestamp_idszWhisperTokenizer.timestamp_idsms8))*h*h*h*hX]^fXgXg*h*h*hiiir"rc|rA|d}|d}||||}|S)a Pre-process the token ids for decoding by removing the prompt tokens ids and timestamp token ids. Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Typically, obtained using the `__call__` method of the tokenizer. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens from the token ids. If `True`, the prompt token ids will be removed. <|startofprev|>rn)rq _strip_prompt)rBrrprompt_token_iddecoder_start_token_ids r rz&WhisperTokenizer._preprocess_token_idsxsT  _"889JKKO%)%?%?@W%X%X "**9oG]^^Ir"c8tj|jd|S)Nr)r:subr=)rBrs r rz&WhisperTokenizer._filter_timestamp_idssvd("i888r"clean_up_tokenization_spacesoutput_offsetsrdecode_with_timestampsrrrc |||} tj| f||||| d| } |r|| ||} n|| } |r|||} | | dS| S)a Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Will remove the previous tokens (pre-prompt) if present. clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. If `None`, will default to `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). output_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output the offsets of the tokens. This should only be set if the model predicted timestamps. If there are previous tokens (pre-prompt) to decode, they will only appear in the decoded text if they contain timestamp tokens. time_precision (`float`, *optional*, defaults to 0.02): The time ratio to convert from token to time. decode_with_timestamps (`bool`, *optional*, defaults to `False`): Whether or not to decode with timestamps included in the raw text. normalize (`bool`, *optional*, defaults to `False`): Whether or not to apply the English text normalizer to the decoded text. Only applicable when the target text is in English. Otherwise, the basic text normalizer should be applied. basic_normalize (`bool`, *optional*, defaults to `False`): Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual target text. remove_diacritics (`bool`, *optional*, defaults to `False`): Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may destroy information in the decoded text, hence it should be used with caution. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. Returns: `str`: The decoded sentence. r)rrrrr)rr)r)rr)rr>rrrr)rBrrrrrrrrrrC filtered_idsrrrGs r rzWhisperTokenizer.decodesd11  32  uww~  3)E+/      " 4//^Qd0DD--d33D  6++In+UUG W55 5 r"rc :|dd|_|||}g}g} |D]n} |r | |jvr| |jvrB| r*||| g} || Y| | o| r(||| d|} |r|| } | S|r| | |} | S| S)Nuse_source_tokenizerFrrr) pop_decode_use_source_tokenizerrNrrQr&convert_tokens_to_stringrcrr) rBrrrrrrCfiltered_tokens sub_textscurrent_sub_textrdr clean_texts r rzWhisperTokenizer._decodes]-3JJ7Mu,U,U)44YTg4hh  $ / /E" u0D'D'D111#*$$T%B%BCS%T%TUUU')$  '''' ''....  N   T::;KLL M M Mwwy!!  --J   --dFW-XXJ Kr"cd|}tfd|Ddj}|S)z:Converts a sequence of tokens (string) in a single string.rc*g|]}j|Sr)r4)rcrBs r r!z=WhisperTokenizer.convert_tokens_to_string..s!===1$+A.===r"r)r%)rc bytearrayrr%)rBtokensrs` r rz)WhisperTokenizer.convert_tokens_to_stringsPwwv=======>>EEgVZVaEbb r"save_directoryfilename_prefixc tj|s td|ddStj||r|dzndt dz}tj||r|dzndt dz}tj||r|dzndt dz}t|dd 5}|tj |j d d d dzdddn #1swxYwYd}t|dd 5}|dt|j dD][\} } || kr td|d| }|d| dz|dz }\ dddn #1swxYwY|j]t|dd 5}|tj |jd d d dzdddn #1swxYwY|||fS)NzVocabulary path (z) should be a directory-rrrrwrrr]TF)indent sort_keys ensure_asciirrz#version: 0.2 c|dS)Nr r)kvs r rZz2WhisperTokenizer.save_vocabulary..s Y[\]Y^r"r[zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!r^r )ospathisdirloggererrorrcVOCAB_FILES_NAMESr-writer.dumpsr0sortedr7r1warningr9) rBrrr merge_filerfr`writerr token_indexs r save_vocabularyz WhisperTokenizer.save_vocabulary sfw}}^,,  LLT^TTT U U U FW\\ oM_s222QbcoQpp  W\\ oM_s222QbcpQqq  ',, oM_s222QbctQuu  *cG 4 4 4 c GGDJt|ATYZZZ]aa b b b c c c c c c c c c c c c c c c *cG 4 4 4  LL* + + ++1$.2F2F2H2HN^N^+_+_+_  ' KK''NNM MMM(E SXXj11D8999                   + 7osW=== Jt?UYhmnnnquu                :66s714D11D58D5BG88G<?G<4II #I c R|d|j}|s|rd|z}||fS)Nr*r^)rr*)rBris_split_into_wordsrCr*s r prepare_for_tokenizationz)WhisperTokenizer.prepare_for_tokenization2s=!::&8$:OPP  "2 :Df~r"Tc|||| |jdd}dt|D}|S)N)r@rrAr c"g|] \}}|dz|f S)r r)rrankrds r r!z;WhisperTokenizer.get_decoder_prompt_ids..?s%\\\KD%tax/\\\r")rlr enumerate)rBr@r no_timestamps forced_tokensforced_decoder_idss r get_decoder_prompt_idsz'WhisperTokenizer.get_decoder_prompt_ids8sV D8TaPabbb *122. \\9]C[C[\\\!!r"c*t|||||S)N)return_timestampsreturn_languager) _decode_asr)rB model_outputsrrrs r rzWhisperTokenizer._decode_asrBs(  /+)     r"rrc2dd|zd}|ddd}tfd|Dd}|(|}td |d || |dS) z`Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`].rr^F)add_special_tokensrr Nc3>K|]}|jdk|VdS)rN)r)rxrBs r rz2WhisperTokenizer.get_prompt_ids..Qs6 \ \qqDDXYZD[?[?[?[?[?[?[ \ \r"zJEncountered text in the prompt corresponding to disallowed special token: rp) tensor_type)stripnextrNrbconvert_to_tensors)rBrreturn_tensorsbatch_encodingprompt_text_idsspecial_token_idrds` r get_prompt_idszWhisperTokenizer.get_prompt_idsKs/tzz||1CX]^^^)5abb9 \ \ \ \O \ \ \^bcc  '../?@@Erjorrrss s))n)EEEk**r"rrct|ts||}|s|S|d|k}|r#||vr|||dSgS|S)Nr)r+r#_convert_to_listr`)rBrrr has_prompts r rzWhisperTokenizer._strip_promptYs)T** 9--i88I  q\_4  %22 1G!H!H!J!JKK r"ct|drxdtt|vr'|}nedtt|vr|}n2dtt|vr|}t |tjr|}|S)Nnumpyr tensorflowjaxlib) rr,rrr(tolistr+rndarray)rs r r%z!WhisperTokenizer._convert_to_listks 9g & & +#d9oo....%MMOO1133 T)__!5!555%OO-- T)__-- - -!((**I i , , +!((**Ir") NrrrrNFNNF)NNNrJ)NF)F)Fr)r)FNFrFFFF)FFFF)NNT)r)1__name__ __module__ __qualname____doc__rvocab_files_namesmodel_input_namesr?propertyintrKrSrjr,boolrlrrrrrrrrrrr staticmethodrrrrrrrrXrrrrrrr rrr#rr% __classcell__)rGs@r r r s&&P*$&67 !!!  H5H5H5H5H5H5T!C!!!X! (((Ttt#tCt\`tttt0tCyXDTTQUVYQZTTTTsx__9_3;DI3F_ko_ c______>III+++$$$OOOO      \   dg    <::::xjjjYjD$999 %*-1$ $', %"'LL"L'+ L  L  L!%LLL L LLLLLLb%* %"' ''d3i('"' '  '  ' ''''T $7$7c$7HSM$7]bcf]g$7$7$7$7N """"    + +3 + + + +tCy3`c$  \     r"r c b )d))fd}g}|}d}|ddz} g} g} d} d} t|j}|d}|d}t|D]\}}|d d }||||}|d kr |d d }d}| }d |vrL|d \}}}||z}||z } |r||z | z}|r*t |D]}|| kr||| z |z| krn|}g}g}t|D]Q\}}||vr||g}|dd}t |d}|s)ri|)krc|sa| |t| } || }!|!|d<| |g} g}|}||d<|)|| kr|| z |z|z}"t|"d}"|r ||krd} | s| r ||krd} |dd  |"|dd <|"|dd kr$|"|dd<| ||d kr| |t| | \} }#|| }!|!|d<|d krt|| |#)||d<| |g} g}g} g}|}| ||d kret|||zd}$|dzt|krt||dz|zd}%nd}%| |$|%fSd |vr|||z z }|r2| ||d kr| |]td| Ds|}g} g}g} g}| rz|rt dt| | \} }#|| }!|!|d<|d krt|| |#)||d<| |dd|D}&|s|r~|D]I}|s|dnt)|d|d<|s|dJ|d kr'g}'|D]}|'|dd|'i}(nd|i}(ni}(|&|(fS)z Internal method meant to only be used by asr pipeline. Handles all the little quirks specific to whisper to handle the various options not allowed in other seq2seq models NcddgddS)Nr)rrrr) last_languagesr new_chunkz_decode_asr..new_chunks)t bQQQr"rror Frrnrrr0token_timestampsstrider]rrTrwordsc3K|]}|VdSrJr)rps r rz_decode_asr..0s"11Aa111111r"zWhisper did not predict an ending timestamp, which can happen if audio is cut off in the middle of a word. Also make sure WhisperTimeStampLogitsProcessor was used during generation.rc3&K|] }|dV dS)rNr)rchunks r rz_decode_asr..Js&::%f ::::::r"chunks)rqr.rr r+rreversedrrrrWr&_find_longest_common_sequenceround_collate_word_timestampsr6anyrrrcrr"ra)* tokenizerrrrrr;rDrC time_offsetrprevious_tokensprevious_token_timestampsskipright_stride_startrrrchunk_idoutputrr<last_timestampfirst_timestamp chunk_len stride_left stride_rightrdcurrent_tokenscurrent_token_timestampsrOrrresolved_tokens resolved_texttimeresolved_token_timestamps start_timeend_time full_text new_chunksoptionalr:s* @r rr{s#$MRRRRRF IKKEK556HIIAMOO " D)344O556GHHO&<<=TUU%m44U*U*&8$Q'..00 ++IH^__  & &%&89!<CCEE  ) v  39(3C 0I{L ; &K!*\!9   Q"-">"P /%i00 / /E//+6!&!8N JM_ _ _!E).#% "),,[ L[ LHAu '' ''00AbDz$==t44'% ,])B)BK\)B'..~>>>*G*X*X(1(8(8(I(I (5f  e,,,+-)+ ) (0E*%$,MM/))/>AKOT1~~!+,e~&=&=  DD$,o$,%/2I2I DD;'*2,0E+&q))u[1!44404k*1-'..~>>>,665<<=UVVVEb+-FFFB)B)2(8(8(I(I (5f ,66-E )? 2 2 2 F**)001IJJJ1111111 *IKKE ON(* %') $   NN]    6S 66 6 22"((99 %f  & &5?,E}VeE'N  e::6:::::IO & &E$ ? +&&&&%*5+=%>%>k"" & *%%%  & &J 2 2!!%.1111 *-HH &)HH h r"c |d}t|}g}r dg}t|ddD]\}d}||ddf}t|} td|| zD]} | dz } td|| z t ||| z| z } t j|| } td| |z t | | }t j||t| tkrtdr.tfdt| D}nt j| k}|| z | z}|dkr||kr|}| |f}!|\} }| zdz}|zdz}| |d|||d}t|}r0| d|dz|d| ||Stdkr| ||fS|gfS)Nrr rg@ziThere is a bug within whisper `decode_asr` function, please report it. Dropping to prevent bad inference.c3|K|]6\}}||kr%|zdz|zk2dV7dS)r Nr) ridxelem left_startleft_token_timestamp_sequenceright right_startseq_idxtoken_timestamp_sequencess r rz0_find_longest_common_sequence..s~!Tc **9*s:JK4Wq[A+PSBSTUU UUUU r"r]) r6r r$maxr_rr RuntimeErrorrra) sequencesrk left_sequence left_lengthtotal_sequencetotal_token_timestamp_sequenceright_sequencemax_ max_indices right_lengthrOeps left_stopleft right_stopmatchesmatchingleft_mid right_midrfrgrhrirjs ` @@@@@r rFrF`saLMm$$KN ,(A!(D%)+&#,Yqrr]#;#;[_[_"KA6 <>** q+ 455( O( OAg+C Q a00JK|)Ca)GHHI8M*Y*>?@@Da[11K\1--JH^K ,BCDDE4yyCJJ&&") 0%.t__&//{S(H{{x$)9k:N ;F8Y Z *q0+-!3 mIXI6777&yzz2 -(( $ _ * 1 12OPYQYPY2Z [ [ [,EgPQk,RS\S]S],^ )-((( ( $%%))&--.KLLL===r!!r"c~ t|||\}}}|rd|ini fdt||D}|S)Nrcpg|]2\}}||dd|ddfd3S)rr r rr)rr0indicesoptional_language_fieldr<s r r!z,_collate_word_timestamps..sj D' *71:6q9;KGTVK;XYZ;[\  & r")_combine_tokens_into_wordsr() rJrr<rrr?_ token_indicestimingsrs ` @r rHrHsq8FHUUE1m8GOz844R ! 66 G Nr" "'“¡¿([{-"'.。,，!！?？:：”)]}、rrprepend_punctuationsappend_punctuationsc||j}|d}|dvrt||\}}}nt||\}}}t||||||||fS)z Groups tokens by word. Returns a tuple containing a list of strings with the words, and a list of `token_id` sequences with the tokens making up each word. Nr6>rrqr8rrDr)r_split_tokens_on_unicode_split_tokens_on_spaces_merge_punctuations)rJrrrrr? word_tokensrs r rrs}%QQQ,DYPV,W,W){MM,CIv,V,V){M{M;OQdeee +} ,,r"c||d}d}g}g}g}g}g}d} t|D]\} } || || ||d} || vs"|| | |z|krU|| ||||g}g}| t | z } |||fS)zlCombine tokens into words by splitting at any position where the tokens are decoded as valid unicode points.T)ru�r)rr r&r`r6) rJr decoded_fullreplacement_charr?rrrWcurrent_indicesunicode_offset token_idxrddecodeds r rrs,##F4#HHL EKMNON%f--++ 5e$$$y)))"">$"OO G + +NW]];K-L-LLMQaaa LL ! ! !   ~ . . .   1 1 1N O c'll *N +} ,,r"c"t||\}}}g}g}g}t|||D]\}} } | d|jk} |d} |dv} | s| s| st |dkr@|||| || |d|z|d<|d| |d| |||fS)zLCombine tokens into words by splitting at whitespace and punctuation tokens.rr^z !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~r )rr(r startswithrr6r&ra)rJrsubwordssubword_tokens_listsubword_indices_listr?rrsubwordsubword_tokenssubword_indicesr with_space punctuations r rr!s;:RS\^d:e:e7H!#7 EKM47BUWk4l4l 6 60 #y'=='',, mmoo)MM  6j 6K 63u::?? LL ! ! !   ~ . . .   1 1 1 1b G+E"I O " "> 2 2 2 "  $ $_ 5 5 5 5 +} ,,r"c0t|dz }t|dz }|dkr||drh|||vrL||||z||<||||z||<||||z||<d||<g||<g||<n|}|dz}|dkd}d}|t|kr||ds\|||vrR||xx||z cc<||xx||z cc<||xx||z cc<d||<g||<g||<n|}|dz }|t|kd|D|dd<d|D|dd<d |D|dd<dS) z1Merges punctuation tokens with neighboring words.r]r rr^rcg|]}||Srr)rr0s r r!z'_merge_punctuations..Zs///$////r"Ncg|]}||Srr)rrds r r!z'_merge_punctuations..[s4445e4444r"cg|]}||Srr)rrds r r!z'_merge_punctuations..\s000#C0#000r")r6rrendswith)r?rr prependedappendedrOris r rr9s E QA E QA q&& 8  s # # a(8(8I(E(EQx%(*E!Hq F1I-F1I gaj0GAJE!HF1IGAJJA Q q&& A A c%jj..Qx  %% %(h*>*> !HHHa HHH 1III "III AJJJ'!* $JJJE!HF1IGAJJA Q c%jj..0////E!!!H44F444F111I00000GAAAJJJr"rJ)Nrr),r0r.rr functoolsrtypingrrrrr(rregexr:tokenization_utilsr r utilsr english_normalizerr rrMAX_MODEL_INPUT_SIZESr, get_loggerr-rr4rrr1rurwr rrFrHr4r,rrrrrr"r rs(' ////////////AAAAAAAAJJJJJJJJ&( 3 0  H % %   e )e )e  (e  ) e   ) e   ( e  (e  *e  ,e  )e  (e  )e  'e  (e  )e   )!e " ,#e e $ '%e & )'e ( ,)e * (+e , +-e . '/e 0 '1e 2 '3e 4 *5e 6 (7e 8 +9e : ';e < +=e > &?e @ &Ae B *Ce D +Ee e e F ,Ge H 'Ie J 'Ke L +Me N 'Oe P (Qe R (Se T )Ue V )We X )Ye Z )[e \ -]e ^ +_e ` )ae b *ce d ,ee f (ge e e h (ie j +ke l *me n (oe p +qe r )se t (ue v *we x )ye z *{e | )}e ~ )e @ )Ae B 'Ce D 'Ee F (Ge H (Ie e e J +Ke L )Me N *Oe P ,Qe R 'Se T (Ue V *We X )Ye Z )[e \ %]e ^ '_e ` )ae b ce d (ee f )ge h )ie j )ke e l               Ie e e  P>>IOO,=,=>>>  &f f f f f *f f f RbbbJu"u"u"u"p   & 0A -- I-- -  ----4-S ---->-tCy----0#1#1#1#1#1r"