
    g]c                        d Z ddlmZ ddlZddlmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ  ej        e          Zddddddddddddddddddddddddddddddddddddddddddi ddddddddddd d!d"dd#d$d%dd&d'd(d)d*dd+dd,dd-dd.dd/dd0diddd1dd1ddd2dddd3d4d5d6d7d8ddd9d:dd;d<d=d>d7d8dd9d:d?d@dA	d=d>d7d8dBdCd9d:d?d@dD
dddddddddddddddEd6dFd;dGdHdIdJdKdLdMdNdddLdMdddddddddOdPZdQdRdSdTdUdVdWdXdYdZd[d\d]d^
dTdUdVdWdXdYdZd[d\d]d^
dTdUdVdWddYdZd[d\d]d^
dTdUdVdWddYdZd[d\d]d_d`dadTdUdVdWddYdZd[d\d]d^
dbdcdddedfdTdUdVdWddYdZd[d\d]d^
dgdWdhd]didjdTdUdVdWddZd[dkd]dl	dgdmdndodhdidpdUdTdWdVdZd[dqdrdsZdtdudvdwdxdbdcdddedydz
d{dQdbdcddded|d}Zd~ Z G d d          Z G d de          Z G d de          Z G d de          Z G d de          Z eeeee e e e e d	Z!defdZ"dS )z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers)BPE   
AddedToken)GPT2ConverterLlamaConverterQwen2Converter)logging)tqdmzmodel.embed_tokenszmodel.layerszmlp.up_projzmlp.down_projzmlp.gate_projpost_attention_layernorminput_layernormzself_attn.q_projzself_attn.v_projzself_attn.k_projzself_attn.o_projzlm_head.weightz
model.norm)
token_embdblkffn_upffn_downffn_gateffn_norm	attn_normattn_qattn_vattn_kattn_outputoutput.weightoutput_normr   r   ffn_up_expszmlp.expertsffn_up_shexpzmlp.shared_expert.up_projffn_down_expsffn_down_shexpzmlp.shared_expert.down_projr   zffn_gate_inp.weightzmlp.gate.weightffn_gate_expsffn_gate_shexpzmlp.shared_expert.gate_projffn_gate_inp_shexpzmlp.shared_expert_gater   r   r   r   r   r   r   zmlp.gate_up_projzself_attn.qkv_proj)r   r   r   r   r   r   r   attn_qkvr   r   r   z"transformer.word_embeddings.weightz%transformer.word_embeddings_layernormztransformer.hzmlp.dense_h_to_4hzmlp.dense_4h_to_hzself_attention.query_key_valuezself_attention.denseztransformer.ln_f)ztoken_embd.weighttoken_embd_normr   r   r   r   r   r'   r   r   r   word_embeddingshz	.lm_head.ln_f)	r   r   r   r   r   r'   r   .output.r   z.ln_mlp.ln_attn)
r   r   r   r   z.attn_norm.attn_norm_2r'   r   r,   r   ztransformer.wteztransformer.wpeln_1zattn.c_attnzattn.c_proj.weightzattn.c_proj.biasln_2zmlp.c_fcz
mlp.c_proj)r   r   position_embdr   r   r'   zattn_output.weightzattn_output.biasr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )llamamistralqwen2qwen2moephi3bloomfalcon7b	falcon40bstablelmgpt2
starcoder2
model_type_model_name_or_path)architecturenamemax_position_embeddingsnum_hidden_layersintermediate_sizehidden_sizehead_dim
rope_thetanum_attention_headsnum_key_value_headsrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonrJ   num_expertsnum_experts_per_tok)rK   rL   rM   rN   rO   rP   rQ   rR   rS   rJ   expert_countexpert_used_countbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idn_layern_headlayer_norm_epsilon)rL   rN   rQ   rJ   attention.layer_norm_epsilonlayer_norm_eps)	rK   rL   rM   rN   rO   rQ   rR   rc   rJ   n_ctxn_embdrM   )rL   rK   rN   rM   rQ   rc   norm_epsilon)rL   rK   rN   rM   rQ   rR   rc   )generalr2   r3   r4   r5   falcon	tokenizerr6   r7   r:   r;   r<   tokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr\   r]   r^   r_   zggml.add_space_prefixchat_template)rr   rq   r\   r]   r^   r_   )rj   tokenizer_configc                    t          |t                    s|g}t          |          dk    r|d         }d }n |d         dk    rt          d          |\  }}|dv rt	          | d                   } n|dv rt          | d                   } nt|dv rt          | d                   } nZ|dv rBt          d	t          |                                                     	                                } n|d
v rt          | |          } | S )N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   ru   r	            
      )      )   )   B)rv   )
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_value	data_typearray_data_types      Z/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/integrations/ggml.pyr   r   X  s   i&&  K	
9~~aL	Q<1oppp%."	?...VAY	g		vay!!	c		fQi	c		sDLL))1133::<<	c		"6?;;M    c                       e Zd Zd ZdS )GGUFTokenizerSkeletonc                 \   |                                 D ]\  }}t          | ||           t          | d          sBt          | d          rt          | d          st          d          | j        }| j        fdt          |          D             t                              d           g }t                                                     D ]\  }}g }t          dt          |                    D ]6}	|d |	         ||	d          }}
|
|v r||v r|                    |
||f           7t          |fdd	
          }|                    |           t          |d d	
          }d |D             }|| _        nVd | j        D             | _        t          | d          s0d t          t          | j                            D             | _        t          | d          sg | _        t          | d          sd | _        t          | d          r| j        | j        | _        d S d S d S )Nro   rl   rm   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                 (    i | ]\  }}||         S  r   ).0itrm   s      r   
<dictcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>|  s#    @@@daQq	@@@r   z:Merges were not in checkpoint, building merges on the fly.ru   c                 <    | d                  | d                  fS )Nr   ru   r   )xvocabs    r   <lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s    U1Q4[%!+4N r   T)keyreversec                     | d         S )Nr	   r   )vals    r   r   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s
    CF r   c                 .    g | ]}|d          |d         fS )r   ru   r   )r   r   s     r   
<listcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>  s%    9993s1vs1v&999r   c                 R    g | ]$}t          |                    d                     %S ) )tuplesplit)r   merges     r   r   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>  s,    LLLu5S!1!122LLLr   c                     g | ]}d S Nr   )r   _s     r   r   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>  s    EEEtEEEr   added_tokensrZ   unknown_token_id)itemssetattrhasattrr   rl   rm   	enumerateloggerwarningr   ranger   appendsortedextendro   r   rZ   r   )selfdict_kvrl   ro   r   piece_scorelocalindexpiece_lpiece_rrm   r   s               @@r   __init__zGGUFTokenizerSkeleton.__init__q  s   KKMM 	  	 DAqD!QtX&& 	F4** '$2I2I  r   [F[F@@@@i.?.?@@@ENNWXXXF&*5;;==&9&9 % %"{"1c%jj11 F FE',VeV}eEFFmWG&((W->->gw%DEEEu*N*N*N*NX\]]]e$$$$F(:(:DIIIF99&999F DKKLLLLLDK4** FEEU3t{3C3C-D-DEEEt^,, 	# "Dt^,, 	% $D 4+,, 	61B1J $ 5D	6 	61J1Jr   N)__name__
__module____qualname__r   r   r   r   r   r   p  s#        '6 '6 '6 '6 '6r   r   c                   2    e Zd Zd Zd Zd Zd Zd Zd ZdS )GGUFLlamaConverterc                     t          |          | _        | j        | _        i | _        t	          | j        dd          dk    | _        d S )Nrk   r2   )r   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dicts     r   r   zGGUFLlamaConverter.__init__  sF    *>::
"&*!#$+DJ8H'$R$RV]$]!!!r   c                 P    t          t          |j        |j                            S r   r   ziprl   rm   r   r   s     r   r   zGGUFLlamaConverter.vocab      Cel33444r   c                     |j         S r   ro   r   s     r   ro   zGGUFLlamaConverter.merges  
    |r   c           	      (   |                      | j                  }|                     | j                  }d t          |          D             }|j        |j        |j                 nd }t          |dd           |j        |j                 nd }t          |dd           |j        |j                 nd }t          t          |||dd                    }g }	t          | j        d          sv|%|	                    t          |dd                     |%|	                    t          |dd                     |%|	                    t          |dd                     ntt          j        t          j        | j        j                  d	k              d
         }
|
D ]7}|	                    t          | j        j        |         dd                     8t#          |	          d
k    r|                    |	           t#          | j        j                  d
k    r)|                    d | j        j        D                        || j        d<   || j        d<   || j        d<   | j        r*d | j        d<   d| j        d<   d| j        d<   d| j        _        |S )Nc                      i | ]\  }\  }}||S r   r   r   r   word_scores       r   r   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>  #    NNN!2NT6T1NNNr   rX   rY   T)	unk_tokenfuse_unkbyte_fallbackrn   F
normalizedspecialrw   r   c                 2    g | ]}t          |d d           S )Fr   r
   )r   added_tokens     r   r   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>  s'    uuukKE5IIIuuur   r   	eos_token	bos_tokenrp   clean_up_tokenization_spaceslegacy)r   r   ro   r   rZ   rl   r   rX   r   r   r   r   r   npwherer   rn   r   add_special_tokensr   
add_tokensr   r   r   r   )r   r   vocab_scoresro   	bpe_vocabr   r   r   rj   special_tokensspecial_tokens_idxidxs               r   rj   zGGUFLlamaConverter.tokenizer  s   zz$*--TZ((NNi6M6MNNN	8=8J8VEL!344\`	8?~W[8\8\8hEL!344nr	8?~W[8\8\8hEL!344nr	#"  
 
	 tz<00 	j$%%juVZ&[&[&[\\\$%%juVZ&[&[&[\\\$%%juVZ&[&[&[\\\ "$"(4:3H*I*IQ*N!O!OPQ!R) j j%%j1B31GTYcg&h&h&hiiii~!##((888tz&''1,,  uu]a]g]tuuu   /8{+.7{+.7{+$ 	39=D"#56EID"#AB/4D"8,-2D#*r   c                    t          j                    t          j                    t          j        dd          g}| j        r|t          j        ddd          gz  }|r|t          j        dd          gz  }t          j        |          S )N   ▁r   FTrp   trim_offsets	use_regexru   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementrp   sequences       r   decoderzGGUFLlamaConverter.decoder  s    !##MOOUC((
 $ 	i+UQVbfggghhH 	>!<<<==H ***r   c                    |                      | j                  }|                     | j                  }|||_        d}d}t          | j        d          r| j        j        }|                     ||          }|||_        |                     ||          |_        |                                 }|r||_        | j	        r5t          j        ddd          |_        t          j        g           |_        |S )Nr   Trp   Fr   )rj   r   
normalizerr   r   rp   pre_tokenizerr  post_processorr   r   r   r   r   )r   rj   r  r   rp   r  r  s          r   	convertedzGGUFLlamaConverter.converted  s   NN4:..	 __TZ00
!#-I 4*,>?? 	H#6G**;8HII$&3I# LL6FGG	,,.. 	6'5I$ $ 	<&4&>!&Ud' ' 'I#
 $/#7#;#;I r   N)	r   r   r   r   r   ro   rj   r  r  r   r   r   r   r     sr        ^ ^ ^5 5 5  8 8 8t+ + +! ! ! ! !r   r   c                   *     e Zd Zd Zdef fdZ xZS )GGUFQwen2Converterc                 <    t          |          | _        i | _        d S r   r   r   r   r   s     r   r   zGGUFQwen2Converter.__init__       "7"G"G!#r   returnc           	      :   d t          | j        j                  D             }| j        j        }t	                                          ||          }|                    t          ddd          t          ddd          t          ddd          g           |S )Nc                     i | ]\  }}||	S r   r   r   r   r   s      r   r   z0GGUFQwen2Converter.converted.<locals>.<dictcomp>      RRRWQqRRRr   <|endoftext|>FTr   z<|im_start|>z
<|im_end|>)r   r   rl   ro   superr  r   r   r   r   ro   rj   	__class__s       r   r  zGGUFQwen2Converter.converted  s    RR	$2I2P(Q(QRRR(/GG%%eV44	$$?udKKK>eTJJJ<E4HHH	
 	
 	
 r   r   r   r   r   r   r  __classcell__r  s   @r   r	  r	    sS        $ $ $9          r   r	  c                   8    e Zd Zd Zd Zd Zd Zd ZdefdZ	dS )	GGUFPhi3Converterc                 T    t          |          | _        | j        | _        i | _        d S r   )r   r   r   r   r   s     r   r   zGGUFPhi3Converter.__init__)  s)    *>::
"&*!#r   c                 P    t          t          |j        |j                            S r   r   r   s     r   r   zGGUFPhi3Converter.vocab.  r   r   c                     |j         S r   r   r   s     r   ro   zGGUFPhi3Converter.merges1  r   r   c                    |                      | j                  }|                     | j                  }d t          |          D             }t	          t          ||                    }|                    t          ddddd          t          ddd          t          dddd	          t          d
ddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          g           |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |S )Nc                      i | ]\  }\  }}||S r   r   r   s       r   r   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>7  r   r   z</s>TF)rstriplstripr   r   r  r   z<|assistant|>)r   r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   	pad_token)r   r   ro   r   r   r   r   r   rZ   rl   r   rY   rX   r[   )r   r   r   ro   r   rj   s         r   rj   zGGUFPhi3Converter.tokenizer4  s&   zz$*--TZ((NNi6M6MNNN	c)V4455	$$6$uX\]]]?udKKK?4ESWXXX-duVZ[[[-duVZ[[[-duVZ[[[-duVZ[[[<PTUUU9TeTRRR-duVZ[[[-duVZ[[[:dudSSS	
 	
 	
$ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ r   c                     t          j                    t          j                    t          j        |d          g}|r|t          j        dd          gz  }t          j        |          S )Nr   ru   r   )r   r   r   r   r   r   r   s       r   r  zGGUFPhi3Converter.decoder[  se    !##MOO[#..
  	>!<<<==H ***r   r  c                     |                      | j                  }d}d}t          | j        d          r| j        j        }|                     ||          |_        |S )Nr   Trp   )rj   r   r   r   rp   r  )r   rj   r   rp   s       r   r  zGGUFPhi3Converter.convertedf  s`    NN4:..	4*,>?? 	H#6G LL6FGG	r   N)
r   r   r   r   r   ro   rj   r  r   r  r   r   r   r  r  (  s{        $ $ $
5 5 5  % % %N	+ 	+ 	+
9 
 
 
 
 
 
r   r  c                   *     e Zd Zd Zdef fdZ xZS )GGUFGPTConverterc                 <    t          |          | _        i | _        d S r   r  r   s     r   r   zGGUFGPTConverter.__init__t  r  r   r  c                     d t          | j        j                  D             }| j        j        }t	                                          ||          }|S )Nc                     i | ]\  }}||	S r   r   r  s      r   r   z.GGUFGPTConverter.converted.<locals>.<dictcomp>y  r  r   )r   r   rl   ro   r  r  r  s       r   r  zGGUFGPTConverter.convertedx  sN    RR	$2I2P(Q(QRRR(/GG%%eV44	r   r  r  s   @r   r&  r&  s  sS        $ $ $9          r   r&  )	r2   r4   	qwen2_moer6   r7   ri   r:   r;   r<   r  c                 j    | }t          |         |          }|                                }||j        fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr  r   )r?   r   tokenizer_class_name	converterfast_tokenizers        r   convert_gguf_tokenizerr0    s<     ('(<=nMMI((**N9666r   )#__doc__r   numpyr   
tokenizersr   r   r   r   tokenizers.modelsr    r   convert_slow_tokenizerr   r   r   utilsr   utils.loggingr   
get_loggerr   r   GGUF_TENSOR_MAPPINGGGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGr   r   r   r	  r  r&  r,  r0  r   r   r   <module>r=     s    
           G G G G G G G G G G G G ! ! ! ! ! !       R R R R R R R R R R                   
	H	%	%
 +##.&$$$))#   +##.&$$$))#   +##.&$$$))# *~ 	} 	3	
 	 	7 	. 	0 	 	7 	6 	& 	$ 	$ 	$  	)!" 	)#$ 	|% * +$#&.&())#  BB%'.&4-))  (%'&4-
 
 (%'! 4-  +##.&$$$))#   (*)!2.   + .&$$$))# _] ] D %% 
 4*2) *& 5#8,:"  4*2) *& 5#8,:"  4*2) $& 5#8,:"  4*2) $& 5#8,:"%2  4*2) $& 5#8,:"  ,+!/!/	  4*2) $& 5#8,:"  !) ("(<  4*2) $ 5#8(8"
 
 !!$4 ((<  +3)2 5#8(6 cz z | ''++!/!/!3  )"++!/!/   0  0(6 (6 (6 (6 (6 (6 (6 (6Vv v v v v v v vr       (H H H H H H H HV	 	 	 	 	} 	 	 	  # "
 
 7I 7 7 7 7 7 7r   