
    g<                       d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlZddlmZ ddlmZ ddlmZmZmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0m1Z1  e(            rddl2m3Z3  e+j4        e5          Z6dZ7dZ8e G d de                      Z9e G d de                      Z:e G d de                      Z;e G d de                      Z< G d dej=                  Z> e$j?        e>            G d  d!ej=                  Z@ G d" d#ej=                  ZA G d$ d%ej=                  ZBd& ZCdHd'ZD G d( d)ej=                  ZEd*ejF        d+eGd,ejF        fd-ZH G d. d/ej=                  ZI G d0 d1eI          ZJ G d2 d3eI          ZKeIeJeKd4ZL G d5 d6ej=                  ZM G d7 d8e"          ZNd9ZOd:ZPd;ZQ G d< d=eNe          ZR e&d>eO           G d? d@eN                      ZS e&dAeO           G dB dCeNe                      ZT e&dDeO           G dE dFeNe                      ZUg dGZVdS )IzPyTorch Moshi model.    N)	dataclass)AnyDictListOptionalTupleUnion)CrossEntropyLoss   )ACT2FN)CacheDynamicCacheSlidingWindowCacheStaticCache)GenerationConfigGenerationMixin)AttentionMaskConverter)BaseModelOutputWithPastCausalLMOutputWithPastModelOutputSeq2SeqLMOutput)PreTrainedModel)ALL_LAYERNORM_LAYERS)add_start_docstrings%add_start_docstrings_to_model_forwardis_flash_attn_2_available#is_flash_attn_greater_or_equal_2_10is_torchdynamo_compilingloggingreplace_return_docstrings   )	AutoModel   )MoshiConfigMoshiDepthConfig)_flash_attention_forwardr$   zkmhf/hf-moshikoc                      e Zd ZU dZdZeej                 ed<   dZ	ej
        ed<   dZeej                 ed<   dZeeej                          ed<   dZeeej                          ed<   dZeej
                 ed<   dZeeeej                                   ed	<   dZeeeej                                   ed
<   dZeeeeej                                            ed<   dZeej
                 ed<   dS )(MoshiConditionalGenerationGenerateOutputa  
    Outputs of [`MoshiForConditionalConditionalGeneration.generate`].

    Args:
        audio_sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, 1, sequence_length)`, *optional*):
            The generated audio waveforms.
        sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
            The generated text sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
            if all batches finished early due to the `eos_token_id`.
        sequences_scores (`torch.FloatTensor` of shape `(batch_size*num_return_sequences)`, *optional*, returned when `output_scores=True`):
            Final beam scores of the generated `sequences`.
        scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True`):
            Beam transition scores for each vocabulary token at each generation step. Beam transition scores consisting
            of log probabilities of tokens conditioned on log softmax of previously generated tokens in this beam.
            Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for each generated token),
            with each tensor of shape `(batch_size*num_beams, config.vocab_size)`.
        logits (`tuple(torch.FloatTensor)` *optional*, returned when `output_logits=True`):
            Unprocessed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
            at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
            each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.
        beam_indices (`torch.LongTensor`, *optional*, returned when `output_scores=True`):
            Beam indices of generated token id at each generation step. `torch.LongTensor` of shape
            `(batch_size*num_return_sequences, sequence_length)`.
        attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`):
            Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
            `torch.FloatTensor` of shape `(batch_size*num_beams, num_heads, generated_length, sequence_length)`.
        hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True`):
            Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
            `torch.FloatTensor` of shape `(batch_size*num_beams*num_return_sequences, generated_length, hidden_size)`.
        past_key_values (`tuple(tuple(torch.FloatTensor)))`, *optional*, returned when `use_cache=True`):
            Returns the model cache, used to speed up decoding. Different models have a different cache format, check
            the model's documentation. Usually, a [`~cache_utils.Cache`] instance.
        audio_codes (`torch.LongTensor` of shape `(batch_size*num_return_sequences, num_codeooks, sequence_length)`, *optional*):
            The generated audio codes. Returned if `return_audio_codes=True`. Intermediate audio "tokens" which transforms to `audio_sequences` once passed through the audio decoder.
    Naudio_sequences	sequencessequences_scoresscoreslogitsbeam_indices
attentionshidden_statespast_key_valuesaudio_codes)__name__
__module____qualname____doc__r)   r   torchTensor__annotations__r*   
LongTensorr+   FloatTensorr,   r   r-   r.   r/   r0   r1   r2        d/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/moshi/modeling_moshi.pyr(   r(   ?   s7        " "H /3OXel+222"&Iu&&&48hu0188815FHU5,-.55515FHU5,-.555/3L(5+,333<@JuU%6789@@@?CM8E%(9":;<CCCHLOXeE%0A*B$CDELLL.2K%*+22222r=   r(   c                      e Zd ZU dZdZeej                 ed<   dZ	ej        ed<   dZ
ej        ed<   dZeeeej                                   ed<   dZeeej        df                  ed<   dZeeej        df                  ed	<   dS )
MoshiCausalLMOutputWithPasta  
    `MoshiForCausalLM` outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Language modeling loss (for next-token prediction).
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlossr-   last_hidden_stater1   .r0   r/   )r3   r4   r5   r6   rA   r   r7   r;   r9   r-   rB   r1   r   r0   r/   r<   r=   r>   r@   r@   q   s          : )-D(5$
%,,, $FE$$$+/u(///AEOXeE%*;$<=>EEE=AM8E%"3S"89:AAA:>Ju0#567>>>>>r=   r@   c                      e Zd ZU dZdZeej                 ed<   dZ	ej        ed<   dZ
ej        ed<   dZeeeej                                   ed<   dZeeej        df                  ed<   dZeeej        df                  ed	<   dZeej                 ed
<   dZej        ed<   dZeeeej                                   ed<   dZeeej        df                  ed<   dZeeej        df                  ed<   dS )(MoshiConditionalGenerationOutputWithPastaf  
    `MoshiForConditionalGeneration` outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `text_labels` is provided):
            Text language modeling loss (for next-token prediction).
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the text language modeling head (scores for each vocabulary token before SoftMax).
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        depth_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `audio_labels` is provided):
            Audio language modeling loss (for next-token prediction).
        audio_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the audio language modeling heads.
        depth_past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Past key-values of the depth decoder.
        depth_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Hidden states of the depth decoder
        depth_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Depth decoder's Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NrA   r-   rB   r1   .r0   r/   
depth_lossaudio_logitsdepth_past_key_valuesdepth_hidden_statesdepth_attentions)r3   r4   r5   r6   rA   r   r7   r;   r9   r-   rB   r1   r   r0   r/   rE   rF   rG   rH   rI   r<   r=   r>   rD   rD      sX        & &P )-D(5$
%,,, $FE$$$+/u(///AEOXeE%*;$<=>EEE=AM8E%"3S"89:AAA:>Ju0#567>>>.2J*+222&*L%#***GK8E%0A*B$CDKKKCG%(93(>"?@GGG@DhuU%6%;<=DDDDDr=   rD   c                   t    e Zd ZU dZdZej        ed<   dZej	        ed<   dZ
ej	        ed<   dZej        ed<   dS )MoshiUnconditionalInputaW  
    Args:
        input_ids (`torch.Tensor `of shape `(batch_size, sequence_length), *optional*):
            The sequence used as a text prompt for the generation.
        user_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*):
            The audio codes used as audio user prompt for the generation. Has priority over `user_input_values` and represents the audio "tokens" of `user_input_values` once passed through the audio encoder.
        moshi_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*):
            The audio codes used as audio Moshi prompt for the generation. Has priority over `moshi_input_values` and represents the audio "tokens" of `moshi_input_values` once passed through the audio encoder.
        attention_mask (`torch.LongTensor`)  of shape `(batch_size, sequence_length)`, *optional*):
            Attention mask to avoid performing attention on padding token indices. Mask values selected in `[0,
            1]`: 1 for tokens that are **not masked**, 0 for tokens that are **masked**.
    N	input_idsuser_audio_codesmoshi_audio_codesattention_mask)r3   r4   r5   r6   rL   r7   r:   r9   rM   r8   rN   rO   r<   r=   r>   rK   rK      sj           #'Iu&&&%)el)))&*u|***'+NE$+++++r=   rK   c                   <     e Zd Zddedef fdZd Zd Zd Z xZ	S )	MoshiRMSNormư>dimepsc                     t                                                       || _        t          j        t          j        |                    | _        d S N)super__init__rT   nn	Parameterr7   onesweight)selfrS   rT   	__class__s      r>   rX   zMoshiRMSNorm.__init__   s=    l5:c??33r=   c                     |t          j        |                    d                              dd          | j        z             z  S )Nr!   T)keepdim)r7   rsqrtpowmeanrT   )r]   xs     r>   _normzMoshiRMSNorm._norm   s8    5;quuQxx}}R}>>IJJJJr=   c                     |                      |                                          }|| j                                        z  }|                    |          S rV   )rf   floatr\   type_as)r]   re   outputs      r>   forwardzMoshiRMSNorm.forward   sE    AGGII&&$+++---~~a   r=   c                 H    t          | j        j                   d| j         S )Nz, eps=)tupler\   shaperT   r]   s    r>   
extra_reprzMoshiRMSNorm.extra_repr   s%    )**<<$(<<<r=   )rR   )
r3   r4   r5   intrh   rX   rf   rk   rp   __classcell__r^   s   @r>   rQ   rQ      s        4 4C 4e 4 4 4 4 4 4
K K K! ! !
= = = = = = =r=   rQ   c                   &     e Zd Z fdZddZ xZS )MoshiFlexibleLinearc                     t                                                       t          j        t	          j        |||                    | _        d S rV   )rW   rX   rY   rZ   r7   randnr\   )r]   
input_sizeoutput_size
num_layersr^   s       r>   rX   zMoshiFlexibleLinear.__init__   s<    l5;z;
#S#STTr=   Nc                 
   |t          j        | j        d|          n| j        }|                    dd          dddddddf         }t          j        |dddddddf         |          }|                    d          S )a  
        `MoshiFlexibleLinear` creates one linear layer per codebook. There's multiple ways to use it.
        In the default case, `sequence_length=num_layers`, so each element of the sequence will be matmul to the weights corresponding to its index on the sequence.

        For more advanced cases, one can specify which codebook's layer(s) to use with `layer_idx`.
        If `layer_idx` indicates a single integer, all of the element of the sequence will be matmul to this single codebook's layer.
        But if `layer_idx` is a tensor of shape `(seq_length,)`, it will matmul each i-th element of the input sequence to the corresponding layer `weight[i]`.


        Args:
            x (`torch.FloatTensor): input to the layer of shape `(batch, num_layers, embed_dim)` or of shape `(batch, seq_length, embed_dim)`
            layer_idx (`torch.Tensor`, *optional*):
                Can be used to specify which codebook's layers(s) to use.
                If it's a tensor of shape `(seq_length,)`, will matmul each element of the sequence to the corresponding weights.
                But if `layer_idx` is a tensor of shape `(seq_length,)`, it will matmul each i-th element of the input sequence to the corresponding layer `weight[i]`.
        Nr   r#   r!   )r7   index_selectr\   	transposematmulsqueeze)r]   re   	layer_idxselected_weightss       r>   rk   zMoshiFlexibleLinear.forward  s    ( MVLa5-dk1iHHHgkgr ,55a;;D!!!QQQMJ L111aaaqqq=)+;<< yy||r=   rV   r3   r4   r5   rX   rk   rr   rs   s   @r>   ru   ru      sQ        U U U U U
       r=   ru   c                   (     e Zd Zd fd	ZddZ xZS )MoshiLinearFc                     t                                                       || _        |st          j        ||d          | _        d S t          |||          | _        d S )NFbias)rz   )rW   rX   use_flexible_linearrY   Linearlinearru   )r]   	input_dim
output_dimnum_codebooksr   r^   s        r>   rX   zMoshiLinear.__init__#  sa    #6 " 	_)IzFFFDKKK-iP]^^^DKKKr=   Nc                 f    | j         r|                     ||          S |                     |          S rV   )r   r   )r]   re   r   s      r>   rk   zMoshiLinear.forward-  s1    # 	";;q),,,;;q>>!r=   FrV   r   rs   s   @r>   r   r   "  sW        _ _ _ _ _ _" " " " " " " "r=   r   c                   P     e Zd Zd fd	Z ej                    d             Z xZS )MoshiRotaryEmbedding   '  Nc                 `   t                                                       || _        || _        || _        d| j        t          j        d| j        dt
          j                                                  	                    |          | j        z  z  z  }| 
                    d|d           d S )N      ?r   r!   dtypeinv_freqF)
persistent)rW   rX   rS   max_position_embeddingsbaser7   arangeint64rh   toregister_buffer)r]   rS   r   r   devicer   r^   s         r>   rX   zMoshiRotaryEmbedding.__init__6  s    '>$	$)Q!5;(W(W(W(](](_(_(b(bci(j(jmqmu(uvwZeDDDDDr=   c                    | j         d d d d f                                                             |j        d         dd          }|d d d d d f                                         }|j        j        }t          |t                    r|dk    r|nd}t          j	        |d          5  |                                |                                z  
                    dd          }t          j        ||fd	          }|                                }|                                }	d d d            n# 1 swxY w Y   |                    |j        
          |	                    |j        
          fS )Nr   r`   r#   mpscpuF)device_typeenabledr!   rS   r   )r   rh   expandrn   r   type
isinstancestrr7   autocastr}   catcossinr   r   )
r]   re   position_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r>   rk   zMoshiRotaryEmbedding.forward?  s   
 !M$4-8>>@@GGHZ[\H]_acdee ,QQQaaaZ 8 > > @ @ hm%/S%A%AekUZFZFZkk`e^UCCC 	 	&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))C''))C		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 vvAGv$$cff17f&;&;;;s   'A>D11D58D5)r   r   N)r3   r4   r5   rX   r7   no_gradrk   rr   rs   s   @r>   r   r   5  sf        E E E E E E U]__< < _< < < < <r=   r   c                     | dd| j         d         dz  f         }| d| j         d         dz  df         }t          j        | |fd          S )z*Rotates half the hidden dims of the input..Nr`   r!   r   )rn   r7   r   )re   x1x2s      r>   rotate_halfr   S  s]    	
3"!'"+"""	#B	
3q """	#B9rc2YB''''r=   c                     |                     |          }|                     |          }| |z  t          |           |z  z   }||z  t          |          |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   r   unsqueeze_dimq_embedk_embeds           r>   apply_rotary_pos_embr   [  sc    ( --
&
&C
--
&
&C3w;q>>C/0G3w;q>>C/0GGr=   c                   J     e Zd Zd fd	Zd	dej        dedej        fdZ xZS )
MoshiGatingMLPFc                    t                                                       t          |j                 | _        |j        }|j        }|r|j        nd}|dk    r=t          j	        ||d          | _
        t          j	        |dz  |d          | _        d S t          |||          | _
        t          |dz  ||          | _        d S )Nr#   Fr   r!   )rW   rX   r   
hidden_actactivation_fnffn_dimhidden_sizer   rY   r   fc1fc2ru   )r]   configr   r   r   rz   r^   s         r>   rX   zMoshiGatingMLP.__init__w  s    #F$56.(-@GV))a
??ygEBBBDHyA{GGGDHHH*;LLDH*7a<jQQDHHHr=   Nr0   r   returnc                 \   ||                      |          n|                      ||          }|j        \  }}}|                    ||dd          }|                     |ddd d f                   |ddd d f         z  }||                     |          n|                     ||          }|S )Nr!   r`   .r   r#   )r   rn   viewr   r   )r]   r0   r   
batch_sizesequence_length_s         r>   rk   zMoshiGatingMLP.forward  s    3<3D///$((S`bkJlJl)6)<&
OQ%**:2NN**=a+CDD}UXZ[]^]^]^U^G__3<3D///$((S`bkJlJlr=   r   rV   )	r3   r4   r5   rX   r7   r8   rq   rk   rr   rs   s   @r>   r   r   v  su        R R R R R R U\ c U\        r=   r   r0   n_repr   c                     | j         \  }}}}|dk    r| S | dddddddddf                             |||||          } |                     |||z  ||          S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rn   r   reshape)r0   r   batchnum_key_value_headsslenhead_dims         r>   	repeat_kvr     s    
 2?1D.Ehzz!!!!QQQaaa"23::5BUW\^bdlmmM  (;e(CT8TTTr=   c                       e Zd ZdZddedee         f fdZ	 	 	 	 	 	 ddej	        d	eej	                 d
eej
                 dee         dededeej
                 deej	        eej	                 eeej	                          f         fdZ xZS )MoshiAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNFTr   r   c                    t                                                       || _        || _        |(t                              d| j        j         d           |j        | _        |j	        | _	        |j
        | _        |j        | _        |j        | _        | j        | j        z  | _        |j        | _        d| _        dt#          j        | j                  z  | _        | j	        | j        z  dk    r t)          d| j	         d| j         d          t+          | j	        | j        | j        z  |j        |          | _        t+          | j	        | j        | j        z  |j        |          | _        t+          | j	        | j        | j        z  |j        |          | _        t+          | j        | j        z  | j	        |j        |          | _        d | _        |r4|j        | _        t;          | j        | j        | j        	          | _        d S d S )
NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tr#   r   z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).)r   r   )rW   rX   r   r   loggerwarning_oncer^   r3   attention_dropoutr   num_attention_heads	num_headsr   r   num_key_value_groupsr   	is_causalmathsqrtscaling
ValueErrorr   r   q_projk_projv_projo_proj
rotary_emb
rope_thetar   r]   r   r   r   use_roper^   s        r>   rX   zMoshiAttention.__init__  s#   ",!8 , , ,   "(!9!-3#)#= $(Nd6N$N!'-'E$49T]333dn,118RVRb 8 8%)^8 8 8  
 "dnt}<f>RTg
 
 "d6FH\^q
 
 "d6FH\^q
 
 "NT]*D,<f>RTg
 

  	$/DO2(,(D_  DOOO	 	r=   r0   rO   r   past_key_valueoutput_attentions	use_cachecache_positionr   c                 H   |                                 \  }}	}
|                     ||          }|                     ||          }|                     ||          }|                    ||	| j        | j                                      dd          }|                    ||	| j        | j                                      dd          }|                    ||	| j        | j                                      dd          }| j	        .| 	                    ||          \  }}t          ||||          \  }}|1| j	        |||dnd|i}|                    ||| j        |          \  }}t          || j                  }t          || j                  }t          j        ||                    dd                    | j        z  }|$|d d d d d d d |j        d         f         }||z   }t&          j                            |dt          j                                      |j                  }t&          j                            || j        | j        	          }t          j        ||          }|                                 || j        |	| j        fk    r5t9          d
|| j        |	| j        f d|                                            |                    dd                                          }|                    ||	d          }|                     ||          }|sd }|||fS )Nr#   r!   r   r   r   r   r   r`   )rS   r   )ptrainingz `attn_output` should be of size z	, but is )sizer   r   r   r   r   r   r}   r   r   r   updater   r   r   r7   r~   r   rn   rY   
functionalsoftmaxfloat32r   r   dropoutr   r   r   
contiguousr   )r]   r0   rO   r   r   r   r   r   bszq_lenr   query_states
key_statesvalue_statesr   r   cache_kwargsattn_weightscausal_maskattn_outputs                       r>   rk   zMoshiAttention.forward  s5    &**,,UA{{=.AA[[??
{{=.AA#((eT^T]SS]]^_abcc__S%1I4=YYccdeghii
#((eT5Mt}]]gghiklmm?&|\BBHC';L*VY[^'_'_$L*% ?. C>JJJ&7 
 (6'<'<ZW[Wegs't't$Jz4+DEE
 t/HII|L*2F2Fq!2L2LMMPTP\\%(AAAqqq2HJ4DR4H2H)HIK'+5L },,\r,WWZZ[g[mnn},,\T=S^b^k,lll<>>#t~udm!LLL)CPTP]3^ ) )$$&&) )  
 "++Aq11<<>>!&&sE266kk+~>>  	 LL.88r=   )NFTNNNFFN)r3   r4   r5   r6   r$   r   rq   rX   r7   r8   r:   r   boolr   rk   rr   rs   s   @r>   r   r     s       GG0 0{ 0x} 0 0 0 0 0 0l 2637*."'59=9 =9|=9 !.=9 u/0	=9
 !=9  =9 =9 !!12=9 
u|Xel3XeEL>Q5RR	S=9 =9 =9 =9 =9 =9 =9 =9r=   r   c                       e Zd ZdZ fdZ	 	 	 	 	 	 ddej        deej                 deej                 dee	         d	e
d
e
deej                 deej        eej                 eeej                          f         fdZ xZS )MoshiFlashAttention2aF  
    Moshi flash attention module. This module inherits from `MoshiAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 b     t                      j        |i | t                       | _        d S rV   )rW   rX   r   _flash_attn_uses_top_left_mask)r]   argskwargsr^   s      r>   rX   zMoshiFlashAttention2.__init__  s9    $)&)))
 3V2W2W.W+++r=   NFr0   rO   r   r   r   r   r   r   c                 J   t          |t                    rt          d          d}|                                \  }}	}
|                     ||          }|                     ||          }|                     ||          }|                    ||	| j        | j	                  
                    dd          }|                    ||	| j        | j	                  
                    dd          }|                    ||	| j        | j	                  
                    dd          }| j        .|                     ||          \  }}t          ||||          \  }}|1| j        |||dnd|i}|                    ||| j        |          \  }}|
                    dd          }|
                    dd          }|
                    dd          }| j        r| j        nd}|j        }|t&          j        k    rt'          j                    rt'          j                    }n3t/          | j        d          r| j        j        }n| j        j        j        }t6                              d	| d
           |                    |          }|                    |          }|                    |          }t=          |||||	||t?          | dd           | j         | j!        
  
        }|"                    ||	d          #                                }| $                    ||          }|sd }|||fS )Nz`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformersFr#   r!   r   r           _pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .sliding_window)r   r   r  r   use_top_left_maskr`   )%r   r   r   r   r   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   r7   r   is_autocast_enabledget_autocast_gpu_dtypehasattrr   r  r\   r   r   r   r&   getattrr   r  r   r   r   )r]   r0   rO   r   r   r   r   r   r   r   r   r  r  r  r   r   r  dropout_rateinput_dtypetarget_dtyper  r  s                         r>   rk   zMoshiFlashAttention2.forward"  sW    nk22 	}  
 "%**,,UA{{=.AA[[??
{{=.AA
 $((eT^T]SS]]^_abcc__S%1I4=YYccdeghii
#((eT5Mt}]]gghiklmm?&|\BBHC';L*VY[^'_'_$L*% ?. C>JJJ&7 
 (6'<'<ZW[Wegs't't$J $--a33))!Q//
#--a3315Gt--C #(%-''(** 8$;==&?@@ 8#{B#{17$ $ $ $   (??<88L#|44J'??<88L.% "4)94@@n"A
 
 
 "))#ub99DDFFkk+~>>  	 LL.88r=   r  )r3   r4   r5   r6   rX   r7   r8   r   r:   r   r	  r   rk   rr   rs   s   @r>   r  r    s        X X X X X 6:37*."'59a9 a9|a9 !!12a9 u/0	a9
 !a9  a9 a9 !!12a9 
u|Xel3XeEL>Q5RR	Sa9 a9 a9 a9 a9 a9 a9 a9r=   r  c                        e Zd ZdZ	 	 	 	 	 	 ddej        deej                 deej                 dee         de	d	e	d
eej                 de
ej        eej                 ee
ej                          f         f fdZ xZS )MoshiSdpaAttentionz
    Moshi attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
    `MoshiAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
    SDPA API.
    NFr0   rO   r   r   r   r   r   r   c           	         |rBt                               d           t                                          |||||||          S |                                \  }	}
}|                     ||          }|                     ||          }|                     ||          }|                    |	|
| j	        | j
                                      dd          }|                    |	|
| j        | j
                                      dd          }|                    |	|
| j        | j
                                      dd          }| j        .|                     ||          \  }}t          ||||          \  }}|1| j        |||dnd|i}|                    ||| j        |          \  }}t#          || j                  }t#          || j                  }|}||d d d d d d d |j        d         f         }|j        j        dk    r>|<|                                }|                                }|                                }||
dk    rd	nd
}t.          j        j                            ||||| j        r| j        nd|          }|                    dd                                          }|                    |	|
d          }|                     ||          }|d |fS )Na  MoshiModel is using MoshiSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r0   rO   r   r   r   r   r   r#   r!   r   r   r   cudaTFr  )	attn_mask	dropout_pr   r`   )r   r   rW   rk   r   r   r   r   r   r   r   r}   r   r   r   r   r   r   r   rn   r   r   r   r7   rY   r   scaled_dot_product_attentionr   r   r   )r]   r0   rO   r   r   r   r   r   r  r   r   r   r  r  r  r   r   r  r  r   r  r^   s                        r>   rk   zMoshiSdpaAttention.forward  s'     	[   77??+-)-"3#- #    &**,,UA{{=.AA[[??
{{=.AA#((eT^T]SS]]^_abcc__S%1I4=YYccdeghii
#((eT5Mt}]]gghiklmm?&|\BBHC';L*VY[^'_'_$L*% ?. C>JJJ&7 
 (6'<'<ZW[Wegs't't$Jz4+DEE
 t/HII$%%aaaAAA/E1A"1E/E&EFK #v--+2I'2244L#..00J'2244L (/EAIIDD5	h)FF!04Fd,,3 G 
 
 "++Aq11<<>>!&&sE266kk+~>>D.00r=   r  )r3   r4   r5   r6   r7   r8   r   r:   r   r	  r   rk   rr   rs   s   @r>   r  r    s          2637*."'59R1 R1|R1 !.R1 u/0	R1
 !R1  R1 R1 !!12R1 
u|Xel3XeEL>Q5RR	SR1 R1 R1 R1 R1 R1 R1 R1 R1 R1r=   r  )eagerflash_attention_2sdpac                       e Zd Zddededef fdZ	 	 	 	 	 	 ddej        d	e	ej                 d
e	ej
                 de	e         de	e         de	e         de	ej
                 deej        e	eej        ej        f                  f         fdZ xZS )MoshiDecoderLayerTr   r   r   c                    t                                                       |j        | _        || _        t	          |j                 ||||          | _        t          ||          | _        t          | j        |j
                  | _        t          | j        |j
                  | _        |j        | _        |j        | _        d S )N)r   r   r   r   rT   )rW   rX   r   r   MOSHI_ATTENTION_CLASSES_attn_implementation	self_attnr   mlprQ   rms_norm_epsinput_layernormpost_attention_layernormr  r   s        r>   rX   zMoshiDecoderLayer.__init__  s    !-#6 01LMYDWbj
 
 
 "&*=>>+D,<&BUVVV(4T5E6K^(_(_(_%$3$*$?!!!r=   NFr0   rO   r   r   r   r   r   r   c                 2   |}	|                      |          } | j        d|||||||d|\  }}
}|	|z   }|}	|                     |          }| j        s|                     |          n|                     ||          }|	|z   }|f}|r||
fz  }|r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        r   r<   )r1  r.  r2  r   r/  )r]   r0   rO   r   r   r   r   r   r  residualself_attn_weightspresent_key_valueoutputss                r>   rk   zMoshiDecoderLayer.forward  s    < !,,];; ?Mdn 	?
')%)/)	?
 	?
 	?
 	?
;(*; !=0 !55mDD+/+CpDHH]###R_aoIpIp 	 !=0 " 	,)++G 	,)++Gr=   )Tr  )r3   r4   r5   r$   rq   r	  rX   r7   r8   r   r:   r   r   r;   rk   rr   rs   s   @r>   r)  r)    s$       @ @{ @s @QU @ @ @ @ @ @& 2637*.,1$)59? ?|? !.? u/0	?
 !? $D>? D>? !!12? 
u (51BEDU1U+V"WW	X? ? ? ? ? ? ? ?r=   r)  c                   <    e Zd ZdZeZdZdZddgZdZ	dZ
dZdZd ZdS )	MoshiPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    modelTr)  MimiTransformerLayerrL   c                    | j         j        }t          |t          j        t          j        f          rJ|j        j                            d|           |j	         |j	        j        
                                 d S d S t          |t          j        t          j        f          r?|j	        j        
                                 |j        j                            d           d S t          |t          j                  rt          j                            |j                   |j	        Yt!          j        |j        |j        |j        d         z  z            }t          j                            |j	        | |           d S d S t          |t          j                  rS|j        j                            d|           |j        -|j        j        |j                 
                                 d S d S d S )Nr  )rd   stdr   r   )ab)r   initializer_ranger   rY   r   Conv1dr\   datanormal_r   zero_	LayerNorm	GroupNormfill_initkaiming_normal_r   r   groupsin_channelskernel_sizeuniform_	Embeddingpadding_idx)r]   moduler=  r   s       r>   _init_weightsz"MoshiPreTrainedModel._init_weightsM  s   k+fry")455 	?M&&CS&999{& &&((((( '&r| <== 	?K""$$$M$$S)))))	** 	?G##FM222{&Ifmv/AFDVWXDY/YZ[[  a 88888 '& -- 	?M&&CS&999!-"6#56<<>>>>>	? 	?--r=   N)r3   r4   r5   r6   r$   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_flash_attn_2_supports_sdpa_supports_cache_classmain_input_namerQ  r<   r=   r>   r9  r9  >  sb         
 L&*#,.DE!N !O? ? ? ? ?r=   r9  a>  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`MoshiConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence text tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

        user_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*):
            The audio waveforms used as audio user prompt for the generation.
        user_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*):
            The audio codes used as audio user prompt for the generation. Has priority over `user_input_values` and represents the audio "tokens" of `user_input_values` once passed through the audio encoder.
        moshi_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*):
            The audio waveforms used as audio Moshi prompt for the generation.
        moshi_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*):
            The audio codes used as audio Moshi prompt for the generation. Has priority over `moshi_input_values` and represents the audio "tokens" of `moshi_input_values` once passed through the audio encoder.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
            representation. If `past_key_values` is used, optionally only the last `inputs_embeds` have to be
            input (see `past_key_values`). This is useful if you want more control over how to convert
            `input_ids` indices into associated vectors than the model's internal embedding lookup matrix.

            If `input_ids` and `inputs_embeds` are both unset, `inputs_embeds` takes the value
            of `inputs_embeds`.
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance;
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.
        text_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for text language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        audio_labels (`torch.LongTensor` of shape `(batch_size, num_codebooks, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.audio_vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance, see our
            [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
            of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
            this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
            the complete sequence length.
c                       e Zd ZdZeZdef fdZ	 	 	 	 	 	 	 	 	 	 	 	 ddeej	                 dej	        deej
                 deeej                          d	eej                 d
ee         dee         dee         dee         deej	                 deej	                 deej	                 deeef         fdZdej        dej        dej        dedef
dZedej        dededej        dej        dej        dededefd            Z xZS )MoshiDepthDecoderz
    Transformer depth decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoshiTransformerLayer`]

    Args:
        config: MoshiConfig
    r   c                 D   t                                                     t          j        j        dz   j                  | _        t          j        fdt          j	        dz
            D                       | _
        t          j        j        j	                  | _        t          j        fdt          j                  D                       | _        t          j        j        j	                  | _        j        | _        d| _        | _        d S )Nr#   c                 T    g | ]$}t          j        j        d z   j                  %S r#   rY   rN  audio_vocab_sizer   .0r   r   s     r>   
<listcomp>z.MoshiDepthDecoder.__init__.<locals>.<listcomp>  0    tttqR\&1A5v7IJJtttr=   c                 6    g | ]}t          |d d          S )TF)r   r   r)  rb  r   r   s     r>   rc  z.MoshiDepthDecoder.__init__.<locals>.<listcomp>  s;        "&)X]^^^  r=   F)rW   rX   rY   rN  
vocab_sizer   text_embed_tokens
ModuleListranger   embed_tokensru   rx   input_projectionsnum_hidden_layerslayersr`  lm_headsr-  gradient_checkpointingr   r]   r   r^   s    `r>   rX   zMoshiDepthDecoder.__init__	  s      !#f.?!.CVEW!X!X MttttTYZ`ZnqrZrTsTsttt
 
 "5V5FHZ\b\p!q!qm   !&v'?!@!@  
 
 ,F,>@WY_Ymnn$*$?!&+#r=   NrL   rB   rO   r1   inputs_embedsr   r   output_hidden_statesreturn_dictr   labelsr   r   c                    ||n| j         j        }||n| j         j        }||n| j         j        }|	|	n| j         j        }	| j        r%| j        r|rt                              d           d}|r|| j        st          j
        |          }|dn|                                }|*t          j        |||j        d         z   |j                  }|
|                    d          }
|g }|D ]}|                                }|dk    r4|                    |                     |dd|gf                              P|                     | j        |dz
           |dd||z
  gf                              t          j        |d          }||                     ||          z  }d}||                     |||||          }|rdnd}|rdnd}d}|}| j        D ]p}|r||fz  }| j        r)| j        r"|                     |j        |||
||||          }n ||||
||||	          }|d         }|r||rd
nd         }|r||d         fz  }q|r||fz  }|r|nd}|                     ||          }d}||                                }t;                      }|                    || j         j        k    d                               d          }|!                    |j                  } ||                     d| j         j                  |          }|	stE          d |||||fD                       S tG          |||||          S )a  
        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens. The first element of the sequence must the text token associated to the audio codebooks.
                The rest of the elements must be flatten audio codebooks. The `cache_position` argument can be used to indicate to which index is associated each token.
            last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Sequence of hidden-states at the output of the last layer of the main decoder. Used to contextualize `input_ids`
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)

                Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
                `past_key_values`).

                If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
                and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
                information on the default strategy.

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
                Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
                blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
                returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

                Two formats are allowed:
                - a [`~cache_utils.Cache`] instance;
                - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
                cache format.

                The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
                legacy cache format will be returned.

                If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
                have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
                of shape `(batch_size, sequence_length)`.
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
                is useful if you want more control over how to convert the inputs into associated vectors than the
                model's internal embedding lookup matrix.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
                `past_key_values`).
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
                tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
                more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
            position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
                config.n_positions - 1]`.

                [What are position IDs?](../glossary#position-ids)
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
        NX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   r#   r   r   r<   rO   r   r   r   r   r   r!   ir`   c              3      K   | ]}||V  	d S rV   r<   rb  vs     r>   	<genexpr>z,MoshiDepthDecoder.forward.<locals>.<genexpr>  s(      ssqeferererererssr=   )rA   r-   r1   r0   r/   )$r   r   rt  r   use_return_dictrq  r   r   r   r   from_legacy_cacheget_seq_lengthr7   r   rn   r   r   itemappendri  rl  r   rm  _update_causal_maskro  _gradient_checkpointing_func__call__rp  rh   r
   masked_fillr`  r   r   rm   r   )r]   rL   rB   rO   r1   rs  r   r   rt  ru  r   rv  r   past_seen_tokensposition_idxr  all_hidden_statesall_self_attnsnext_decoder_cacher0   decoder_layerlayer_outputs
next_cacher-   rA   loss_fcts                             r>   rk   zMoshiDepthDecoder.forward!  sh   l 2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]& 	4= 	Y 	j   I 	N00*<_MMO / 711_=[=[=]=]!"\ "2Y_Q5G"GPYP`  N )33A66L  M .  +00221$$!(()?)?	!!!l^J[@\)])]^^^^!((=)<!+;=iL[kLkKlHl>mnn    "Im;;;M//0A>RRR%22~Pa K
 #7@BBD0:d!%![  	6  	6M# 6!m%55!* t}  $ A A!*! #%"	! 	! !.!#.!-#2&7'#1! ! ! *!,M R%28I3P11q%Q"  6=#3"55   	2-!11+4>''$
}n==\\^^F'))H''$+2N(NPTUU]]^`aaFYYv}--F8FNN2t{/KLLfUUD 	tssT6:?PR`$assssss%&+%
 
 
 	
r=   input_tensorc                    | j         j        dk    r
|d|v r|S d S ||                                nd}t          |t                    }t          |t
                    }| j         j        dk    r0|s.|s,|s*t          j        |||| j         j        | j	                  rd S |j
        |j        }
}	t          j        |	          j        }|j        d         }|s|r|                                }n/t          |t          j                  r|j        d         n||z   dz   }|                     ||||	|
||j        d         | j         |	  	        }| j         j        dk    r)|'|j        j        d	k    r|st          j        ||          }|S 
Nr&  r  r   r'  )rs  past_key_values_lengthr  is_trainingr#   r`   )r   target_lengthr   r   r   r   r   r1   r!  r   r-  r  r   r   r   r   _ignore_causal_mask_sdpar  r   r   r   r7   finfominrn   get_max_cache_shaper8   5_prepare_4d_causal_attention_mask_with_cache_positionr   _unmask_unattendedr]   rO   r  r   r1   r   r  using_static_cacheusing_sliding_window_cacher   r   	min_dtyper   r  r  s                  r>   r  z%MoshiDepthDecoder._update_causal_mask      ;+/BBB)c^.C.C%%4
 @O?Z?99;;;`a'EE%/AS%T%T" K,66' 7+E 7% 7 &>*'7#{9 M    t$*L,?vK&&*	&,Q/% 	); 	+??AAMM
 nel;;<$R((%7!;  PP+')#)!,;+ Q 

 

 K,66*%*f44% 5 1CKQZ[[Kr=   r   r  r   r   r   c	                 \   | |                                  dk    r| }	nt          j        |          j        }
t          j        ||f|
||          }	t          j        ||          |                    dd          k    }|j        Vt          |t                    r||k    r;t          j        ||          |                    dd          |j        z
  k    }||z  }|	|z  }	|	ddddddf         
                    |ddd          }	| |	                                }	| j        d         |k    r| ddd|f         } | j        d         }|	ddddddd|f         | ddddddf         z   }|dk    }|	ddddddd|f                             ||
          |	ddddddd|f<   |	S )aY  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            device (`torch.device`):
                The device to plcae the 4D attention mask on.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
            config (`MoshiDepthConfig`):
                The model's configuration class
            past_key_values (`Cache`):
                The cache class that is being used currently to generate
        N   
fill_valuer   r   ry  r`   r#   r   rS   r7   r  r  fullr   r   r  r   r   r   clonern   r  rO   r   r  r   r   r   r   r   r1   r  r  diagonal_attend_masksliding_attend_maskmask_lengthpadding_masks                  r>   r  zGMoshiDepthDecoder._prepare_4d_causal_attention_mask_with_cache_position7  2   J %.*<*<*>*>!*C*C(KKE**.I* -0Ye\b  K $)<f#M#M#MP^PfPfgiklPmPm#m $0 "/3EFF @/\iJiJi*/,}V*T*T*T&..r1558MM+' ),??(//K%dD!!!QQQ&67>>z1bRTUUK))//11!'+m;;%3AAA~~4E%FN,226*111aaaL[L+@ANSTSTSTVZ\`bcbcbcScDdd+q05@AAAqqq,;,AV5W5c5c )6 6AAAqqq!!!\k\12 r=   )NNNNNNNNNNNN)r3   r4   r5   r6   r%   rR  rX   r   r7   r:   
BoolTensorr   r;   r	  r	   r   rk   r8   r   r  staticmethodrq   r   r   r  rr   rs   s   @r>   r[  r[    sf         $L/      4 15.259;?59$(,0/3&*37-159H
 H
E,-H
 !+H
 !!12	H

 uU%678H
   12H
 D>H
 $D>H
 'tnH
 d^H
 u/0H
 )*H
 !!12H
 
u--	.H
 H
 H
 H
VII lI 	I
 I  I I I IV @@@ @ {	@
 @ @ @ !@ @ @ @ \@ @ @ @ @r=   r[  zSThe bare Moshi Model outputting raw hidden-states without any specific head on top.c                       e Zd ZdZdef fdZd Zd Z ee	          	 	 	 	 	 	 	 	 	 	 dde
j        dee
j                 d	ee
j                 d
eeeee
j                 f                  dee
j                 dee         dee         dee         dee         dee
j                 deeef         fd            Zde
j        de
j        de
j        d
edef
dZede
j        dedede
j        de
j        de
j        deded
efd            Z xZS )
MoshiModelz
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoshiDecoderLayer`]

    Args:
        config: MoshiConfig
    r   c                    t                                                     j        | _        j        | _        t          j        j        dz   j        | j                  | _        t          j	        fdt          j                  D                       | _        t          j        j                  | _        d| _        |                                  d S )Nr#   c                 4    g | ]}t          |d           S )F)r   rf  rg  s     r>   rc  z'MoshiModel.__init__.<locals>.<listcomp>  s8        "&)OOO  r=   r+  F)rW   rX   pad_token_idrO  rh  rY   rN  r   rl  rj  rk  rn  ro  rQ   r0  normrq  	post_initrr  s    `r>   rX   zMoshiModel.__init__  s       !. +L):Q)>@RTXTdeem   !&v'?!@!@  
 
 !!39LMMM	&+# 	r=   c                     | j         S rV   rl  ro   s    r>   get_input_embeddingszMoshiModel.get_input_embeddings  s      r=   c                     || _         d S rV   r  r]   values     r>   set_input_embeddingszMoshiModel.set_input_embeddings  s    !r=   NrL   rO   r   r1   rs  r   r   rt  ru  r   r   c                    ||n| j         j        }||n| j         j        }||n| j         j        }|	|	n| j         j        }	| j        r%| j        r|rt                              d           d}|| 	                    |          }d}|r2t          |t                    s| j        sd}t          j        |          }|
B||                                nd}t          j        |||j        d         z   |j                  }
||
                    d          }d }||                     |||
||          }|}|rLt          |t                    s7| j        s0d}t          j        |          }t                              d           |rdnd }|rdnd }d }| j        D ]p}|r||fz  }| j        r)| j        r"|                     |j        |||||||
          }n ||||||||
	          }|d         }|r||rd
nd         }|r||d         fz  }q|                     |          }|r||fz  }|r|nd }|r|                                }|	st5          d ||||fD                       S t7          ||||          S )Nrx  FTr   r#   ry  zWe detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/internal/generation_utils#transformers.Cache)r<   rz  r!   c              3      K   | ]}||V  	d S rV   r<   r|  s     r>   r~  z%MoshiModel.forward.<locals>.<genexpr>  s(      ttqfgfsfsfsfsfsttr=   )rB   r1   r0   r/   )r   r   rt  r   r  rq  r   r   r   rl  r   r   r   r  r  r7   r   rn   r   r   r  ro  r  r  r  to_legacy_cacherm   r   )r]   rL   rO   r   r1   rs  r   r   rt  ru  r   return_legacy_cacher  r  r0   r  r  r  r  r  r  s                        r>   rk   zMoshiModel.forward  s    2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]& 	4= 	Y 	j   I  --i88M#	N(%@@	NIM	N #'*<_MMO!CRC^==???de"\ "2]5H5K"KTaTh  N )33A66L%22~Pa K
 & 	(%@@	IM	 #'*<_MMOR   #7@BBD0:d!![  	6  	6M# 6!m%55!* t}  $ A A!*! #%"	! 	! !.!#.!-#2&7'#1! ! ! *!,M R%28I3P11q%Q"  6=#3"55		-00   	2-!11+4>''$
 	6#3355J 	utt]J@QSa$btttttt&+&+%	
 
 
 	
r=   r  c                    | j         j        dk    r
|d|v r|S d S ||                                nd}t          |t                    }t          |t
                    }| j         j        dk    r0|s.|s,|s*t          j        |||| j         j        | j	                  rd S |j
        |j        }
}	t          j        |	          j        }|j        d         }|s|r|                                }n/t          |t          j                  r|j        d         n||z   dz   }|                     ||||	|
||j        d         | j         |	  	        }| j         j        dk    r)|'|j        j        d	k    r|st          j        ||          }|S r  r  r  s                  r>   r  zMoshiModel._update_causal_mask  r  r=   r   r  r   r   r   c	                 \   | |                                  dk    r| }	nt          j        |          j        }
t          j        ||f|
||          }	t          j        ||          |                    dd          k    }|j        Vt          |t                    r||k    r;t          j        ||          |                    dd          |j        z
  k    }||z  }|	|z  }	|	ddddddf         
                    |ddd          }	| |	                                }	| j        d         |k    r| ddd|f         } | j        d         }|	ddddddd|f         | ddddddf         z   }|dk    }|	ddddddd|f                             ||
          |	ddddddd|f<   |	S )aT  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            device (`torch.device`):
                The device to plcae the 4D attention mask on.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
            config (`MoshiConfig`):
                The model's configuration class
            past_key_values (`Cache`):
                The cache class that is being used currently to generate
        Nr  r  ry  r`   r#   r   r  r  s                  r>   r  z@MoshiModel._prepare_4d_causal_attention_mask_with_cache_positiong  r  r=   )
NNNNNNNNNN)r3   r4   r5   r6   r$   rX   r  r  r   MOSHI_DECODER_INPUTS_DOCSTRINGr7   r:   r   r8   r	   r   r   r;   r	  r   r   rk   r  r  rq   r   r   r  rr   rs   s   @r>   r  r  |  ss       
 {      $! ! !" " " +*+IJJ '+1537KO59$(,0/3&*59x
 x
#x
 !.x
 u/0	x

 "%tE4E/F(F"GHx
   12x
 D>x
 $D>x
 'tnx
 d^x
 !!12x
 
u--	.x
 x
 x
 KJx
vII lI 	I
 I  I I I IV @@@ @ {	@
 @ @ @ @ @ @ @ \@ @ @ @ @r=   r  zYThe Moshi decoder model with a text language modelling head on top. Only usable for text.c                       e Zd ZddgZ fdZd Zd Zd Zd Zd Z	d	 Z
 ee           eee
          	 	 	 	 	 	 	 	 	 	 	 	 ddej        deej                 deej                 deeeeej                 f                  deej                 dee         dee         dee         dee         deej                 deej                 dedeeef         fd                        Z xZS )MoshiForCausalLMzmodel.embed_tokens.weightzlm_head.weightc                     t                                          |           t          |          | _        |j        | _        t          j        |j        |j        d          | _        | 	                                 d S )NFr   )
rW   rX   r  r:  rh  rY   r   r   lm_headr  rr  s     r>   rX   zMoshiForCausalLM.__init__  sj       ''
 +y!3V5FUSSS 	r=   c                     | j         j        S rV   r:  rl  ro   s    r>   r  z%MoshiForCausalLM.get_input_embeddings  s    z&&r=   c                     || j         _        d S rV   r  r  s     r>   r  z%MoshiForCausalLM.set_input_embeddings  s    "'
r=   c                     | j         S rV   r  ro   s    r>   get_output_embeddingsz&MoshiForCausalLM.get_output_embeddings  
    |r=   c                     || _         d S rV   r  r]   new_embeddingss     r>   set_output_embeddingsz&MoshiForCausalLM.set_output_embeddings  s    %r=   c                     || _         d S rV   r:  )r]   decoders     r>   set_decoderzMoshiForCausalLM.set_decoder  s    


r=   c                     | j         S rV   r  ro   s    r>   get_decoderzMoshiForCausalLM.get_decoder  s
    zr=   output_typerR  Nr   rL   rO   r   r1   rs  r   r   rt  ru  r   rv  num_logits_to_keepr   c                 L   ||n| j         j        }||n| j         j        }|	|	n| j         j        }	|                     |||||||||	|

  
        }|d         }|(t                      st                              d           |                     |dd| dddf                   }d}||	                                }|dddddf         
                                }|dddf         
                                }t                      }|                    d| j         j                  }|                    d          }|                    |j                  } |||          }|	s||f|dd         z   }||f|z   n|S t!          ||||j        |j        |j                  S )	a  
        Args:
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

            num_logits_to_keep (`int`, *optional*):
                Calculate logits for the last `num_logits_to_keep` tokens. If `0`, calculate logits for all
                `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
                token can save memory, which becomes pretty significant for long sequences or large vocabulary size.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MoshiForCausalLM

        >>> model = MoshiForCausalLM.from_pretrained("kmhf/hf-moshiko")
        >>> tokenizer = AutoTokenizer.from_pretrained("kmhf/hf-moshiko")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```N)
rL   rO   r   r1   rs  r   r   rt  ru  r   r   zStarting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32).r`   r#   )rA   r-   rB   r1   r0   r/   )r   r   rt  r  r:  r   r   r   r  rh   r   r
   r   rh  r   r   r@   r1   r0   r/   )r]   rL   rO   r   r1   rs  r   r   rt  ru  r   rv  r  r7  r0   r-   rA   shift_logitsshift_labelsr  rj   s                        r>   rk   zMoshiForCausalLM.forward  s   ^ 2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B] **)%+'/!5#)  
 
  
>":"<"<> [   mAAA0B/B/C/CQQQ,FGHH\\^^F!#ssAAA+.99;;L!#qrr'?5577L'))H',,R1GHHL',,R00L'??<+>??L8L,77D 	D F (,'7D7V##VC*+#3!/)
 
 
 	
r=   )NNNNNNNNNNNr   )r3   r4   r5   _tied_weights_keysrX   r  r  r  r  r  r  r   r  r    r@   _CONFIG_FOR_DOCr7   r:   r   r8   r	   r   r   r;   r	  rq   r   rk   rr   rs   s   @r>   r  r    s	       
 67GH    ' ' '( ( (  & & &     +*+IJJ+FUdeee '+1537KO59$(,0/3&*59-1"#f
 f
#f
 !.f
 u/0	f

 "%tE4E/F(F"GHf
   12f
 D>f
 $D>f
 'tnf
 d^f
 !!12f
 )*f
  f
 
u11	2f
 f
 f
 fe KJf
 f
 f
 f
 f
r=   r  zpThe original Moshi model with an audio encoder, a Moshi depth decoder and a Moshi decoder, for speech-to-speech.c            #           e Zd ZddgZeZdZdZdZdZ	def fdZ
d Zd Zd	 Z ee           eee
          	 	 	 	 	 	 	 	 	 	 	 	 	 	 d:deej                 deej                 deej                 deej                 deej                 deej                 deeej                          deej                 deej                 deej                 dee         dee         dee         dee         deeef         fd                        Z	 	 	 	 	 	 	 	 	 	 d;deej                 deej                 deej                 deej                 deej                 deej                 deej                 dee         dedefdZ ej                    	 	 	 	 	 	 	 	 	 d<deej                 deej                 deej                 deej                 deej                 deej                 d ee         d!ee         dee         dej        f fd"            Z 	 	 	 	 	 	 	 	 	 	 	 d=d#eej                 f fd$Z!	 	 d>d&e"d'e#e$e%f         d(ed)e&de#e$e%f         f
 fd*Z'd+ Z(d, Z)d- Z*d. Z+d/ Z,d0 Z-e.d1             Z/	 d?dej        d2e&d3e&d4e&fd5Z0d@d6Z1	 	 	 	 	 	 	 dAd7Z2e.deeej                          d8ej        deeej                          fd9            Z3 xZ4S )BMoshiForConditionalGenerationz!decoder.model.embed_tokens.weightzdecoder.lm_head.weightrL   Tr   c                    t                                                     t          j        fdt	          dj        z            D                       | _        t          j        j	        j
                  | _        t                    | _        j
        j        _        t!          j                  | _        j        | _        |                                  d S )Nc                 T    g | ]$}t          j        j        d z   j                  %S r^  r_  ra  s     r>   rc  z:MoshiForConditionalGeneration.__init__.<locals>.<listcomp>K  rd  r=   r!   )attn_implementation)rW   rX   rY   rj  rk  r   rl  r"   from_configaudio_encoder_configr-  audio_encoderr  r  depth_decoder_config_attn_implementation_internalr[  depth_decoderr  rr  s    `r>   rX   z&MoshiForConditionalGeneration.__init__G  s       MttttTYZ[^d^rZrTsTsttt
 
 '2'V=X
 
 
 (//DJD_#A.v/JKK#1r=   c                     | j         S rV   )r  ro   s    r>   get_audio_encoderz/MoshiForConditionalGeneration.get_audio_encoderX      !!r=   c                     | j         S rV   r  ro   s    r>   get_depth_decoderz/MoshiForConditionalGeneration.get_depth_decoder[  r  r=   c                     | j         S rV   )r  ro   s    r>   r  z)MoshiForConditionalGeneration.get_decoder^  r  r=   r  NrO   user_input_valuesrM   moshi_input_valuesrN   r1   rs  text_labelsaudio_labelsr   r   rt  ru  r   c                 
    ||n j         j        }d |                                D             }d |                                D             }d |                                D             }||"|   j        j        |fd j        i|d         }|"|   j        j        |fd j        i|d         }t          j        ||gd          |t          d	          | j	        j
                            |          }Ut           fd
t          j        d                   D                       }||n||                    |j                  z   }  j	        d||||||d|	d|}|j        }d}|j        }|	|
                     |
 j         j         j         j        |
j        d         dz             d         }
|	                    dd          }	|
                    dd                              d|
j        d                   }
t          j        |	|
gd          }|ddddf         }|                    dd|j        d                   }  j        d||||
d|}||j        z  }|s/|                                }|||                                z  }|S t5          |j        |j        ||j        |j        |j        |dn|j        |dn|j        |dn|j        |dn|j        |dn|j                  S )a  
        Returns:

        Examples:
        ```python
        >>> from transformers import MoshiForConditionalGeneration
        >>> import torch

        >>> model = MoshiForConditionalGeneration.from_pretrained("kmhf/hf-moshiko")
        >>> inputs = moshi.get_unconditional_inputs()

        >>> logits = model(**inputs, ).logits
        >>> logits.shape  # (bsz, seq_len, text_vocab_size)
        torch.Size([1, 1, 32000])
        ```Nc                 j    i | ]0\  }}|                     d           |t          d                    |1S )audio_encoder_
startswithlenrb  argumentr  s      r>   
<dictcomp>z9MoshiForConditionalGeneration.forward.<locals>.<dictcomp>  sQ      
  
  
%""#344 
S)**+U 
  
  
r=   c                 n    i | ]2\  }}|                     d           |t          d           d         |3S )decoder_Nr  r   s      r>   r  z9MoshiForConditionalGeneration.forward.<locals>.<dictcomp>  sT     
 
 
3B8UX`XkXklvXwXw
S__&&'
 
 
r=   c                 n    i | ]2\  }}|                     d           |t          d           d         |3S depth_decoder_Nr  r   s      r>   r  z9MoshiForConditionalGeneration.forward.<locals>.<dictcomp>  sW      
  
  
%""#344 
S)**,,-u 
  
  
r=   num_quantizersr   r#   r   z`You must provide at least one of `input_ids`, `inputs_embeds`, `input_values` and `audio_codes`.c                 P    g | ]"} j         |         d d |f                   #S rV   r  rb  codebookr2   r]   s     r>   rc  z9MoshiForConditionalGeneration.forward.<locals>.<listcomp>  9    wwwx0T&x0QQQ[1IJJwwwr=   T)rO   rs  r   rt  r   r1   ru  rv  r`   bos_token_idr  
max_lengthr!   )rB   rL   rO   rv  )rA   r-   rB   r1   r0   r/   rE   rF   rG   rH   rI   r<   )r   r  itemsr  encoder   r7   r   r   r  r:  rl  sumrk  rn   r   r   rB   rA   build_delay_pattern_maskr`  r   r}   r   r  to_tuplerD   r-   r1   r0   r/   )r]   rL   rO   r  rM   r  rN   r1   rs  r  r  r   r   rt  ru  r  kwargs_audio_encoderkwargs_decoderkwargs_depth_decoderaudio_inputs_embedsdecoder_outputsdecoder_last_hidden_statedepth_decoder_outputs
final_lossdepth_input_idsr7  r2   s   `                         @r>   rk   z%MoshiForConditionalGeneration.forwarda  sC   F &1%<kk$+B] 
  
#)<<>> 
  
  

 
FLllnn
 
 
 
  
#)<<>> 
  
  
   ,1A1I#<4#5#<%$ $6:6H$L`$ $$  "-2C2K$=D$6$=&% %7;7I%Ma% %%!  )%68H$IqQQQK [%8 v   $ $ 2 ? ?	 J J&&)wwwww[`alarstau[v[vwww' '#
 %, (',}/?/?@S@Z/[/[[  '$, 

)'/!5+

 

 

 

 %4$E! $$)
"|'?  88![9![9'-b1A5	 9  
 L &**2q11K (11!Q77??LDVWXDYZZL#il(CKKKO-aaa"f5O )B(F(Fr1NgNmnpNq(r(r%$6D$6 %";)-#	% %
 '% %! /44J 	%..00G$0099;;;N7 %")7+;)7&14<ttBWB\!6!>DYD`*9*A$$Gf(7(?_Eb%4%<TT/B\
 
 
 	
r=   Fgeneration_configapply_delay_pattern_maskconcat_unconditional_inputsc                 B    d }d }||||||t          d          ||	rV|)|' j                            | j                  d         }|)|' j                            | j                  d         }||
r                     |j        d                   }t          j        |j        |gd          }t          j        |j	        |gd          }t          j        |j
        |gd          }|t          j        |j        |gd          }||	rr|	r7|5                     | j        j         j        j        |j                  \  }}|	r7|5                     | j        j         j        j        |j                  \  }}|d }|O|Mt          j        ||gd          t!           fd	t#          j        d                   D                       }nq|7|t!           fd
t#          j        d                   D                       }n8|6|t!           fdt#          j        d                   D                       }| j        j                            |          }|!||n||                    |j                  z   }|||||||fS )NzYou must provide at least one of `input_ids`, `user_input_values`, `moshi_input_values`, `user_audio_codes`, `moshi_audio_codes` or `inputs_embeds`.r  r   )num_samplesr!   r   r#   r  c                 P    g | ]"} j         |         d d |f                   #S rV   r  r
  s     r>   rc  zWMoshiForConditionalGeneration._prepare_inputs_embeds_for_generation.<locals>.<listcomp>?  r  r=   c                 P    g | ]"} j         |         d d |f                   #S rV   r  r
  s     r>   rc  zWMoshiForConditionalGeneration._prepare_inputs_embeds_for_generation.<locals>.<listcomp>D  r  r=   c                 `    g | ]*} j         |         d d |j        z   f                   +S rV   )rl  r   r
  s     r>   rc  zWMoshiForConditionalGeneration._prepare_inputs_embeds_for_generation.<locals>.<listcomp>I  sQ       $ 4)(3K8dN`C`@`4abb  r=   )r   r  r  r   get_unconditional_inputsrn   r7   r   rN   rM   rL   rO   r  r   r`  r  r  rk  r  r:  rl  r   r   )r]   rL   r  rM   r  rN   rs  rO   r  r  r   user_delay_pattern_maskmoshi_delay_pattern_maskunconditional_inputsr  r2   s   `              @r>   %_prepare_inputs_embeds_for_generationzCMoshiForConditionalGeneration._prepare_inputs_embeds_for_generation  s    #'#'  !!!) ("*!) g  
  $<  ,1A1I#'#5#<#<=N_c_q#<#r#rst#u !-2C2K$($6$=$=>Paeas$=$t$tuv$w! %@ #'#@#@M]McdeMf#@#g#g  %	+?+QSd*ekl m m m$y*>*OQa)bhijjj	#7#A9"MSTUUUI)!&,@,OQ_+`fg!h!h!h $< ' ,<,H<@<Y<Y$!%!=!%!=0;	 =Z = =9 "9 ( ,=,I>B>[>[%!%!=!%!=0;	 ?\ ? ?;!#;  "&+0A0M#i):<L(MSTUUU&)wwwww[`alarstau[v[vwww' '## #./&)wwwww[`alarstau[v[vwww' '## "-.&)    (-k.?.B(C(C  ' '# $ $ 2 ? ?	 J J". %, (',}/?/?@S@Z/[/[[  #$
 	
r=   return_audio_waveformsreturn_audio_codesc
                   
$ t          | d          rt          | j        d          si | j        _        d| j        v r| j        | j        _        nXd | j                                        D             d         $$fd| j                                        D             | j        _        d | j                                        D             | _        d 
D             }
fd|D             } | j        
                    d	d
          fi 
\  }
|                     |||||||	          \  }}}}	||n|}|	r|j        d         dz   n|j        d         }
	                    d          d
u o|j
        d
u}
	                    d          d
u o|j        d
u}|                     ||||dnd||          }t          |d          r|j        }n| j        dz   | j        dz   dd}|                    |           |}
                    dd
          }|                     ||||||||d|	
  
        \  }}}}}}}t#          j        |j        d         dt'          | j        j        | j        j        j        z            f| j        | j                  }| j                            || j                  d         }||n
	                    d          
d<   ||n
	                    d          
d<   t#          j        |t;          |j        |j                  d          | _         |j        dk    p|j!        }|j        dk    p|j"        } tG                      j$        d$||||||||d
}|s|s|r|j!        s|j%        S |S |s|}n|j%        }|j        dk    rt#          j        ||j        d          }|j        dk    r|j&        d
d
d
|j        d          f         }| j         d
d
d
d
|j        d         d
f         }|d
d
d
df         '                    d          (                    d| j        d          }t#          j)        |d|          }t#          j        ||j        d          }t#          j*        ||fd           | _         t#          j+        | j,        d|d
d
df                   | _,        | j,        -                    dd| j,        j        d                   } | j        j$        d$||d
d
dd
f         -                    dd          d!|} | d
d
dd
f         '                    d           } t#          j*        | j         | gd           | _         | .                    | j         |          }!|| j        j/        k    }"|!|"         0                    |"j        d         | j        d          }!d
}#|r| j        1                    |!          j2        }#|r|!nd
}!|j!        rtg          d$|#|!d"|S tg          |#||!#          S )%as  
        Generates sequences of text token ids and audio tokens ids.

        Parameters:
            input_ids (`torch.Tensor `of shape `(batch_size, sequence_length), *optional*):
                The sequence used as a text prompt for the generation.
            user_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*):
                The audio waveforms used as audio user prompt for the generation.
            user_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*):
                The audio codes used as audio user prompt for the generation. Has priority over `user_input_values` and represents the audio "tokens" of `user_input_values` once passed through the audio encoder.
            moshi_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*):
                The audio waveforms used as audio Moshi prompt for the generation.
            moshi_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*):
                The audio codes used as audio Moshi prompt for the generation. Has priority over `moshi_input_values` and represents the audio "tokens" of `moshi_input_values` once passed through the audio encoder.
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` and the audio inputs you can choose to directly pass an embedded representation. This
                is useful if you want more control over how to convert the inputs into associated vectors than the
                model's internal embedding lookup matrix.
            return_audio_waveforms (`bool`, *optional*, defaults to `True`):
                If `False`, won't generate the audio waveforms.
            return_audio_codes (`bool`, *optional*):
                If `True`, will also returns the generated audio codes, i.e the intermediate audio "tokens" which transforms to `audio_sequences` once passed through the audio decoder.
            concat_unconditional_inputs (`bool`, *optional*, defaults to `True`):
                If `False`, won't concatenate initial audio and text tokens.
            kwargs (`Dict[str, Any]`, *optional*):
                Remaining dictionary of keyword arguments that are passed to the `generate` method. Refers to the
                original [`generate` docstrings](https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate)
                for more information on how to use them.
                Note that keywords with a *depth_* prefix will be input for the `generate` method of the
                depth decoder. Otherwise, the latter will use its default generation config.
        Return:
            [`MoshiConditionalGenerationGenerateOutput`]
        hf_device_map c                     g | ]}|d v|	S )r   diskr<   )rb  ds     r>   rc  z:MoshiForConditionalGeneration.generate.<locals>.<listcomp>  s#    bbbQRaIaIaqIaIaIar=   r   c                 |    i | ]8\  }}|                     d           |t          d           d         |dv rn|9S )r  Nr2  r  )rb  keyr  main_devices      r>   r  z:MoshiForConditionalGeneration.generate.<locals>.<dictcomp>  sc     4 4 4"U~~o664O,,../@X@X^c4 4 4r=   c                 D    i | ]\  }}|                     d           ||S r  r  )rb  r6  r  s      r>   r  z:MoshiForConditionalGeneration.generate.<locals>.<dictcomp>  s@     " " ")sEs~~^mOnOn"U" " "r=   c                 <    h | ]}|                     d           |S )r  r9  )rb  r  s     r>   	<setcomp>z9MoshiForConditionalGeneration.generate.<locals>.<setcomp>  s,    $n$n$n(H[H[\lHmHm$nX$n$n$nr=   c                 f    i | ]-}|t          d           d                             |          .S r  )r  pop)rb  r  r  s     r>   r  z:MoshiForConditionalGeneration.generate.<locals>.<dictcomp>  sH      
  
  
HPHS)**,,-vzz(/C/C 
  
  
r=   r  N)rL   r  rM   r  rN   rs  r   r`   r#   r  
min_lengthrs  rL   )r  has_default_max_lengthhas_default_min_lengthmodel_input_nameinputs_tensorinput_ids_lengthr  r  )r>  r  cache_implementationrO   T)
rL   r  rM   r  rN   rs  rO   r  r  r   r   r   r"  r(  r)  r   )rs  rL   r  blank_user_audio_codesr  return_dict_in_generateoutput_scoresrO   )rS   indexr!   rB   rL   )r)   r2   )r)   r*   r2   r<   )4r  r  r/  valuesr  _prepare_generation_configr=  !_check_and_maybe_initalize_inputsrn   getr  r>  _prepare_generated_lengthr  r   r   r+  r7   zerosrq   r   sampling_rater  
frame_rater   r   r  r  repeat_interleavemax	num_beamsnum_return_sequencesgenerated_audio_codesrG  rH  rW   generater*   r.   r   r   gatherr   r|   rB   r   r  r`  r   decodeaudio_valuesr(   )&r]   rL   r  rM   r  rN   rs  r,  r-  r   r  depth_decoder_kwargs_keysr  r  inputsrC  r?  r@  depth_decoder_generation_configrO   r(  r)  blank_input_valuesrF  rG  rH  r7  output_text_idsr.   rW  expanded_beam_indicesrB   last_generated_audio_codesoutput_audio_codesmaskoutput_valuesr7  r^   s&             `                         @r>   rX  z&MoshiForConditionalGeneration.generatec  s   ` 4)) 	'$:Lo2^2^ 	/1D,T'''373E"00bb$*<*C*C*E*Ebbbcde4 4 4 4&*&8&>&>&@&@4 4 4"0" "-1-?-E-E-G-G" " "D %o$nf$n$n$n! 
  
  
  
Tm 
  
  

 %DD$CFJJObdhDiDi$t$tms$t$t!6 22#"3!1#5"3+,G 3   	T	#%68S #,"33Nd6<+a//TZT`acTd!'L!9!9T!A!nFWFbjnFn!'L!9!9T!A!nFWFbjnFn ::/#9#9090A__{ - ; 
 
 $&<== 	.?.T++ #014"014(8/ /+ 	(../CDDD>$4d;; 66/-1/')/%)(C 7 
 
	
#$ #[ #QDK,EHhHs,s(t(tu*;
 
 

 "&!3!:!:;M^b^p!:!q!qrs!t (?'J##PVPZPZ[tPuPu 	()
 (3 %$677 	)* &+%<s#4#>@Q@fggmn&
 &
 &
" #4"="A"nEVEn)3a7Z;L;Z"%''" 

'/#9!5$;')

 

 

 

 & 	.@ 	& )/@/X )((N ' 	0%OO%/O1A55','>(*;*PVW( ( ($ &** #/3Q6G6Mb6Q5Q3Q0QRL$($>qqq!!!EVE\]_E`EbEb?b$c! %1CRC$8$B$B1$E$E$L$LRQUQceg$h$h!$)L1FAUj$k$k$k! !& 7!#4#Iq! ! ! */4EG\3]cd)e)e)eD& &+%78NTU]ijkjkjkmojo]p%q%q%qD" !277At?U?[\^?_``%@T%7%@ &
/%aaaf-222q99&
 &
 #&
 &
" &@122%F%P%PQR%S%S"%*Y0JLf/gmn%o%o%o" "::4;UWopp (4;+GG/5==djmTM_acdd! 	 .55"   4FO//44 	;  -;M QX   8)_Rd
 
 
 	
r=   rF  c                 ,    t                      j        d||||||||d|}|d         g|                    d          }|                    dd|j        d                   }|                    d          } | j        j        d||                    dd          d|}|d d dd f                             d          }| 	                    t          j        | j        |                    | j        j                  gd          |	          d d d d dd f         }| 	                    t          j        | j        |gd          |
          | _        |                     || j        d d d d dd f         |	          \  }}}}}}}d |d<   ||d
<   |S )N)rL   r1   rO   rs  r   r   r   r  rL   rB   r`   r#   rJ  r!   r   )rN   rM   rs  r<   )rW   prepare_inputs_for_generationrN  r   rn   r=  r  rX  r   r  r7   r   rW  r   r   r+  )r]   rL   r1   rO   rs  r   r   r   r  r(  r)  r  rF  r  model_inputsrB   rW  rM   r   r^   s                      r>   rg  z;MoshiForConditionalGeneration.prepare_inputs_for_generation[	  s   $ =uww< 

+)')%1

 

 

 

 $0 &

+> ? ? 1 6 6r1>O>UVX>Y Z Z$((55I$?D$6$? %"3#..Q//% % '% %! %:!!!QRR%$@$J$J1$M$M!#<<	/1G1J1J4KeKl1m1mntu   (	   
 aaBCCi  *.)F)F	457LMSTUUUWo* *D& /3.X.XT-G111bcc	-Reu /Y / /+M1aAq! )-L%,9L)r=   r#   r7  model_kwargsis_encoder_decodernum_new_tokensc                     t                                          ||||          }|                    d          d d dd f         |d<   |                    d          d d dd f         | _        |S )NrB   r`   )rW   #_update_model_kwargs_for_generationrN  rB   )r]   r7  ri  rj  rk  r^   s        r>   rm  zAMoshiForConditionalGeneration._update_model_kwargs_for_generation	  s     wwBB\#5~
 

 -4KK8K,L,LQQQPRPSPSV,T() ")-@!A!A!!!RSS&!Ir=   c                 4    | j                                         S rV   )r  r  ro   s    r>   r  z2MoshiForConditionalGeneration.get_input_embeddings	  s    |00222r=   c                 :    | j                             |           d S rV   )r  r  r  s     r>   r  z2MoshiForConditionalGeneration.set_input_embeddings	  s    ))%00000r=   c                 4    | j                                         S rV   )r  r  ro   s    r>   r  z3MoshiForConditionalGeneration.get_output_embeddings	  s    |11333r=   c                 :    | j                             |           d S rV   )r  r  r  s     r>   r  z3MoshiForConditionalGeneration.set_output_embeddings	  s    **>:::::r=   c                 d    | j                                         D ]	}d|_        
d| j         _        dS )z3
        Freeze the audio encoder weights.
        FN)r  
parametersrequires_grad_requires_gradr]   params     r>   freeze_audio_encoderz2MoshiForConditionalGeneration.freeze_audio_encoder	  >     '2244 	( 	(E"'E,1)))r=   c                 d    | j                                         D ]	}d|_        
d| j         _        dS )z3
        Freeze the depth encoder weights.
        FN)r  rs  rt  ru  rv  s     r>   freeze_depth_decoderz2MoshiForConditionalGeneration.freeze_depth_decoder	  ry  r=   c                 l    | j         d         }|dd|f         }t          j        |dk    | |          } | S )zApply a delay pattern mask to the decoder input ids, only preserving predictions where
        the mask is set to -1, and otherwise setting to the value detailed in the mask.r`   .N)rn   r7   where)rL   decoder_pad_token_maskseq_lens      r>   r  z6MoshiForConditionalGeneration.apply_delay_pattern_mask	  sC    
 /"%!7XgX!FK 6" <iI_``	r=   r  r  r  c                 x   |j         \  }}}||n| j        j        }t          j        |||ft          j        |j                  dz  }t          ||dz
            }	|dddd|	f         |dddd|	f<   |ddddd|	f         |ddddd|	dz   f<   ||dddddf<   ||ddddf<   |}
|dd|	f         }||
fS )aH  Build a delayed pattern mask to the input_ids. Each codebook, except the first one, is offset by
        one, giving a delayed pattern mask at the start of sequence and end of sequence. Take the example where there
        are 4 codebooks and a max sequence length of 6, we have the delayed pattern mask of shape `(codebooks,
        seq_len)`:
        - [-1, -1, -1, -1, -1,  P]
        - [ B, -1, -1, -1, -1, -1]
        - [ B, -1, -1, -1, -1, -1]
        - [ B, -1, -1, -1, -1, -1]
        where B is the begining-of-sentence token, P is the special padding token id and -1 indicates that the token is valid for prediction. If we include
        a prompt (input ids), the -1 positions indicate where new tokens should be predicted. Otherwise, the
        mask is set to the value in the prompt:
        - [ a0, a1, -1, -1, -1,  P]
        - [ B,  b0, b1, -1, -1, -1]
        - [ B,  c0, c1, -1, -1, -1]
        - [ B,  d0, d1, -1, -1, -1]
        where a-d indicate the codebook channel and 0/1 indicates the temporality. Now, we only override the -1
        tokens in our prediction.
        NrE  r`   r#   r   .)rn   r  r  r7   r[   longr   r  )r]   rL   r  r  r  r   r   r  input_ids_shiftedseq_len_to_keeppattern_masks              r>   r  z6MoshiForConditionalGeneration.build_delay_pattern_mask	  s1   * '0o#]G#-#9ZZt?U?`
J]J7uzR[Rbcccfhh 	
 gzA~664=aaaDT_DT>T4U!!!Q 0 001 =FaaaM]oM]F]<^!!!QRR_q%8!889 '3!!!QRR(#&2!!!Q(# )%c+;O+;&;<	,&&r=   c                    t          j        |df| j        t           j                  | j        j        z  }t          j        || j        df| j        t           j                  | j        j        z  }t          j        || j        df| j        t           j                  | j        j        z  }t          j        |df| j        t           j                  }t          ||||          S )a  
        Helper function to get null inputs for unconditional generation, enabling the model to be used without the
        feature extractor or tokenizer.

        Args:
            num_samples (int, *optional*):
                Number of audio samples to unconditionally generate.
            max_new_tokens (int, *optional*):
                Number of tokens to generate for each sample. More tokens means longer audio samples, at the expense of
                longer inference (since more audio tokens need to be generated per sample).

        Example:
        ```python
        >>> from transformers import MoshiForConditionalGeneration

        >>> model = MoshiForConditionalGeneration.from_pretrained("kmhf/hf-moshiko-pytorch-bf16")

        >>> # get the unconditional (or 'null') inputs for the model
        >>> unconditional_inputs = model.get_unconditional_inputs(num_samples=1)
        >>> audio_samples = model.generate(**unconditional_inputs, max_new_tokens=256)
        ```r#   )r   r   )rL   rM   rN   rO   )
r7   r[   r   r   r   rh  r   r`  r  rK   )r]   r#  rL   rM   rN   rO   s         r>   r'  z6MoshiForConditionalGeneration.get_unconditional_inputs
  s    . JQ/5;WWWZ^ZeZpp	JT%7;DKW\Wbccck*+ 	
 JT%7;DKW\Wbccck*+ 	 [!$4T[PUPZ[[[&-/)	
 
 
 	
r=   c                 Z   ||n|}||n|}	||n|}
|	d up|
d up|d u}|o|d uo|d u  }|r|	t          d          |r|
t          d          |r|t          d          |s,|                                 }|j        }|j        }|j        }d}n|	j        d         }|
j        d         }|j        d         }| j        j        j        | j        j	        z  }|t          j        ||z            n|}|t          j        ||z            n|}||k    s||k    rt          d          ||||fS )Na  No user audio inputs have been passed alongside the other inputs. Make sure either `user_input_values` or `user_audio_codes` is passed or use `MoshiForConditionalGeneration.get_unconditional_inputs`. Check the `MoshiForConditionalGeneration` docstrings for more information.a  No Moshi audio inputs have been passed alongside the other inputs. Make sure either `moshi_input_values` or `moshi_audio_codes` is passed or use `MoshiForConditionalGeneration.get_unconditional_inputs`. Check the `MoshiForConditionalGeneration` docstrings for more information.zNo `input_ids` or `inputs_embeds` have been passed alongside the other inputs. Make sure `input_ids` is passed or use `MoshiForConditionalGeneration.get_unconditional_inputs`. Check the `MoshiForConditionalGeneration` docstrings for more information.Fr`   r#   zAt least one of the 3 inputs of `MoshiForConditionalGeneration` doesn't have the same sequence length as the others.Make sure that they all have the same sequence length. Check the `MoshiForConditionalGeneration` docstrings for more information.)r   r'  rL   rM   rN   rn   r   r  rR  rQ  r   ceil)r]   rL   r  rM   r  rN   rs  r   r]  
user_inputmoshi_inputone_input_has_been_passedr*  user_seq_lengthmoshi_seq_lengthtokens_seq_lengthratios                    r>   rM  z?MoshiForConditionalGeneration._check_and_maybe_initalize_inputs-
  s    ,3):)B%%HY
+=+E''K]%/t%;$qTXAX$q^dlp^p! 'B '
%;)t*;G
#
 % #	**< e   ' 	K,? h   ' 	FN M   + 	#'#@#@#B#B ,6I3D 4 F +0'' ).r2O*04 &QK4?$+B[[EFWF_ty)9E)ABBBeuDTD\di%(?@@@bqO $4448I_8\8\ X  
 *,=?ZZZr=   beam_idxc                 :    t          fd| D                       S )a  
        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
        [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
        beam_idx at every generation step.
        c              3   N   K   | ]}t          fd |D                       V   dS )c              3   t   K   | ]2}|                     d                     |j                            V  3dS )r   N)r|   r   r   )rb  
past_stater  s     r>   r~  zIMoshiForConditionalGeneration._reorder_cache.<locals>.<genexpr>.<genexpr>t
  sC      jjQ[*))!X[[9J-K-KLLjjjjjjr=   Nrm   )rb  
layer_pastr  s     r>   r~  z?MoshiForConditionalGeneration._reorder_cache.<locals>.<genexpr>s
  sU       
 
 jjjj_ijjjjj
 
 
 
 
 
r=   r  )r1   r  s    `r>   _reorder_cachez,MoshiForConditionalGeneration._reorder_cachej
  s=      
 
 
 
-
 
 
 
 
 	
r=   )NNNNNNNNNNNNNN)
NNNNNNNNFF)	NNNNNNTNT)NNNNNTNNNNN)Fr#   rV   r^  )NNNNNNN)5r3   r4   r5   r  r$   rR  rY  rT  rV  rW  rX   r  r  r  r   MOSHI_INPUTS_DOCSTRINGr    r   r  r   r7   r:   r  r;   r8   r   r	  r	   rk   r   r+  r   rX  rg  r   r   r   r   rq   rm  r  r  r  r  rx  r{  r  r  r  r'  rM  r  rr   rs   s   @r>   r  r  :  s2        >?WXL!O&*#!N{      "" " "" " "   +*+ABB?YYY 15599=37:>48;?592637$(,0/3&*W
 W
E,-W
 !!12W
 $E$56	W

 #5<0W
 %U%67W
 $EL1W
 uU%678W
   12W
 e./W
 u/0W
 D>W
 $D>W
 'tnW
 d^W
" 
uo%	&#W
 W
 W
 ZY CBW
v 159=37:>4859158<).,1e
 e
E,-e
 $E$56e
 #5<0	e

 %U%67e
 $EL1e
   12e
 !.e
 $$45e
 #'e
 &*e
 e
 e
 e
N U]__ 159=37:>485915-16:u
 u
E,-u
 $E$56u
 #5<0	u

 %U%67u
 $EL1u
   12u
 !)u
 %TNu
 &.d^u
 
	u
 u
 u
 u
 u
 _u
t  $!%!>BB B !)): ;B B B B B BP $)  38n !	
  
c3h     $3 3 31 1 14 4 4; ; ;2 2 22 2 2   \ dh+' +')+'9<+'LO+']`+' +' +' +'Z'
 '
 '
 '
V $(;[ ;[ ;[ ;[z 
uU\23
?D|
	uU\"	#
 
 
 \
 
 
 
 
r=   r  )r  r  r  r9  )Nr#   )Wr6   r   dataclassesr   typingr   r   r   r   r   r	   r7   torch.nnrY   torch.utils.checkpointr
   activationsr   cache_utilsr   r   r   r   
generationr   r   modeling_attn_mask_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   pytorch_utilsr   utilsr   r   r   r   r   r   r    auto.modeling_autor"   configuration_moshir$   r%   modeling_flash_attention_utilsr&   
get_loggerr3   r   r  _CHECKPOINT_FOR_DOCr(   r@   rD   rK   ModulerQ   r  ru   r   r   r   r   r   r8   rq   r   r   r  r  r,  r)  r9  MOSHI_START_DOCSTRINGr  r  r[  r  r  r  __all__r<   r=   r>   <module>r     s      ! ! ! ! ! ! : : : : : : : : : : : : : : : :            % % % % % % ! ! ! ! ! ! O O O O O O O O O O O O        ? > > > > >            . - - - - - 1 1 1 1 1 1                  + * * * * * > > > > > > > >  KJJJJJJ		H	%	%'  .3 .3 .3 .3 .3{ .3 .3 .3b #? #? #? #? #?+ #? #? #?L 3E 3E 3E 3E 3E{ 3E 3E 3El , , , , ,k , , ,*= = = = =29 = = =&   L ) ) )$ $ $ $ $") $ $ $N" " " " "") " " "&< < < < <29 < < <<( ( (   6    RY   4	UU\ 	U# 	U%, 	U 	U 	U 	Us9 s9 s9 s9 s9RY s9 s9 s9np9 p9 p9 p9 p9> p9 p9 p9hZ1 Z1 Z1 Z1 Z1 Z1 Z1 Z1| -  P P P P P	 P P Pf ?  ?  ?  ?  ??  ?  ?  ?F  A FH" Vz z z z z,o z z zz Y i i i i i% i i	 iX	 _ G
 G
 G
 G
 G
+_ G
 G
	 G
T  
w
 w
 w
 w
 w
$8/ w
 w
 
w
t f
e
er=   