gUdZddlmZmZmZmZddlZddlZddlmZddl m Z ddl m Z ddl mZdd lmZmZmZd d lmZejeZd ZGd dejZdZd#dZGddejZGddejZGddejZGddejZ GddejZ!dZ"Gdde Z#dZ$dZ%ed e"Gd!d"e#Z&dS)$zPyTorch Pixtral model.)ListOptionalTupleUnionN)nn)PreTrainedModel)ACT2FN)BaseModelOutput)add_start_docstrings%add_start_docstrings_to_model_forwardlogging)PixtralVisionConfigcg}|D]}|jdd\}}tjtj|tj|d}tj|ddddd\}}||z|z} || dddftj|S)Nij)indexingdimr) shapetorchmeshgridarangestackreshapechunkappendcat) patch_embeds_list max_width positionspatchheightwidthmeshh_gridv_grididss h/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/pixtral/modeling_pixtral.pyposition_ids_in_meshgridr-%sI"$$ BCC( ~el622EL4G4GRVWWWTr222::2qAAGG2NNy 6)QQQT#### 9Y  cXeZdZdZfdZejdZdZxZ S)PixtralRotaryEmbeddinga The key with pixtral embedding is just that you have a frequency for each pixel positions. If you have height x width pixels (or embedding pixels), then the frequency used for ROPE is given by indexing the pre_computed frequency on the width and height. What you output is of dimension (batch, height * width, dim) with dim the embed dim. This simply means that for each image hidden state, you are going to add a corresponding positional embedding, based on its index in the grid. ctd|_|j|_|j|_|j|jz}d|jtj d|jd |jz zz }tj ||j }tj ||j }tj ||ddd }tj ||ddd }tj|dddddfd|d|dddddf|ddgdd|jdz} |d tj| | fdd dS) Ndefaultg?rrdevicerrrinv_freqF persistent)super__init__ rope_typehead_dimr rope_thetabase image_size patch_sizerrfloatr4outerr!repeatrregister_buffer) selfconfigr4max_patches_per_sidefreqshwfreqs_hfreqs_wr5 __class__s r,r9zPixtralRotaryEmbedding.__init__<s "?% %0F4EEtyU\!TXq%A%A%G%G%I%IDH%TUV L-el C C C L-el C C C+asss,,2244+aqt!t--335594 #**1.BAFFaaa #**+?AFF     '"dh!m $ $  ZHh3GR)P)P)P]bcccccr.cd|jvr|||j|j|}|jj}t |t r|dkr|nd}tj|d5|}| }| }dddn #1swxYwY| |j | |j fS)Ndynamicr3mpscpuF) device_typeenabled)dtype) r:_dynamic_frequency_updater4r5type isinstancestrrautocastcossintorS)rDx position_idsrGrQembrYrZs r,forwardzPixtralRotaryEmbedding.forwardUs  & &  * *< * I I I l+hm %/ S%A%AekUZFZFZkk`e ^ U C C C  C''))C''))C               vvAGv$$cff17f&;&;;;s4+B++B/2B/c^tj|dz}||jkrB|j|j|fd|i|j\}|_|d|d||_||jkr;|j|jkr-|d|j d|j|_dSdSdS)a dynamic RoPE layers should recompute `inv_freq` in the following situations: 1 - growing beyond the cached sequence length (allow scaling) 2 - the current sequence length is in the original scale (avoid losing precision with small sequences) rseq_lenr5Fr6N) rmaxmax_seq_len_cached rope_init_fnrE rope_kwargsattention_scalingrCoriginal_max_seq_lenoriginal_inv_freq)rDr]r4rar5s r,rTz0PixtralRotaryEmbedding._dynamic_frequency_updatefs )L))A- T, , ,/@t/@ V00-408<8H00 ,Hd,  X% H H H&-D # T. . .43JTMf3f3f  T-CPU V V V&*&?D # # # / .3f3fr.) __name__ __module__ __qualname____doc__r9rno_gradr_rT __classcell__rLs@r,r0r00s  ddddd2U]__<<_< @@@@@@@r.r0c|dd|jddzf}|d|jddzdf}tj| |fdS)z*Rotates half the hidden dims of the input..Nrrr)rrr!)r\x1x2s r, rotate_halfrszs] 3"!'"+"" " #B 3 q "" " #B 9rc2YB ' ' ''r.c||}||}||zt||zz}||zt||zz}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. ) unsqueezers)qkrYrZr] unsqueeze_dimq_embedk_embeds r,apply_rotary_pos_embr{sc( -- & &C -- & &C3w;q>>C/0G3w;q>>C/0G G r.ceZdZdZfdZ d dejdeejdeejdeed e ejeejff d Z xZ S) PixtralAttentionz=Multi-headed attention from 'Attention Is All You Need' paperct||_|j|_|j|_|j|jz|_|jdz|_|j |_ tj |j|jd|_ tj |j|jd|_tj |j|jd|_tj |j|jd|_dS)NgFbias)r8r9rE hidden_size embed_dimnum_attention_heads num_headsr;scaleattention_dropoutdropoutrLineark_projv_projq_projo_projrDrErLs r,r9zPixtralAttention.__init__s  +3$.8 ]D( / iUKKK iUKKK iUKKK iUKKK r.NF hidden_statesattention_maskposition_embeddingsoutput_attentionsreturncV|\}}}||}||} ||} ||||j|jdd}| |||j|jdd} | |||j|jdd} |\} } t|| | | d\}} tj || dd|j z} || |z} tj | dtj|j} tj | |j|j } tj | | }|dd}|||d}||}|| fS) z#Input shape: Batch x Time x Channelrrr)rxrNr)rrS)ptraining)sizerrrviewrr; transposer{rmatmulrr functionalsoftmaxfloat32r[rSrr contiguousrr)rDrrrr batch_sizepatches_ query_states key_states value_statesrYrZ attn_weights attn_outputs r,r_zPixtralAttention.forwards"/!3!3!5!5 GQ{{=11 [[// {{=11 #((Wdndm\\ffghjkll __Z$.$-XXbbcdfghh #((Wdndm\\ffghjkll &S#7 jRUWZjk#l#l#l j|L*2F2Fq!2L2LMMPTPZZ  %'.8L},,\r,WWZZ[g[mnn },,\T\TXTa,bb l<>> !++Aq11<<>> !))*grBB kk+.. L((r.)NNF) rirjrkrlr9rTensorrboolrr_rnros@r,r}r}sGG L L L L L$266:,1 %)%)|%)!.%)&el3 %) $D> %) u|Xel33 4 %)%)%)%)%)%)%)%)r.r}c$eZdZfdZdZxZS) PixtralMLPct|j|_|j|_t j|j|jd|_t j|j|jd|_t j|j|jd|_t|j |_ dS)NFr) r8r9rintermediate_sizerr gate_projup_proj down_projr hidden_actact_fnrs r,r9zPixtralMLP.__init__s !-!'!94#3T5KRWXXXy!143IPUVVV 4#94;KRWXXXV./ r.c||||||zSN)rrrr)rD hidden_states r,r_zPixtralMLP.forwards>~~dkk$..*F*FGG$,,WcJdJddeeer.)rirjrkr9r_rnros@r,rrsN00000fffffffr.rc,eZdZdfd ZdZdZxZS)PixtralRMSNormư>cttjt j||_||_dS)z= PixtralRMSNorm is equivalent to T5LayerNorm N)r8r9r Parameterronesweightvariance_epsilon)rDrepsrLs r,r9zPixtralRMSNorm.__init__sD l5:k#:#:;; #r.c|j}|tj}|ddd}|tj||jzz}|j||zS)NrrT)keepdim) rSr[rrpowmeanrsqrtrr)rDr input_dtypevariances r,r_zPixtralRMSNorm.forwards|#) %((77  $$Q'',,R,>>% Ht?T4T(U(UU {]--k::::r.cHt|jjd|jS)Nz, eps=)tuplerrr)rDs r, extra_reprzPixtralRMSNorm.extra_reprs& )**II$2GIIIr.)r)rirjrkr9r_rrnros@r,rrsb$$$$$$;;;JJJJJJJr.rc eZdZfdZ d dejdejdeejdeedeej f d Z xZ S) PixtralAttentionLayerctt|jd|_t ||_t||_t|jd|_ dS)Nh㈵>r) r8r9rrattention_normr feed_forwardr} attentionffn_normrs r,r9zPixtralAttentionLayer.__init__sk ,V-?TJJJ&v..)&11&v'9tDDD r.NFrrrrrc|}||}|||||\}}||z}|}||}||}||z}|f}|r||fz }|S)a= Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. )rrrr)rrrr)rDrrrrresidualroutputss r,r_zPixtralAttentionLayer.forwards"!++M:: &*nn') 3/ '5' ' # | !=0   m44 ))-88  =0  "  '   &Gr.NF) rirjrkr9rrrrr FloatTensorr_rnros@r,rrsEEEEE7;,1 %%|% %&el3 % $D> % u ! %%%%%%%%r.rceZdZfdZ d deejdeejdeedeedeedee e ff d Z xZ S) PixtralTransformerc&t||_tj|_t|jD])}|j t|*d|_ dSr) r8r9rErr ModuleListlayersrangenum_hidden_layersr rgradient_checkpointing)rDrErrLs r,r9zPixtralTransformer.__init__,s  h))++ v/00 > >A K  4V<< = = = =&+###r.Nrrroutput_hidden_states return_dictrc||n |jj}||n |jj}||n |jj}|rdnd}|rdnd}|} |jD]\} |r|| fz}|jr&|jr|| j| |||} n| | |||} | d} |r || dfz}]|r|| fz}|std| ||fDSt| | g|S)av Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Embeddings which serve as input to the Transformer. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. N)rrrrc3K|]}||V dSrr).0vs r, z-PixtralTransformer.forward..ws(eeqWXWdWdWdWdWdeer.)last_hidden_stater attentions) rErruse_return_dictrrr_gradient_checkpointing_func__call__rr ) rD inputs_embedsrrrrrencoder_statesall_attentionsr encoder_layer layer_outputss r,r_zPixtralTransformer.forward4s:2C1N--TXT_Tq$8$D $+Jj &1%>N fee]NN$Seeeee e+M?We    r.)NNNNN) rirjrkr9rrrrrrr r_rnros@r,rr+s,,,,,266:,0/3&*F F !.F &el3 F $D> F 'tn F d^F  uo% &F F F F F F F F r.ra[ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`PixtralVisionConfig`]): Model configuration class with all the parameters of the vision encoder. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. c.eZdZeZdZdZdgZdZdZ dZ dS)PixtralPreTrainedModelmodelTPixtralVisionAttentionpast_key_valuesc(t|jdr |jjn|jjj}t |t jt jfrJ|jj d||j |j j dSdSt |t j rS|jj d||j-|jj |j dSdSdS)Ninitializer_rangeg)rstd)hasattrrEr text_configrVrrConv2drdatanormal_rzero_ Embedding padding_idx)rDmodulers r, _init_weightsz$PixtralPreTrainedModel._init_weightss t{$788 ;DK ) )(: fry")4 5 5 ? M  & &CS & 9 9 9{&  &&((((('&  - - ? M  & &CS & 9 9 9!- "6#56<<>>>>> ? ?--r.N) rirjrkr config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_cache_classrrr.r,rrsI&L&*#12"3 ?????r.ra  Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`AutoImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. c|j}|j}|jd}tj|j}tj||f|||}tj|d}tjdg|ddzd}t||D]\} } d|| | | | f<|ddddddf |jdddd}|S)Nr) fill_valuerSr4rr) rSr4rrfinfominfulltensorcumsumzipexpand) r"r rSr4rad_min causal_mask block_end_idxblock_start_idxstartends r,generate_block_attention_maskrs  LE ]Fl1oG K   "E*gw/EW]^^^KL!233::2>>MlA3):3B3)?#?@@GGKKO/=99.. s,- E#IuSy())dD!!!QQQ./66v|A2rRRK r.z^The bare Pixtral vision encoder outputting raw hidden-states without any specific head on top.ceZdZdZfdZee d deej de e de e de e d e e eff d ZxZS) PixtralVisionModelvision_encodercPt|||_tj|j|j|j|jd|_t|jd|_ t||_ t||j|_dS)NF) in_channels out_channels kernel_sizestriderrrr3)r8r9rErr num_channelsrr? patch_convrln_prer transformerr0r4patch_positional_embeddingrs r,r9zPixtralVisionModel.__init__s     )++)$    %V%7TBBB -f55*@PTP[*\*\*\'''r.FN pixel_valuesrrrrcfd|D}tjd|Dd}|}t|jjjjzj} || } td|D|} || | S)z Returns: pixel_values: tensor of token features for all tokens of all images of shape (N_toks, D) cg|]B}|djCS)r)r rur[rS)rimgrDs r, z.PixtralVisionModel.forward..sBfffRUT__S]]1-=-=-@-@-L-LMMfffr.cbg|],}|dddd-S)rrr)flattenpermuterrs r,r(z.PixtralVisionModel.forward..s4![![![A!))A,,"6"6q!Q"?"?![![![r.rr)r#cDg|]}|jd|jdzS)rr)rr,s r,r(z.PixtralVisionModel.forward..s) B B B1QWR[172; & B B Br.) rr!r!r-rEr>r?r[r4r#rr") rDr$rrrargskwargsr" patch_embedsr]position_embeddingrs ` r,r_zPixtralVisionModel.forwards gfffYefffy![![IZ![![![abccc {{<00 0 )?4;CY)Y   "T[// "<<\<XX6 B B0A B B BL   n>PQQQr.)FNN)rirjrkrr9r PIXTRAL_INPUTS_DOCSTRINGrrrrrrrr r_rnros@r,rrs ) ] ] ] ] ]+*+CDD05,0&* RR5<(R'tnR$D> R d^ R uo% &RRREDRRRRRr.r)Nr)'rltypingrrrrrtorch.utils.checkpointrr activationsr modeling_outputsr utilsr r rconfiguration_pixtralr get_loggerriloggerr-Moduler0rsr{r}rrrrPIXTRAL_START_DOCSTRINGrr2rrrr.r,r>s//////////// !!!!!!////// 766666  H % %   F@F@F@F@F@RYF@F@F@T(((67)7)7)7)7)ry7)7)7)v f f f f f f f fJJJJJRYJJJ(-----BI---`O O O O O O O O d"?????_???2     d0R0R0R0R0R/0R0R 0R0R0Rr.