
    g                       d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z	ddl
Z	ddl	mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+m,Z,  e'j-        e.          Z/dZ0dZ1e G d de                      Z2 G d dej3                  Z4 G d dej3                  Z5 G d dej3                  Z6de5iZ7 G d dej3                  Z8 G d  d!ej3                  Z9 G d" d#ej3                  Z: G d$ d%ej3                  Z; G d& d'ej3                  Z< G d( d)e          Z=d*Z>d+Z? G d, d-ej3                  Z@ G d. d/ej3                  ZA G d0 d1ej3                  ZB G d2 d3ej3                  ZC G d4 d5ej3                  ZDd6ZE G d7 d8ej3                  ZF e%d9e>           G d: d;e=                      ZG G d< d=ej3                  ZH e%d>e>           G d? d@e=                      ZI e%dAe>           G dB dCe=e                      ZJdS )DzPyTorch GIT model.    N)	dataclass)ListOptionalTupleUnion)nn)CrossEntropyLoss   )ACT2FN)CacheDynamicCache)ModelOutput)GenerationMixin)_prepare_4d_attention_mask)BaseModelOutputBaseModelOutputWithPastBaseModelOutputWithPoolingCausalLMOutputWithPast)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings	torch_int   )	GitConfigGitVisionConfigzmicrosoft/git-baser   c                       e Zd ZU dZdZeej                 ed<   dZ	ej        ed<   dZ
eeej        df                  ed<   dZeeej        df                  ed<   dS )GitVisionModelOutputa  
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.

    Args:
        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nimage_embedslast_hidden_state.hidden_states
attentions)__name__
__module____qualname____doc__r#   r   torchFloatTensor__annotations__r$   r%   r   r&        `/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/git/modeling_git.pyr"   r"   8   s          * 15L(5,-444+/u(///=AM8E%"3S"89:AAA:>Ju0#567>>>>>r/   r"   c                        e Zd ZdZ fdZ	 	 	 	 ddeej                 deej                 deej                 de	d	ej
        f
d
Z xZS )GitEmbeddingsz;Construct the embeddings from word and position embeddings.c                    t                                                       t          j        |j        |j        |j                  | _        t          j        |j        |j                  | _	        t          j
        |j        |j                  | _
        t          j        |j                  | _        t          |dd          | _        |                     dt%          j        |j                                      d          d           d S )	N)padding_idxepsposition_embedding_typeabsoluteposition_idsr   F
persistent)super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutgetattrr7   register_bufferr+   arangeexpandselfconfig	__class__s     r0   r?   zGitEmbeddings.__init__Y   s    !|F,=v?Q_e_rsss#%<0NPVPb#c#c  f&8f>STTTz&"<=='.v7PR\']']$EL)GHHOOPWXXej 	 	
 	
 	
 	
 	
r/   Nr   	input_idsr9   inputs_embedspast_key_values_lengthreturnc                 ~   ||                                 }n|                                 d d         }|d         }|| j        d d |||z   f         }||                     |          }n|}| j        dk    r|                     |          }||z  }|                     |          }|                     |          }|S )Nr;   r   r8   )sizer9   rD   r7   rF   rG   rK   )	rQ   rT   r9   rU   rV   input_shape
seq_length
embeddingsrF   s	            r0   forwardzGitEmbeddings.forwardh   s      #..**KK',,..ss3K ^
,QQQ0FVlIl0l-lmL --i88JJ&J':55"&":":<"H"H--J^^J//
\\*--
r/   )NNNr   )r'   r(   r)   r*   r?   r   r+   
LongTensorr,   intTensorr]   __classcell__rS   s   @r0   r2   r2   V   s        EE
 
 
 
 
" 153759&' E,- u/0   12	
 !$ 
       r/   r2   c                        e Zd Zd fd	Zdej        dej        fdZ	 	 	 	 	 ddej        deej                 d	eej                 d
ee	         dee
         dee
         deej                 fdZ xZS )GitSelfAttentionNc                 ,   t                                                       |j        |j        z  dk    r0t	          |d          s t          d|j         d|j         d          || _        |(t                              d| j	        j
         d           |j        | _        t          |j        |j        z            | _        | j        | j        z  | _        t          |j        j        |j        j        z  dz  d	z             | _        |j        | xj        |j        z  c_        t'          j        |j        | j                  | _        t'          j        |j        | j                  | _        t'          j        |j        | j                  | _        t'          j        |j                  | _        |pt7          |d
d          | _        | j        dk    s| j        dk    r8|j        | _        t'          j        d|j        z  d	z
  | j                  | _        d S d S )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.   r   r7   r8   relative_keyrelative_key_query) r>   r?   rB   num_attention_headshasattr
ValueError	layer_idxloggerwarning_oncerS   r'   r_   attention_head_sizeall_head_sizevision_config
image_size
patch_sizeimage_patch_tokensnum_image_with_embeddingr   LinearquerykeyvaluerI   attention_probs_dropout_probrK   rL   r7   rE   r@   distance_embeddingrQ   rR   r7   rn   rS   s       r0   r?   zGitSelfAttention.__init__   s%    ::a??PVXhHiHi?8F$6 8 8 48 8 8   #,!8 , , ,   $*#= #&v'9F<V'V#W#W !58PP"%v';'FI]Ih'hmn&nqr&r"s"s*6##v'FF##Yv143EFF
9V/1CDDYv143EFF
z&"EFF'> (
'-zC
 C
$ '>99T=Y]q=q=q+1+ID(&(l1v7U3UXY3Y[_[s&t&tD### >r=qr/   xrW   c                     |                                 d d         | j        | j        fz   }|                    |          }|                    dddd          S )Nr;   r   rh   r   r
   )rY   rk   rq   viewpermute)rQ   r   new_x_shapes      r0   transpose_for_scoresz%GitSelfAttention.transpose_for_scores   sP    ffhhssmt'?AY&ZZFF;yyAq!$$$r/   Fr%   attention_mask	head_maskpast_key_valueoutput_attentionspixel_values_presentc           	      z   |                      |          }|r| j        nd}|                     |                     |                    }	|                     |                     |                    }
||                    |	d d d d |d d d f         |
d d d d |d d d f         | j                  \  }}t          j        |	d d d d d |d d f         |gd          }	t          j        |
d d d d d |d d f         |gd          }
|                     |          }t          j	        ||	
                    dd                    }| j        dk    s| j        dk    rt|j        d         |	j        d         }}|>t          j        |dz
  t          j        |j        	                              dd          }n:t          j        |t          j        |j        	                              dd          }t          j        |t          j        |j        	                              dd          }||z
  }|                     || j        z   dz
            }|                    |j        
          }| j        dk    rt          j        d||          }||z   }n?| j        dk    r4t          j        d||          }t          j        d|	|          }||z   |z   }|t/          j        | j                  z  }|||z   }t4          j                            |d          }|                     |          }|||z  }t          j	        ||
          }|                    dddd                                          }|                                 d d         | j!        fz   }|                    |          }|r||fn|f}||fz   }|S )Nr   rh   dimr;   ri   rj   r   dtypedevicer   zbhld,lrd->bhlrzbhrd,lrd->bhlrr
   )"ry   rv   r   rz   r{   updatern   r+   catmatmul	transposer7   shapetensorlongr   r   rN   r}   rE   tor   einsummathsqrtrq   r   
functionalsoftmaxrK   r   
contiguousrY   rr   )rQ   r%   r   r   r   r   r   mixed_query_layercutoff	key_layervalue_layerkey_layer_pastvalue_layer_pastquery_layerattention_scoresquery_length
key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probscontext_layernew_context_layer_shapeoutputss                               r0   r]   zGitSelfAttention.forward   s    !JJ}55,@G((a--dhh}.E.EFF	//

=0I0IJJ%/=/D/D!!!QQQ*+[AAAvww9I-JDN0 0,N, 	9QQQ7F7AAA-=#>"OUVWWWI)[AAAww1A%BDT$U[\]]]K//0ABB !<Y5H5HR5P5PQQ'>99T=Y]q=q=q'2'8';Y_Q=O*L)!&j1nEJWdWk!l!l!l!q!q" " "'l%*UbUi!j!j!j!o!oprtu!v!v"\*EJ}OcdddiijkmoppN%6H#'#:#:8dFb;bef;f#g#g #7#:#:AR#:#S#S +~==+0<8H+Wk+l+l(#36N#N  -1EEE16>NP[]q1r1r./4|<LiYm/n/n,#36T#TWs#s +di8P.Q.QQ%/.@ -//0@b/II ,,77  -	9O_kBB%--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S%**+BCC6G]=/22mM]^--r/   NNNNNFF)r'   r(   r)   r?   r+   r`   r   r   r,   r   boolr   r]   ra   rb   s   @r0   rd   rd      s        u  u  u  u  u  uD%el %u| % % % % 7;15*.,1/4J J|J !!23J E-.	J
 !J $D>J 'tnJ 
u|	J J J J J J J Jr/   rd   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )GitSelfOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j        |j	                  | _
        d S Nr5   )r>   r?   r   rx   rB   denserG   rH   rI   rJ   rK   rP   s     r0   r?   zGitSelfOutput.__init__   sf    Yv163EFF
f&8f>STTTz&"<==r/   r%   input_tensorrW   c                     |                      |          }|                     |          }|                     ||z             }|S Nr   rK   rG   rQ   r%   r   s      r0   r]   zGitSelfOutput.forward  @    

=11]33}|'CDDr/   r'   r(   r)   r?   r+   r`   r]   ra   rb   s   @r0   r   r      i        > > > > >U\  RWR^        r/   r   eagerc                        e Zd Zd fd	Zd Z	 	 	 	 	 ddej        deej                 deej                 dee	         d	ee
         d
ee
         deej                 fdZ xZS )GitAttentionNc                     t                                                       t          |j                 |||          | _        t          |          | _        t                      | _        d S )N)r7   rn   )	r>   r?   GIT_SELF_ATTENTION_CLASSES_attn_implementationrQ   r   outputsetpruned_headsr~   s       r0   r?   zGitAttention.__init__  sb    .v/JK,Cy
 
 
	 $F++EEr/   c                    t          |          dk    rd S t          || j        j        | j        j        | j                  \  }}t          | j        j        |          | j        _        t          | j        j        |          | j        _        t          | j        j	        |          | j        _	        t          | j
        j        |d          | j
        _        | j        j        t          |          z
  | j        _        | j        j        | j        j        z  | j        _        | j                            |          | _        d S )Nr   r   r   )lenr   rQ   rk   rq   r   r   ry   rz   r{   r   r   rr   union)rQ   headsindexs      r0   prune_headszGitAttention.prune_heads  s    u::??F7490$)2OQUQb
 
u
 -TY_eDD	*49=%@@	,TY_eDD	.t{/@%QOOO )-	(EE

(R	%"&)"?$)B_"_	 -33E::r/   Fr%   r   r   r   r   r   rW   c                     |                      ||||||          }|                     |d         |          }|f|dd          z   }	|	S )Nr   r   )rQ   r   )
rQ   r%   r   r   r   r   r   self_outputsattention_outputr   s
             r0   r]   zGitAttention.forward+  s^     yy 
 
  ;;|AFF#%QRR(88r/   r   r   )r'   r(   r)   r?   r   r+   r`   r   r,   r   r   r   r]   ra   rb   s   @r0   r   r     s        " " " " " "; ; ;* 7;15*.,1/4 | !!23 E-.	
 ! $D> 'tn 
u|	       r/   r   c                   B     e Zd Z fdZdej        dej        fdZ xZS )GitIntermediatec                    t                                                       t          j        |j        |j                  | _        t          |j        t                    rt          |j                 | _        d S |j        | _        d S r   )r>   r?   r   rx   rB   intermediate_sizer   
isinstance
hidden_actstrr   intermediate_act_fnrP   s     r0   r?   zGitIntermediate.__init__C  sn    Yv163KLL
f'-- 	9'-f.?'@D$$$'-'8D$$$r/   r%   rW   c                 Z    |                      |          }|                     |          }|S r   )r   r   rQ   r%   s     r0   r]   zGitIntermediate.forwardK  s,    

=1100??r/   r   rb   s   @r0   r   r   B  s^        9 9 9 9 9U\ el        r/   r   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )	GitOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j	        |j
                  | _        d S r   )r>   r?   r   rx   r   rB   r   rG   rH   rI   rJ   rK   rP   s     r0   r?   zGitOutput.__init__S  sf    Yv79KLL
f&8f>STTTz&"<==r/   r%   r   rW   c                     |                      |          }|                     |          }|                     ||z             }|S r   r   r   s      r0   r]   zGitOutput.forwardY  r   r/   r   rb   s   @r0   r   r   R  r   r/   r   c                        e Zd Zd fd	Z	 	 	 	 	 ddej        deej                 deej                 dee         dee	         d	ee	         d
e
ej                 fdZd Z xZS )GitLayerNc                     t                                                       |j        | _        d| _        t	          ||          | _        t          |          | _        t          |          | _	        d S )Nr   )rn   )
r>   r?   chunk_size_feed_forwardseq_len_dimr   	attentionr   intermediater   r   )rQ   rR   rn   rS   s      r0   r?   zGitLayer.__init__a  sc    '-'E$%f	BBB+F33''r/   Fr%   r   r   r   r   r   rW   c                     |                      ||||||          }|d         }|dd         }	|d         }
t          | j        | j        | j        |          }|f|	z   }	|	|
fz   }	|	S )N)r   r   r   r   r   r;   )r   r   feed_forward_chunkr   r   )rQ   r%   r   r   r   r   r   self_attention_outputsr   r   present_key_valuelayer_outputs               r0   r]   zGitLayer.forwardi  s     "&/)!5 "0 "
 "
 2!4 )2.2260#T%A4CSUe
 
  /G+ .00r/   c                 \    |                      |          }|                     ||          }|S r   )r   r   )rQ   r   intermediate_outputr   s       r0   r   zGitLayer.feed_forward_chunk  s2    "//0@AA{{#68HIIr/   r   r   )r'   r(   r)   r?   r+   r`   r   r,   r   r   r   r]   r   ra   rb   s   @r0   r   r   `  s        ( ( ( ( ( ( 7;15*.,1/4   |  !!23  E-.	 
 !  $D>  'tn  
u|	       D      r/   r   c                   6    e Zd Z fdZ	 	 	 	 	 	 	 	 ddej        deej                 deej                 deee	e
e
ej                          f                  d	ee         d
ee         dee         dee         dee         dee
ej                 ef         fdZ xZS )
GitEncoderc                     t                                                       | _        t          j        fdt          j                  D                       | _        d| _        d S )Nc                 0    g | ]}t          |          S r.   )r   ).0irR   s     r0   
<listcomp>z'GitEncoder.__init__.<locals>.<listcomp>  s#    #a#a#aAHVQ$7$7#a#a#ar/   F)	r>   r?   rR   r   
ModuleListrangenum_hidden_layerslayergradient_checkpointingrP   s    `r0   r?   zGitEncoder.__init__  s`    ]#a#a#a#avG_A`A`#a#a#abb
&+###r/   NFTr%   r   r   past_key_values	use_cacher   output_hidden_statesr   return_dictrW   c
           	         | j         r%| j        r|rt                              d           d}d}
|rVt	          |t
                    sAd}
|t                      }n.t          j        |          }t                              d           |rdnd }|rdnd }d }t          | j	                  D ]w\  }}|r||fz   }|||         nd }| j         r'| j        r | 
                    |j        |||||          }n |||||||          }|d         }|r|d         }|r||d         fz   }x|r||fz   }|r|nd }|
r|                                }|	st          d	 ||||fD                       S t          ||||
          S )NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FTzWe detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class (https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)r.   r   r;   r   c              3      K   | ]}||V  	d S r   r.   r   vs     r0   	<genexpr>z%GitEncoder.forward.<locals>.<genexpr>  s4       	 	 =  !===	 	r/   r$   r   r%   r&   )r   trainingro   rp   r   r   r   from_legacy_cache	enumerater   _gradient_checkpointing_func__call__to_legacy_cachetupler   )rQ   r%   r   r   r   r   r   r  r   r  return_legacy_cacheall_hidden_statesall_self_attentionsnext_decoder_cacher   layer_modulelayer_head_masklayer_outputs
next_caches                      r0   r]   zGitEncoder.forward  sb    & 	"4= 	" "##p   "	 $ 
	Z?? 
	"&&"..."."@"Q"Q##^   #7@BBD$5?bb4!(44 	P 	POA|# I$58H$H!.7.CillO* t}  $ A A )!"##%! ! !-!"##%(! ! *!,M 7%22%6"  P&9]1=M<O&O# 	E 1]4D D+4>''$
 	6#3355J 
	 	 	 "%'		 	 	 	 	 	 '+&+*	
 
 
 	
r/   )NNNNFFFT)r'   r(   r)   r?   r+   r`   r   r,   r   r   r   r   r   r]   ra   rb   s   @r0   r   r     s4       , , , , , 7;15SW$(,1/4/4&*Z
 Z
|Z
 !!23Z
 E-.	Z

 "%uU5;L5M/N(N"OPZ
 D>Z
 $D>Z
 'tnZ
 'tnZ
 d^Z
 
uU\"$;;	<Z
 Z
 Z
 Z
 Z
 Z
 Z
 Z
r/   r   c                   ,    e Zd ZdZeZdZdZdZdZ	d Z
dS )GitPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    gitTc                    t          |t                    rt          j                            |j        d| j        j                   t          j                            |j        j	        | j        j                   t          j                            |j
        j	        | j        j                   t          |t          j                  rT|j	        j                            d| j        j                   |j         |j        j                                         dS dS t          |t          j                  r_|j	        j                            d| j        j                   |j        +|j	        j        |j                                                  dS dS t          |t          j                  r?|j        j                                         |j	        j                            d           dS dS )zInitialize the weightsg        )meanstd)r  Ng      ?)r   GitVisionEmbeddingsr   initnormal_class_embeddingrR   initializer_rangepatch_embeddingweightposition_embeddingrx   databiaszero_r@   r4   rG   fill_)rQ   modules     r0   _init_weightsz GitPreTrainedModel._init_weights  s   f122 	aGOOF2$+B_O```GOOF29t{?\O]]]GOOF5<$+B_O```fbi(( 	* M&&CT[5R&SSS{& &&((((( '&-- 	*M&&CT[5R&SSS!-"6#56<<>>>>> .--- 	*K""$$$M$$S)))))	* 	*r/   N)r'   r(   r)   r*   r   config_classbase_model_prefixsupports_gradient_checkpointing_supports_cache_class_supports_quantized_cacher+  r.   r/   r0   r  r    sL         
 L&*#  $* * * * *r/   r  a=  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`GitConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
an  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)

        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`CLIPImageProcessor.__call__`] for details.

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance, see our
            [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
            of shape `(batch_size, sequence_length)`.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   v     e Zd Zdef fdZdej        dededej        fdZdd	ej	        dej        fd
Z
 xZS )r  rR   c                 z   t                                                       || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        | j                            | _        t          j        |j        | j        | j        | j        d          | _        | j        | j        z  dz  | _        | j        dz   | _        t          j        | j        | j                  | _        |                     dt          j        | j                                      d          d           d S )NF)in_channelsout_channelskernel_sizestrider'  rh   r   r9   r:   r<   )r>   r?   rR   rB   	embed_dimrt   ru   r   	Parameterr+   randnr!  Conv2dnum_channelsr#  num_patchesnum_positionsr@   r%  rM   rN   rO   rP   s     r0   r?   zGitVisionEmbeddings.__init__l  s   + + +!|EK,G,GHH!y+? 
  
  
 !Ot>1D!-1"$,t/A4>"R"R^U\$:L-M-M-T-TU\-]-]jopppppr/   r\   heightwidthrW   c                    |j         d         dz
  }| j        j                            d          }|j         d         dz
  }t          j                                        s&||k    r ||k    r|                     | j                  S |ddddf         }|ddddf         }|j         d         }	|| j        z  }
|| j        z  }t          |dz            }|
                    d|||	          }|                    dddd          }t          j                            ||
|fdd	
          }|                    dddd                              dd|	          }t	          j        ||fd          S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   Nr;   g      ?r
   rh   bicubicF)rY   modealign_cornersr   )r   r%  r$  	unsqueezer+   jit
is_tracingr9   ru   r   reshaper   r   r   interpolater   r   )rQ   r\   r>  r?  r<  r%  r=  class_pos_embedpatch_pos_embedr   
new_height	new_widthsqrt_num_positionss                r0   interpolate_pos_encodingz,GitVisionEmbeddings.interpolate_pos_encoding  s    !&q)A-!4;EEaHH*03a7 y##%% 	>+*F*F6UZ??**4+<===,QQQU3,QQQU3r"t.
T_,	&}c'9::)11!5GI[]`aa)11!Q1==-33i(	 4 
 
 *11!Q1==BB1b#NNy/?;CCCCr/   Fpixel_valuesc                 <   |j         \  }}}}|s<|| j        k    s|| j        k    r&t          d| d| d| j         d| j         d	          | j        j        j        }|                     |                    |                    }|                    d                              dd          }| j	        
                    |dd          }	t          j        |	|gd	          }
|r|
|                     |
||          z   }
n|
|                     | j                  z   }
|
S )
NzInput image size (*z) doesn't match model ().r   rh   r   r;   r   )r   rt   rm   r#  r$  r   r   flattenr   r!  rO   r+   r   rN  r%  r9   )rQ   rO  rN  
batch_size_r>  r?  target_dtypepatch_embedsclass_embedsr\   s              r0   r]   zGitVisionEmbeddings.forward  sF   '3'9$
Avu' 	Vt-F-F%SWSbJbJbuVuueuuPTP_uubfbquuu   +28++LOO,O,O,OPP#++A..88A>>+22:q"EEYl;CCC
# 	Q#d&C&CJPVX]&^&^^JJ#d&=&=d>O&P&PPJr/   F)r'   r(   r)   r    r?   r+   r`   r_   rN  r,   r]   ra   rb   s   @r0   r  r  k  s        q q q q q q q,'D5< 'D 'DUX 'D]b]i 'D 'D 'D 'DR E$5 Z_Zf        r/   r  c                   B     e Zd Z fdZdej        dej        fdZ xZS )GitVisionMLPc                    t                                                       || _        t          |j                 | _        t          j        |j        |j	                  | _
        t          j        |j	        |j                  | _        d S r   )r>   r?   rR   r   r   activation_fnr   rx   rB   r   fc1fc2rP   s     r0   r?   zGitVisionMLP.__init__  sf    #F$569V/1IJJ9V5v7IJJr/   r%   rW   c                     |                      |          }|                     |          }|                     |          }|S r   )r^  r]  r_  r   s     r0   r]   zGitVisionMLP.forward  s=    //**=99//r/   r   rb   s   @r0   r[  r[    sc        K K K K KU\ el        r/   r[  c                        e Zd ZdZ fdZdej        dedefdZ	 	 	 dd	ej        d
e	ej                 de	ej                 de	e
         deej        e	ej                 f         f
dZ xZS )GitVisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 t   t                                                       || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j         d| j         d          | j        dz  | _	        |j
        | _        t          j        | j        | j                  | _        t          j        | j        | j                  | _        t          j        | j        | j                  | _        t          j        | j        | j                  | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: rR  g      )r>   r?   rR   rB   r7  rk   	num_headshead_dimrm   scaleattention_dropoutrK   r   rx   k_projv_projq_projout_projrP   s     r0   r?   zGitVisionAttention.__init__  s   +3$.8=4>)T^;;'dn ' 'N' ' '   ]D(
/i??i??i??	$.$.AAr/   r   seq_lenbszc                     |                     ||| j        | j                                      dd                                          S )Nr   rh   )r   rd  re  r   r   )rQ   r   rl  rm  s       r0   _shapezGitVisionAttention._shape  s<    {{3GGQQRSUVWWbbdddr/   NFr%   r   causal_attention_maskr   rW   c                    |                                 \  }}}|                     |          | j        z  }|                     |                     |          d|          }	|                     |                     |          d|          }
|| j        z  d| j        f} |                     |||          j        | } |	j        | }	 |
j        | }
|	                     d          }t          j
        ||	                    dd                    }|                                 || j        z  ||fk    r2t          d|| j        z  ||f d|                                            ||                                 |d||fk    r+t          d|d||f d|                                            |                    || j        ||          |z   }|                    || j        z  ||          }||                                 |d||fk    r+t          d|d||f d|                                            |                    || j        ||          |z   }|                    || j        z  ||          }t          j                            |d          }|r=|                    || j        ||          }|                    || j        z  ||          }nd}t          j                            || j        | j        	          }t          j
        ||
          }|                                 || j        z  || j        fk    r5t          d
|| j        || j        f d|                                            |                    || j        || j                  }|                    dd          }|                    |||          }|                     |          }||fS )z#Input shape: Batch x Time x Channelr;   r   rh   z$Attention weights should be of size z	, but is Nz!Attention mask should be of size r   )pr	  z `attn_output` should be of size )rY   rj  rf  ro  rh  ri  rd  re  r   r+   bmmr   rm   r   r   r   rK   r	  rG  rk  )rQ   r%   r   rp  r   rm  tgt_lenr7  query_states
key_statesvalue_states
proj_shapesrc_lenattn_weightsattn_weights_reshaped
attn_probsattn_outputs                    r0   r]   zGitVisionAttention.forward  s0    #0"4"4"6"6Wi {{=11DJ>[[]!;!;REE
{{4;;}#=#=r3GGDN*B>
Ct{{<#>>CZP$Z_j1
(|(*5//!$$yz/C/CAq/I/IJJ3#7'"JJJ*dn8LgW^7_ * * %%''* *   !,$))++Q/III 7a'8R 7 7-22447 7   (,,S$.'7SSVkkL',,S4>-A7GTTL%""$$a'(BBB ta'8Rtt]k]p]p]r]rtt   (,,S$.'7SSVddL',,S4>-A7GTTL},,\r,BB 	)
 %1$5$5c4>7T[$\$\!055cDN6JGU\]]LL$(!]**<4<RVR_*``
i
L99#"6!OOO)CRVR_3` ) )$$&&) )  
 "&&sDNGT]SS!++Aq11!))#w	BBmmK00111r/   )NNF)r'   r(   r)   r*   r?   r+   r`   r_   ro  r   r   r   r]   ra   rb   s   @r0   rb  rb    s        GGB B B B B&eU\ eC ec e e e e 268<,1L2 L2|L2 !.L2  (5	L2
 $D>L2 
u|Xel33	4L2 L2 L2 L2 L2 L2 L2 L2r/   rb  c                        e Zd Zdef fdZ	 d
dej        dej        dej        dee         de	ej
                 f
d	Z xZS )GitVisionEncoderLayerrR   c                 D   t                                                       |j        | _        t	          |          | _        t          j        | j        |j                  | _	        t          |          | _        t          j        | j        |j                  | _        d S r   )r>   r?   rB   r7  rb  	self_attnr   rG   rH   layer_norm1r[  mlplayer_norm2rP   s     r0   r?   zGitVisionEncoderLayer.__init__9  s    ++F33<F<QRRR''<F<QRRRr/   Fr%   r   rp  r   rW   c                     |}|                      |          }|                     ||||          \  }}||z   }|}|                     |          }|                     |          }||z   }|f}|r||fz  }|S )aI  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r%   r   rp  r   )r  r  r  r  )rQ   r%   r   rp  r   residualrz  r   s           r0   r]   zGitVisionEncoderLayer.forwardA  s    " !((77&*nn')"7/	 '5 '
 '
#| !=0 ((77// =0 " 	'&Gr/   rY  )r'   r(   r)   r    r?   r+   r`   r   r   r   r,   r]   ra   rb   s   @r0   r  r  8  s        S S S S S S S -2& &|& &  %|	&
 $D>& 
u 	!& & & & & & & &r/   r  c                        e Zd ZdZdef fdZ	 	 	 	 	 ddeej                 deej                 dee	         dee	         d	ee	         d
e
eef         fdZ xZS )GitVisionEncoderz
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`GitVisionEncoderLayer`].

    Args:
        config: GitVisionConfig
    rR   c                     t                                                       | _        t          j        fdt          j                  D                       | _        d| _        d S )Nc                 .    g | ]}t                    S r.   )r  r   rU  rR   s     r0   r   z-GitVisionEncoder.__init__.<locals>.<listcomp>w  s"    $l$l$lq%:6%B%B$l$l$lr/   F)	r>   r?   rR   r   r   r   r   layersr   rP   s    `r0   r?   zGitVisionEncoder.__init__t  sa    m$l$l$l$lERXRjLkLk$l$l$lmm&+###r/   Nr   rp  r   r  r  rW   c                    ||n| j         j        }||n| j         j        }||n| j         j        }|rdnd}|rdnd}|}	t	          | j                  D ]_\  }
}|r||	fz   }| j        r&| j        r|                     |j	        |	|||          }n ||	|||          }|d         }	|r||d         fz   }`|r||	fz   }|st          d |	||fD                       S t          |	||          S )a  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr.   )r   r   r   c              3      K   | ]}||V  	d S r   r.   r  s     r0   r  z+GitVisionEncoder.forward.<locals>.<genexpr>  s(      eeqWXWdWdWdWdWdeer/   r$   r%   r&   )rR   r   r  use_return_dictr  r  r   r	  r  r  r  r   )rQ   rU   r   rp  r   r  r  encoder_statesall_attentionsr%   idxencoder_layerr  s                r0   r]   zGitVisionEncoder.forwardz  s   L 2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B]3=0:d%"+DK"8"8 	F 	FC# C!/=2B!B* t}  $ A A!*!")%! ! !.!")&7	! ! ! *!,M  F!/=3C2E!E 	?+}.>>N 	fee]NN$Seeeeee+>Vd
 
 
 	
r/   )NNNNN)r'   r(   r)   r*   r    r?   r   r+   r`   r   r   r   r   r]   ra   rb   s   @r0   r  r  k  s         , , , , , , , 268<,0/3&*O
 O
 !.O
  (5	O

 $D>O
 'tnO
 d^O
 
uo%	&O
 O
 O
 O
 O
 O
 O
 O
r/   r  a  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                        e Zd Zdef fdZ ee           eee          	 	 	 	 	 dde	e
j                 de	e         de	e         d	e	e         d
e	e         deeef         fd                        Z xZS )GitVisionTransformerrR   c                 4   t                                                       || _        |j        }t	          |          | _        t          j        ||j                  | _	        t          |          | _        t          j        ||j                  | _        d S r   )r>   r?   rR   rB   r  r\   r   rG   rH   pre_layrnormr  encoderpost_layernorm)rQ   rR   r7  rS   s      r0   r?   zGitVisionTransformer.__init__  s    &	-f55L8MNNN'// l9&:OPPPr/   output_typer,  NFrO  r   r  rN  r  rW   c                    ||n| j         j        }||n| j         j        }||n| j         j        }|t	          d          |                     ||          }|                     |          }|                     ||||          }|d         }|                     |          }|s|f|dd         z   S t          ||j
        |j                  S )z
        Returns:

        Nz You have to specify pixel_valuesrN  )rU   r   r  r  r   r   r  )rR   r   r  r  rm   r\   r  r  r  r   r%   r&   )	rQ   rO  r   r  rN  r  r%   encoder_outputsr$   s	            r0   r]   zGitVisionTransformer.forward  s    2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B]?@@@Oghh))-88,,'/!5#	 ' 
 
 ,A. //0ABB 	>%'/!""*===/)7&1
 
 
 	
r/   NNNFN)r'   r(   r)   r    r?   r   GIT_VISION_INPUTS_DOCSTRINGr   r   r   r+   r,   r   r   r   r]   ra   rb   s   @r0   r  r    s       Q Q Q Q Q Q Q +*+FGG?YYY 59,0/338&**
 *
u01*
 $D>*
 'tn	*

 #+4.*
 d^*
 
uo%	&*
 *
 *
 ZY HG*
 *
 *
 *
 *
r/   r  zOThe vision model from CLIP, used in GIT, without any head or projection on top.c                        e Zd ZeZdZdef fdZdej        fdZ	 e
e           eee          	 	 	 	 	 ddeej                 d	ee         d
ee         dedee         deeef         fd                        Z xZS )GitVisionModelrO  rR   c                     t                                          |           t          |          | _        |                                  d S r   )r>   r?   r  vision_model	post_initrP   s     r0   r?   zGitVisionModel.__init__"  sA       088r/   rW   c                 $    | j         j        j        S r   )r  r\   r#  rQ   s    r0   get_input_embeddingsz#GitVisionModel.get_input_embeddings(  s     +;;r/   r  NFr   r  rN  r  c                 V    ||n| j         j        }|                     |||||          S )a  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, GitVisionModel

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base")
        >>> model = GitVisionModel.from_pretrained("microsoft/git-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        ```N)rO  r   r  rN  r  )rR   r  r  )rQ   rO  r   r  rN  r  s         r0   r]   zGitVisionModel.forward+  sC    > &1%<kk$+B]  %/!5%=# ! 
 
 	
r/   r  )r'   r(   r)   r    r,  main_input_namer?   r   Moduler  r   r  r   r   r   r+   r,   r   r   r   r]   ra   rb   s   @r0   r  r    s%       
 #L$O      <bi < < < < +*+FGG?YYY 59,0/3).&*%
 %
u01%
 $D>%
 'tn	%

 #'%
 d^%
 
uo%	&%
 %
 %
 ZY HG%
 %
 %
 %
 %
r/   r  c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )GitProjectionrR   c                    t                                                       || _        t          j        t          j        |j        j        |j                  t          j        |j        |j        j	                            | _
        d S r   )r>   r?   rR   r   
Sequentialrx   rs   rB   rG   rH   visual_projectionrP   s     r0   r?   zGitProjection.__init__V  sm    !#If*68JKKL+1E1TUUU"
 "
r/   r\   rW   c                 ,    |                      |          S r   )r  )rQ   r\   s     r0   r]   zGitProjection.forward^  s    %%j111r/   )	r'   r(   r)   r   r?   r+   r`   r]   ra   rb   s   @r0   r  r  U  sj        
y 
 
 
 
 
 
2%, 25< 2 2 2 2 2 2 2 2r/   r  zThe bare GIT Model transformer consisting of a CLIP image encoder and text decoder outputting raw hidden-states without any specific head on top.c                   >    e Zd Z fdZd Zd Zd Zdedej	        dej
        dej        fd	ZddZ ee                    d                     eee          	 	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 deej                 deej                 deeeeej                 f                  dee         dee         dee         dedee         deeej                 ef         fd                        Z xZS )GitModelc                    t                                                     | _        t                    | _        t          j                  | _        t                    | _	        t                    | _        j        7t          j        fdt          j                  D                       | _        |                                  d S )Nc              3   |   K   | ]6}t          j        t          j        d d j        j                            V  7dS )r   N)r   r8  r+   zerosrs   rB   r  s     r0   r  z$GitModel.__init__.<locals>.<genexpr>s  sU       ; ; U[Av/C/OPPQQ; ; ; ; ; ;r/   )r>   r?   rR   r2   r\   r  rs   image_encoderr   r  r  r  rw   r   ParameterListr   img_temperal_embeddingr  rP   s    `r0   r?   zGitModel.__init__h  s       '//+F,@AA!&))!.v!6!6*6*,*: ; ; ; ;v>??; ; ; + +D' 	r/   c                     | j         j        S r   r\   rD   r  s    r0   r  zGitModel.get_input_embeddings{  s    ..r/   c                     || j         _        d S r   r  )rQ   r{   s     r0   set_input_embeddingszGitModel.set_input_embeddings~  s    */'''r/   c                     |                                 D ]/\  }}| j        j        |         j                            |           0dS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr  r   r   r   )rQ   heads_to_pruner   r   s       r0   _prune_headszGitModel._prune_heads  sU    
 +0022 	C 	CLE5Lu%/;;EBBBB	C 	Cr/   rY   r   r   rW   c                     t          j        t          j        ||||          d          }|                    |dk    t	          d                    }|S )Nr   r   r   )diagonal-inf)r+   triuonesmasked_fillfloat)rQ   rY   r   r   masks        r0   _generate_future_maskzGitModel._generate_future_mask  sN    z%*T4eLLLWXYYY	5==99r/   Nc                 p   |j         d         }|j         d         }|j        }|j        }	t          j        ||f||	          }
t          j        |||z   ft          d          |j        |	          }t          j        ||f|	|j                  }|dk    r7t          j        |j         d         |j         d         |z   f|	|j                  }t          j        |
|fd          }t          j        ||                    |	          fd          }t          j        ||fd          d d d f         }|/t          j        |j         d         |j         d         fd|          }|j        t          j	        k    rt          d	          t          j        ||j        
          }t          d          ||<   |                    |j         d         ||z   ||z   |z   f          }|                                }|d d d d d |f         }|d d d d d f         }||z   |d d d d d |f<   |d d d d d d d f         }|S )Nr   r  r  r   r   r   F)
fill_valuer   z1Memory key padding mask must be a boolean tensor.r   )r   r   r   r+   r  fullr  r   r   r   rm   
zeros_likerO   clone)rQ   tgtmemorytgt_maskrV   memory_key_padding_masknum_tgt
num_memoryr   r   top_left	top_rightbottom_leftleftrightfull_attention_maskzero_negative_infinityorigin_leftr   s                      r0   create_attention_maskzGitModel.create_attention_mask  s   )A,\!_
	;
J7eTTTJ#99:&MM:	
 
 
	 kj!?
 
 
 "A%%{"HN1$58N$NO  H y(K0a888	9hkk%&8&89qAAA#iu1===dAAAgF"*&+j&,q/6<PQ?1S`ent&u&u&u#"(EJ66PQQQ!&!12IQTQZ![![![:?--67188$*1-zG/CZRhEhkrErs
 
 27799)!!!QQQ*;<'4
31<v1EAAAqqq+:+-. 2!!!T111aaa-@""r/   batch_size, sequence_lengthr  FrT   r   r9   rO  r   rU   r   r   r   r  rN  r  c                 z   |	|	n| j         j        }	|
|
n| j         j        }
||n| j         j        }||n| j         j        }||t          d          |+|                     ||           |                                }n.||                                dd         }nt          d          |d         }d}|Bt          |t                    s|d         d         j
        d         n|                                }|                     || j         j                  }d}||j        dk    r|                     ||	          j        }n|j        d
k    rg }t#          |j
        d                   D ]S}|                     |dd|ddddf         |	          j        }|| j        |         z  }|                    |           Tt)          j        |d          }nt          d          |                     |          }|                     ||||          }|:t)          j        |j
        d         d|j
        d         f|j        |j                  }|                    |                    d          |                    d          z  dd          }t)          j        ||fd          }|                     ||j        |j                  }|                     ||||          }|{t=          ||j        |d                                       |j                  }|dk    r|dddd| dddf         }n*|dddd|d          d|d          dfxx         |z  cc<   |                      ||||||	|
||du	  	        }|d         }|s|f|dd         z   S tC          ||j"        |j#        |j$                  S )a  
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoProcessor, AutoModel
        >>> import requests
        >>> from PIL import Image

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base")
        >>> model = AutoModel.from_pretrained("microsoft/git-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> text = "this is an image of two cats"

        >>> inputs = processor(images=image, text=text, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer;   z5You have to specify either input_ids or inputs_embedsr   r   rh      r     r   z#pixel_values must be of rank 4 or 5)rT   r9   rU   rV   r   )r  r  r  rV   )rt  )r   r   r   r   r   r  r  r   r  )%rR   r   r  r   r  rm   %warn_if_padding_and_no_attention_maskrY   r   r   r   get_seq_lengthget_head_maskr   ndimr  r$   r   r  appendr+   r   r  r\   r  r   r   repeatr  r  r   r   r  r   r   r%   r&   )rQ   rT   r   r9   rO  r   rU   r   r   r   r  rN  r  rZ   r[   rV   projected_visual_featuresvisual_features	frame_idxvisual_features_frameembedding_outputr%   r  combined_attention_maskexpanded_attn_maskr  sequence_outputs                              r0   r]   zGitModel.forward  s   X 2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B] ]%>cddd"66y.QQQ#..**KK&',,..ss3KKTUUU ^
 "#& "/5996"1%+A..$3355 # &&y$+2OPP	$(!# A%%"&"4"4 ;S #5 # ##   "a''"$!&|'9!'<!=!= B BI,0,>,>$QQQ	111aaa%78Sk -? - -' * *T-H-SS)#**+@AAAA #()O"C"C"C !!FGGG(,(>(>(O(O%??%'#9	 + 
 
 %,(-!'*A/?/Ea/HI&,'.) ) )% %>$D$D!!!$$(A(F(Fq(I(II1a%
 %
!
 	#<>N"OUVWWW --j:J:PRbRijj #'"<"< ,#9	 #= #
 #
 % "< 0 6B" " "b!())  &))%7111?U>U>V>VXYXYXY8Y%Z""'111{1~o.?.?+a.ARAR(RSSSWiiSSS,,2+/!5#!-T!9 ' 

 

 *!, 	<#%(;;;&-+;)7&1	
 
 
 	
r/   r   )NNNNNNNNNNFN)r'   r(   r)   r?   r  r  r  r_   r+   r   r   r`   r  r  r   GIT_INPUTS_DOCSTRINGformatr   r   _CONFIG_FOR_DOCr   r   r   r   r,   r   r   r]   ra   rb   s   @r0   r  r  b  s4           &/ / /0 0 0C C C# ek 5< \a\h    0# 0# 0# 0#d +*+?+F+FGd+e+eff+ETcddd -115/3/3,004KO$(,0/3).&*i
 i
EL)i
 !.i
 u|,	i

 u|,i
 EL)i
  -i
 "%tE4E/F(F"GHi
 D>i
 $D>i
 'tni
 #'i
 d^i
 
uU\"$>>	?i
 i
 i
 ed gfi
 i
 i
 i
 i
r/   r  zVGIT Model with a `language modeling` head on top for autoregressive language modeling.c            !       ,    e Zd ZdgZ fdZd Zd Z ee	                    d                     e
ee          	 	 	 	 	 	 	 	 	 	 	 	 	 dd	eej                 d
eej                 deej                 deej                 deej                 deej                 deej                 deeeeej                 f                  dee         dee         dee         dedee         deeej                 ef         fd                        Z	 ddZd Z xZS )GitForCausalLMzoutput.weightc                     t                                          |           t          |          | _        t	          j        |j        |j                  | _        | 	                                 d S r   )
r>   r?   r  r  r   rx   rB   rA   r   r  rP   s     r0   r?   zGitForCausalLM.__init__u  s[       F##i 2F4EFF 	r/   c                     | j         S r   r   r  s    r0   get_output_embeddingsz$GitForCausalLM.get_output_embeddings~  s
    {r/   c                     || _         d S r   r  )rQ   new_embeddingss     r0   set_output_embeddingsz$GitForCausalLM.set_output_embeddings  s    $r/   r  r  NFrT   r   r9   rO  r   rU   labelsr   r   r   r  rN  r  rW   c                    ||n| j         j        }|d}	|                     ||||||||	|
|||          }|d         }|                     |          }d}|| j        j        j        d         j        j        j        }|dd|dddf         	                                }|ddddf         	                                }t                      } ||                    d| j         j                  |                    d                    }|s|f|dd         z   }||f|z   n|S t          |||j        |j        |j                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).

        Returns:

        Examples:

        Image captioning example:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForCausalLM
        >>> import requests
        >>> from PIL import Image

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-coco")
        >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-coco")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values

        >>> generated_ids = model.generate(pixel_values=pixel_values, max_length=50)
        >>> generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> print(generated_caption)
        two cats sleeping on a pink blanket next to remotes.
        ```

        Visual question answering (VQA) example:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForCausalLM
        >>> from huggingface_hub import hf_hub_download
        >>> from PIL import Image

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-textvqa")
        >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-textvqa")

        >>> file_path = hf_hub_download(repo_id="nielsr/textvqa-sample", filename="bus.png", repo_type="dataset")
        >>> image = Image.open(file_path).convert("RGB")

        >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values

        >>> question = "what does the front of the bus say at the top?"

        >>> input_ids = processor(text=question, add_special_tokens=False).input_ids
        >>> input_ids = [processor.tokenizer.cls_token_id] + input_ids
        >>> input_ids = torch.tensor(input_ids).unsqueeze(0)

        >>> generated_ids = model.generate(pixel_values=pixel_values, input_ids=input_ids, max_length=50)
        >>> print(processor.batch_decode(generated_ids, skip_special_tokens=True))
        ['what does the front of the bus say at the top? special']
        ```

        Video captioning example:

        ```python
        >>> import av
        >>> import numpy as np
        >>> from PIL import Image
        >>> from huggingface_hub import hf_hub_download
        >>> from transformers import AutoProcessor, AutoModelForCausalLM

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-vatex")
        >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-vatex")

        >>> # set seed for reproducability
        >>> np.random.seed(45)


        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`List[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])


        >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len):
        ...     '''
        ...     Sample a given number of frame indices from the video.
        ...     Args:
        ...         clip_len (`int`): Total number of frames to sample.
        ...         frame_sample_rate (`int`): Sample every n-th frame.
        ...         seg_len (`int`): Maximum allowed index of sample's last frame.
        ...     Returns:
        ...         indices (`List[int]`): List of sampled frame indices
        ...     '''
        ...     converted_len = int(clip_len * frame_sample_rate)
        ...     end_idx = np.random.randint(converted_len, seg_len)
        ...     start_idx = end_idx - converted_len
        ...     indices = np.linspace(start_idx, end_idx, num=clip_len)
        ...     indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
        ...     return indices


        >>> # load video
        >>> file_path = hf_hub_download(
        ...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample frames
        >>> num_frames = model.config.num_image_with_embedding
        >>> indices = sample_frame_indices(
        ...     clip_len=num_frames, frame_sample_rate=4, seg_len=container.streams.video[0].frames
        ... )
        >>> frames = read_video_pyav(container, indices)

        >>> pixel_values = processor(images=list(frames), return_tensors="pt").pixel_values

        >>> generated_ids = model.generate(pixel_values=pixel_values, max_length=50)

        >>> print("Generated caption:", processor.batch_decode(generated_ids, skip_special_tokens=True))
        Generated caption: ['a woman is sitting at a table and she is talking about the food she is holding.']
        ```
        NF)r   r9   rO  r   rU   r   r   r   r  rN  r  r   r;   r   )losslogitsr   r%   r&   )rR   r  r  r   r  r   r   rQ   rv   r   r	   r   rA   r   r   r%   r&   )rQ   rT   r   r9   rO  r   rU   r  r   r   r   r  rN  r  r   r  r  r  num_image_tokensshifted_logitsloss_fctr   s                         r0   r]   zGitForCausalLM.forward  s   t &1%<kk$+B]I(()%%'+/!5%=#  
 
 "!*_--#x/5a8BGZ#AAA'7':AAA$=>IIKKNAAAqrrE]--//F'))H8N//DK4JKKV[[Y[__]]D 	FY,F)-)9TGf$$vE%#3!/)
 
 
 	
r/   c                    |F|                                 }|j        d         |k    r|}n|j        d         dz
  }|d d |d f         }|j        }||                    |          }|||                    dd           ||dS )Nr   rO  )rT   r   rO  r   r   )r  r   new_onesget)	rQ   rT   r   r   r   kwargspast_lengthremove_prefix_lengthrZ   s	            r0   prepare_inputs_for_generationz,GitForCausalLM.prepare_inputs_for_generationI  s     &)88::K q!K//'2$$ (1q'9A'=$!!!!%9%:%:":;I  o!&//<<N #,"JJ~t<<."
 
 	
r/   c                 T    d}|D ]!}|t          fd|D                       fz  }"|S )Nr.   c              3   t   K   | ]2}|                     d                     |j                            V  3dS )r   N)index_selectr   r   )r   
past_statebeam_idxs     r0   r  z0GitForCausalLM._reorder_cache.<locals>.<genexpr>l  sC      nnU_j--aZ=N1O1OPPnnnnnnr/   )r  )rQ   r   r  reordered_past
layer_pasts     `  r0   _reorder_cachezGitForCausalLM._reorder_cacheh  sQ    ) 	 	Jnnnncmnnnnn NN r/   )NNNNNNNNNNNFN)NNN)r'   r(   r)   _tied_weights_keysr?   r  r  r   r  r  r   r   r  r   r+   r`   r   r   r   r   r   r]   r  r  ra   rb   s   @r0   r  r  o  s        **      % % % +*+?+F+FGd+e+eff+AP_``` -115/3/3,004)-FJ$(,0/3).&*A
 A
EL)A
 !.A
 u|,	A

 u|,A
 EL)A
  -A
 &A
 "%tEL/A(A"BCA
 D>A
 $D>A
 'tnA
 #'A
 d^A
 
uU\"$::	;A
 A
 A
 a` gfA
H OS
 
 
 
>      r/   r  )Kr*   r   dataclassesr   typingr   r   r   r   r+   torch.utils.checkpointr   torch.nnr	   activationsr   cache_utilsr   r   
file_utilsr   
generationr   modeling_attn_mask_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   r   configuration_gitr   r    
get_loggerr'   ro   _CHECKPOINT_FOR_DOCr  r"   r  r2   rd   r   r   r   r   r   r   r   r  GIT_START_DOCSTRINGr  r  r[  rb  r  r  r  r  r  r  r  r  r.   r/   r0   <module>r'     s,       ! ! ! ! ! ! / / / / / / / / / / / /            % % % % % % ! ! ! ! ! ! . . . . . . . . % % % % % % ) ) ) ) ) ) B B B B B B            . - - - - - l l l l l l l l l l              : 9 9 9 9 9 9 9 
	H	%	%*  ? ? ? ? ?; ? ? ?8- - - - -BI - - -`r r r r rry r r rl    BI     
/ / / / /29 / / /f    bi        	   . . . . .ry . . .ba
 a
 a
 a
 a
 a
 a
 a
H* * * * * * * *B  A JP P P P P") P P Ph    29    e2 e2 e2 e2 e2 e2 e2 e2R/ / / / /BI / / /f^
 ^
 ^
 ^
 ^
ry ^
 ^
 ^
B $8
 8
 8
 8
 8
29 8
 8
 8
v Y 5
 5
 5
 5
 5
' 5
 5
	 5
p
2 
2 
2 
2 
2BI 
2 
2 
2 ) 
E
 E
 E
 E
 E
! E
 E
 
E
P `bu | | | | |' | | | | |r/   