
    g                        d Z ddlZddlmZmZmZ ddlZddlZddlZddlm	Z	 ddl
mZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+  e#            rddl,m-Z-  e&j.        e/          Z0dZ1dZ2dZ3de4de4dej5        fdZ6ej7        j8        d             Z9dej5        dej5        fdZ:dej5        dej5        dej5        dej5        fdZ; G d  d!e	j<                  Z= G d" d#e=          Z>e=e>d$Z? G d% d&e	j<                  Z@ G d' d(e	j<                  ZA G d) d*e          ZBd+ZCd,ZDd-ZEd.ZF e!d/eC           G d0 d1eB                      ZG e!d2eC           G d3 d4eBe                      ZH e!d5eC           G d6 d7eB                      ZI e!d8eC           G d9 d:eB                      ZJdS );zPyTorch GPT-J model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)CacheDynamicCacheStaticCache)GenerationMixin)AttentionMaskConverter)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPast)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_flash_attn_2_available#is_flash_attn_greater_or_equal_2_10is_torch_fx_proxylogging)assert_device_mapget_device_map   )
GPTJConfig)_flash_attention_forwardz$hf-internal-testing/tiny-random-gptjzEleutherAI/gpt-j-6Br    num_posdimreturnc                    ddt          j        d|dt           j                  |z  z  z  }t          j        dt          j        | t           j                                                  |                                          }t          j        t          j        |          t          j        |          fd          S )	N      ?i'  r      dtypezi , j -> i jr   r#   )torcharangeint64einsumfloatcatsincos)r"   r#   inv_freqsinusoid_inps       b/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/gptj/modeling_gptj.pycreate_sinusoidal_positionsr6   =   s    eQQek J J JS PQRH<WEK0X0X0X0^0^0`0`bjkkqqssL9ei--uy/F/FGQOOOO    c                 v    |                      |j                                      |j        d         dd          S Nr   r   )todevicerepeatshape)embed_positionsposition_idss     r5   get_embed_positionsr@   C   s5    l12299,:LQ:OQRTUVVVr7   xc                     | d d d d d d d d df         }| d d d d d d dd df         }t          j        | |fd          } |                     d          S )Nr'   r   r*   )r+   stackflatten)rA   x1x2s      r5   rotate_every_tworI   H   ss    	
111aaaCCaC<B	
111aaaADqD=	BbS"I2&&&A99R==r7   tensorr1   r2   c                     t          j        |d d d d d d d f         dd          }t          j        |d d d d d d d f         dd          }| |z  t          |           |z  z   S )Nr'   r
   )r+   repeat_interleaverI   )rJ   r1   r2   s      r5   apply_rotary_pos_embrM   O   sy    

!#aaaD!!!m"4a
;
;C

!#aaaD!!!m"4a
;
;CSL-f55;<<r7   c                       e Zd Zd fd	Zd Zd Z	 	 ddZd Z	 	 	 	 	 	 	 ddej	        d	e
e         d
e
ej	                 de
ej                 de
ej	                 de
e         de
e         de
ej                 deeej        eej                 f         e
eej        eej                 eej        df         f                  f         fdZ xZS )GPTJAttentionNc                 b   t                                                       || _        |j        }t	          j        |j                  | _        t	          j        |j                  | _	        d| _
        || _        |(t                              d| j        j         d           |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t)          d| j         d| j         d          t+          j        t+          j        | j        t*          j                                                t+          j                              | _        t	          j        | j        | j        d	          | _        t	          j        | j        | j        d	          | _        t	          j        | j        | j        d	          | _        t	          j        | j        | j        d	          | _         |j!        | _!        | j!        p| j        }tE          ||          | _#        d S )
NTzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r(   Fbias)$super__init__configmax_position_embeddingsr   Dropout
attn_pdropattn_dropoutresid_pdropresid_dropout	is_causal	layer_idxloggerwarning_once	__class____name__hidden_size	embed_dimnum_attention_headshead_dim
ValueErrorr+   sqrtrJ   float32r:   get_default_dtype
scale_attnLineark_projv_projq_projout_proj
rotary_dimr6   r>   )selfrU   r]   max_positionspos_embd_dimr`   s        r5   rT   zGPTJAttention.__init__V   s   6Jv'899Z(:;;",!8 , , ,    +#)#= $*BB=433t~EEHX\Xf H H+/+CH H H    *U\$-u}%U%U%UVVYYZ_ZqZsZsttiUKKKiUKKKiUKKK	$.$.uMMM +8$.:=,WWr7   c                    |                                 dd         ||fz   }|                    |          }|r|S t          |j                  dk    r|                    ddddd          S t          |j                  dk    r|                    dddd          S t          d	t          |j                             )
zO
        Splits hidden dim into attn_head_size and num_attention_heads
        NrC      r   r   r
   r'      3Input tensor rank should be one of [4, 5], but is: )sizeviewlenr=   permuterf   )rq   rJ   rd   attn_head_sizerotary	new_shapes         r5   _split_headszGPTJAttention._split_headsy   s     KKMM#2#&*=~)NN	Y'' 	Mv|!!>>!Q1a000!##>>!Q1---fSVW]WcSdSdffgggr7   c                    t          |j                  dk    r,|                    ddddd                                          }ngt          |j                  dk    r+|                    dddd                                          }n$t	          dt          |j                             |                                dd	         ||z  fz   }|                    |          S )
zR
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        ru   r   r   r
   r'   rv   rw   NrD   )rz   r=   r{   
contiguousrf   rx   ry   )rq   rJ   rd   r|   r~   s        r5   _merge_headszGPTJAttention._merge_heads   s     v|!!^^Aq!Q22==??FF!##^^Aq!Q//::<<FFfSVW]WcSdSdffgggKKMM#2#&*=*N)PP	{{9%%%r7   c                    |                     t          j                  }|                     t          j                  }t          j        ||                    dd                    }|| j        z  }|$|d d d d d d d |j        d         f         }||z   }t          j        	                    |d          }|                     |j
                  }|                     |          }|||z  }t          j        ||          }||fS )NrC   rD   r*   )r:   r+   rh   matmul	transposerj   r=   r   
functionalsoftmaxr)   rY   )	rq   querykeyvalueattention_mask	head_maskattn_weightscausal_maskattn_outputs	            r5   _attnzGPTJAttention._attn   s     ''ffU]##|E3==R+@+@AA#do5%(AAAqqq/CIbM/)ABK'+5L},,\r,BB#u{33((66  ')3Ll<77L((r7   c                     | j         }|j        |j        k    r!|                    |j                  }|| _         |                    |j        d         dd          S r9   )r>   r;   r:   r<   r=   )rq   r?   r>   s      r5   _get_embed_positionsz"GPTJAttention._get_embed_positions   sY    .!\%888-001DEEO#2D %%l&8&;QBBBr7   Fhidden_states
layer_pastr   r?   r   	use_cacheoutput_attentionscache_positionr$   .c	                 H   |                      |          }	|                     |          }
|                     |          }|                     |	| j        | j        d          }	|                     |
| j        | j        d          }
|                     || j        | j        d          }t          |          st          j        	                                rt          | j        |          }n|                     |          }|                    d                              dd|j        d                   }t          j        |d|          }t          j        ||j        d         dz  d          \  }}| j        |
d d d d d d d | j        f         }|
d d d d d d | j        d f         }|	d d d d d d d | j        f         }|	d d d d d d | j        d f         }t'          |||          }t'          |||          }t          j        ||gd          }
t          j        ||gd          }	n"t'          |
||          }
t'          |	||          }	|
                    dddd          }
|	                    dddd          }	|,||| j        |d	}|                    |
|| j        |          \  }
}|                     |	|
|||          \  }}|                     || j        | j                  }|                     |          }|                     |          }||f}|r||fz  }|S )
NTFrC   r   r'   r*   r   r
   r1   r2   partial_rotation_sizer   )rn   rl   rm   r   rd   re   r   r+   jit
is_tracingr@   r>   r   	unsqueezer<   r=   gathersplitrp   rM   r0   r{   updater]   r   r   ro   r[   )rq   r   r   r   r?   r   r   r   r   r   r   r   r>   repeated_position_idssincosr1   r2   k_rotk_passq_rotq_passcache_kwargsr   r   outputss                            r5   forwardzGPTJAttention.forward   sY    M**kk-((M**!!%)A4=RVWWT%=t}dSS!!%)A4=RWXX\** 	Fei.B.B.D.D 	F 2$2FUUOO"77EEO , 6 6r : : A A!QH]^`Ha b boq2GHH;vv|B'71'<"EEES?&111aaa!24?!223EAAAqqq$/"3"334F!!!QQQ#4T_#445E111aaaDO$5$556F(S99E(S99E)UFO444CIufo2666EE&sC55C(S99Ekk!Q1%%aAq))!)-"0	 L $**3t~|TTJC %)JJuc5.R[$\$\!\''T5Mt}]]mmK00((55
+ 	'&Gr7   N)NNNNNNFFN)ra   
__module____qualname__rT   r   r   r   r   r+   FloatTensorr   r   
LongTensorboolr   r   Tensorr   __classcell__r`   s   @r5   rO   rO   U   s       !X !X !X !X !X !XFh h h& & &$ ) ) ) )>C C C '+6:3715$),159H H(H UOH !!23	H
 u/0H E-.H D>H $D>H !!12H 
elE%,//0u|U5<%8%c@Q:RRST	V
H H H H H H H Hr7   rO   c                   |    e Zd ZdZ fdZ	 	 	 	 	 	 	 ddej        dee         deej                 deej	                 d	eej                 d
ee
         dee
         deej	                 deeej        eej                 f         eeej        eej                 eej        df         f                  f         fdZ xZS )GPTJFlashAttention2aD  
    GPTJ flash attention module. This module inherits from `GPTJAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 b     t                      j        |i | t                       | _        d S r   )rS   rT   r   _flash_attn_uses_top_left_mask)rq   argskwargsr`   s      r5   rT   zGPTJFlashAttention2.__init__  s9    $)&)))
 3V2W2W.W+++r7   NFr   r   r   r?   r   r   r   r   r$   .c	           
      x	   |                      |          }	|                     |          }
|                     |          }|                     |	| j        | j        d          }	|                     |
| j        | j        d          }
|                     || j        | j        d          }t          |          st          j        	                                rt          | j        |          }n|                     |          }|                    d                              dd|j        d                   }t          j        |d|          }t          j        ||j        d         dz  d          \  }}| j        |
d d d d d d d | j        f         }|
d d d d d d | j        d f         }|	d d d d d d d | j        f         }|	d d d d d d | j        d f         }t'          |||          }t'          |||          }t          j        ||gd          }
t          j        ||gd          }	n"t'          |
||          }
t'          |	||          }	|
                    dddd          }
|	                    dddd          }	|,||| j        |d	}|                    |
|| j        |          \  }
}|
                    dddd                                          }
|	                    dddd                                          }	|                    dddd                                          }|	j        }|t          j        k    rt          j                    rt          j                    }n3t;          | j        d
          r| j        j        }n| j         j         j        }tB          "                    d| d           |	#                    |          }	|
#                    |          }
|#                    |          }| j$        r| j        j%        nd}|	j        d         }tM          |	|
||||| j'        | j(                  }|)                    |j        d         |j        d         |j        d         |j        d         z            }| *                    |          }| +                    |          }||f}|r||fz  }|S )NTFrC   r   r'   r*   r   r
   r   _pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .        )dropoutr\   use_top_left_mask),rn   rl   rm   r   rd   re   r   r+   r   r   r@   r>   r   r   r<   r=   r   r   rp   rM   r0   r{   r   r]   r   r)   rh   is_autocast_enabledget_autocast_gpu_dtypehasattrrU   r   weightr^   r_   r:   trainingrX   r!   r\   r   reshapero   r[   )rq   r   r   r   r?   r   r   r   r   r   r   r   r>   r   r   r1   r2   r   r   r   r   r   input_dtypetarget_dtypeattention_dropoutquery_lengthr   r   r   s                                r5   r   zGPTJFlashAttention2.forward  s    M**kk-((M**!!%)A4=RVWWT%=t}dSS!!%)A4=RWXX\** 	Fei.B.B.D.D 	F 2$2FUUOO"77EEO , 6 6r : : A A!QH]^`Ha b boq2GHH;vv|B'71'<"EEES?&111aaa!24?!223EAAAqqq$/"3"334F!!!QQQ#4T_#445E111aaaDO$5$556F(S99E(S99E)UFO444CIufo2666EE&sC55C(S99E
 kk!Q1%%aAq)) !)-"0	 L $**3t~|TTJC kk!Q1%%0022aAq))4466aAq))4466 k%-''(** 8$;==&?@@ 8#{B#{17$ $ $ $   HH\**E&&&&CHH\**E6:mLDK22{1~ 0%n"A	
 	
 	
 #**q!<#5a#8,:LQ:OR^RdefRg:g
 
 mmK00((55
+ 	'&Gr7   r   )ra   r   r   __doc__rT   r+   r   r   r   r   r   r   r   r   r   r   r   s   @r5   r   r     sZ        X X X X X '+6:3715$),159~ ~(~ UO~ !!23	~
 u/0~ E-.~ D>~ $D>~ !!12~ 
elE%,//0u|U5<%8%c@Q:RRST	V
~ ~ ~ ~ ~ ~ ~ ~r7   r   )eagerflash_attention_2c                   N     e Zd Z fdZdeej                 dej        fdZ xZS )GPTJMLPc                 (   t                                                       |j        }t          j        ||          | _        t          j        ||          | _        t          |j                 | _	        t          j
        |j                  | _        d S r   )rS   rT   n_embdr   rk   fc_infc_outr   activation_functionactrW   rZ   r   )rq   intermediate_sizerU   rc   r`   s       r5   rT   zGPTJMLP.__init__  so    M	Yy*;<<
i 19==&45z&"455r7   r   r$   c                     |                      |          }|                     |          }|                     |          }|                     |          }|S r   )r   r   r   r   )rq   r   s     r5   r   zGPTJMLP.forward  sL    

=11//M22]33r7   )	ra   r   r   rT   r   r+   r   r   r   r   s   @r5   r   r     se        6 6 6 6 6Xe.?%@ UEV        r7   r   c                   T    e Zd Zd fd	Z	 	 	 	 	 	 	 ddeej                 dee         deej                 deej                 deej                 d	ee	         d
ee	         deej                 de
eej                 eeej        eej        df         f                  f         fdZ xZS )	GPTJBlockNc                 ,   t                                                       |j        |j        n	d|j        z  }t	          j        |j        |j                  | _        t          |j	                 ||          | _
        t          ||          | _        d S )Nrv   eps)rS   rT   n_innerr   r   	LayerNormlayer_norm_epsilonln_1GPTJ_ATTENTION_CLASSES_attn_implementationattnr   mlp)rq   rU   r]   	inner_dimr`   s       r5   rT   zGPTJBlock.__init__  s|    &,n&@FNNa&-FW	LF4MNNN	*6+FGPYZZ	9f--r7   Fr   r   r   r?   r   r   r   r   r$   .c	           
          |}	|                      |          }|                     ||||||||          }
|
d         }|
dd          }|                     |          }||z   |	z   }|r|f|z   }n|f|dd          z   }|S )Nr   r   r   r?   r   r   r   r   r   r   )r   r   r   )rq   r   r   r   r?   r   r   r   r   residualattn_outputsr   r   feed_forward_hidden_statess                 r5   r   zGPTJBlock.forward  s     !		-00yy'!)%/) ! 	
 	
 #1oqrr"%)XXm%<%<"#&@@8K 	5$&0GG$&4Gr7   r   r   )ra   r   r   rT   r   r+   r   r   r   r   r   r   r   r   r   r   s   @r5   r   r     s)       . . . . . . '+6:3715$),159" " 12" UO" !!23	"
 u/0" E-." D>" $D>" !!12" 
uU\"HU5<uGXZ]G]A^3^-_$``	a" " " " " " " "r7   r   c                   V     e Zd ZdZeZdZdZdZdgZ	dZ
dZdZdZdZdZ fdZd Z xZS )	GPTJPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    transformerTr   past_key_valuesFc                 :     t                      j        |i | d S r   )rS   rT   )rq   inputsr   r`   s      r5   rT   zGPTJPreTrainedModel.__init__  s%    &+F+++++r7   c                    t          |t          j        f          rT|j        j                            d| j        j                   |j         |j        j        	                                 dS dS t          |t          j
                  r_|j        j                            d| j        j                   |j        +|j        j        |j                 	                                 dS dS t          |t          j                  r?|j        j        	                                 |j        j                            d           dS dS )zInitialize the weights.r   )meanstdNr&   )
isinstancer   rk   r   datanormal_rU   initializer_rangerR   zero_	Embeddingpadding_idxr   fill_)rq   modules     r5   _init_weightsz!GPTJPreTrainedModel._init_weights  s+   fryl++ 	* M&&CT[5R&SSS{& &&((((( '&-- 	*M&&CT[5R&SSS!-"6#56<<>>>>> .--- 	*K""$$$M$$S)))))	* 	*r7   )ra   r   r   r   r    config_classbase_model_prefixis_parallelizablesupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_cache_class_supports_quantized_cache_supports_static_cache!_supports_param_buffer_assignmentrT   r   r   r   s   @r5   r   r     s         
 L%&*#$"3!  $!(-%, , , , ,* * * * * * *r7   r   aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`GPTJConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance, see our
            [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
            of shape `(batch_size, sequence_length)`.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
            this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
            the complete sequence length.
a  
    This is an experimental feature and is a subject to change at a moment's notice. Uses a device map to distribute
    attention modules of the model across several devices. If no device map is given, it will evenly distribute blocks
    across all devices.

    Args:
        device_map (`Dict[int, list]`, *optional*):
            A dictionary that maps attention modules to devices. Note that the embedding module and LMHead are always
            automatically mapped to the first device (for esoteric reasons). That means that the first device should
            have fewer attention modules mapped to it than other devices. For reference, the GPT-J models have the
            following number of attention modules:

                - gpt-j-6B: 28

    Example:

    ```python
    # Here is an example of a device map on a machine with 4 GPUs using gpt-j-6B, which has a total of 28 attention modules:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)
    ```
a6  
    Moves the model to CPU from a model parallel state.

    Example:

    ```python
    # On a 4 GPU machine with gpt-j-6B:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)  # Splits the model across several devices
    model.deparallelize()  # Put the model back on cpu and cleans memory by calling torch.cuda.empty_cache()
    ```
z_The bare GPT-J Model transformer outputting raw hidden-states without any specific head on top.c                       e Zd Z fdZ ee          dd            Z ee          d             Zd Z	d Z
 ee                    d                     eeeee          	 	 	 	 	 	 	 	 	 	 	 	 d d	eej                 d
eeeeeej                          f                  deej                 deej                 deej                 deej                 deej                 dee         dee         dee         dee         deej                 deeef         fd                        Zdej        dej        dej        d
edef
dZedej        dededej         dej!        dej        defd            Z" xZ#S )!	GPTJModelc                 *   t                                                     j        | _        j        | _        t          j        j        | j                  | _        t          j        j	                  | _
        t          j        fdt          j                  D                       | _        t          j        | j        j                  | _        d| _        d | _        d| _        |                                  j        dk    | _        d S )Nc                 2    g | ]}t          |           S ))r]   )r   ).0irU   s     r5   
<listcomp>z&GPTJModel.__init__.<locals>.<listcomp>  s&    ^^^1	&A > > >^^^r7   r   Fr   )rS   rT   r   rc   
vocab_sizer   r   wterW   
embd_pdropdrop
ModuleListrangen_layerhr   r   ln_fmodel_parallel
device_mapgradient_checkpointing	post_initr   _use_flash_attention_2rq   rU   r`   s    `r5   rT   zGPTJModel.__init__  s        +< 14>BBJv011	^^^^fnH]H]^^^__LV5NOOO	 $&+# 	&,&AEX&X###r7   Nc                    t          j        dt                     |Kt          t	          | j                  t          t          j        	                                                    n|| _
        t          | j
        t	          | j                             d| _        d| j
                                        v rdn5dt          t          | j
                                                            z   | _        dt          t#          | j
                                                            z   | _        | j                            | j                  | _        | j
                                        D ]D\  }}|D ]<}dt          |          z   }| j        |                             |          | j        |<   =E| j                            | j                  | _        d S )Na6  `GPTJModel.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'h.0': 0, 'h.1': 1, ...}Tcpucuda:)warningswarnFutureWarningr   rz   r  r  r+   cudadevice_countr  r   r  keysstrminfirst_devicemaxlast_devicer  r:   itemsr  )rq   r  kvblockcuda_devices         r5   parallelizezGPTJModel.parallelize  s    	
 	
 	
 NXM_N3tv;;ej.E.E.G.G(H(HIIIeo 	 	$/3tv;;777"%*do.B.B.D.D%D%DEE'TWX[\`\k\p\p\r\rXsXsTtTtJt"ST_-A-A-C-C)D)D%E%EE8;;t011O))++ 	> 	>DAq > >%A. $u 0 0 = =u> ILL!122			r7   c                    t          j        dt                     d| _        d | _        d| _        d| _        | j                            d          | _        t          t          | j                            D ]*}| j        |                             d          | j        |<   +| j                            d          | _        t          j                                         d S )N\Like `parallelize`, `deparallelize` is deprecated and will be removed in v5 of Transformers.Fr  )r!  r"  r#  r  r  r)  r+  r  r:   r  rz   r  r  r+   r$  empty_cache)rq   indexs     r5   deparallelizezGPTJModel.deparallelize  s    j	
 	
 	
 $! 8;;u%%3tv;;'' 	4 	4E F5M,,U33DF5MMILL''	
     r7   c                     | j         S r   r  rq   s    r5   get_input_embeddingszGPTJModel.get_input_embeddings  s	    xr7   c                     || _         d S r   r8  rq   new_embeddingss     r5   set_input_embeddingszGPTJModel.set_input_embeddings  s    !r7   batch_size, sequence_length
checkpointoutput_typer   real_checkpoint	input_idsr   r   token_type_idsr?   r   inputs_embedsr   r   output_hidden_statesreturn_dictr   r$   c                    |	|	n| j         j        }	|
|
n| j         j        }
||n| j         j        }||n| j         j        }|d u |d uz  rt          d          | j        r%| j        r|rt          	                    d           d}|| 
                    |          }d}|rVt          |t                    sAd}|t                      }n.t          j        |          }t          	                    d           |j        d         }|7||                                nd}t#          j        |||z   |j                  }||                    d          }|                     |||||	          }|                     || j         j                  }|}|0|                    d	|          }| 
                    |          }||z   }|                     |          }d	||                    d	          f}d }|	rd
nd }|
rd
nd }t7          | j                  D ]\  }}| j        rt"          j                            |j                   |H|j         !                    |j                  |_         |j"        !                    |j                  |_"        ||!                    |j                  }t          |t"          j#                  r|!                    |j                  }|
r||fz   }| j        r0| j        r)| $                    |j%        |d ||||         ||	|	  	        }n |||||||         ||	|          }|d         }|du r|d         }|	r|||rdnd         fz   }| j        rn| j&        '                                D ]T\  }}||d	         k    rCdtQ          |          z   | j)        k    r(|!                    dtQ          |dz             z             }U| *                    |          }|                    |          }|
r||fz   }|r|nd }|r|+                                }|stY          d ||||fD                       S t[          ||||          S )Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FTzWe detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class (https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)r   r   r;   rC    r   r'   r   c              3      K   | ]}||V  	d S r   rK  )r  r.  s     r5   	<genexpr>z$GPTJModel.forward.<locals>.<genexpr>j  s1        bcbobobobobo r7   )last_hidden_stater   r   
attentions).rU   r   rG  r   use_return_dictrf   r  r   r^   r_   r  r   r   r   from_legacy_cacher=   get_seq_lengthr+   r,   r;   r   _update_causal_maskget_head_maskr  ry   r  rx   	enumerater  r  r$  
set_device	key_cacher:   value_cacher   _gradient_checkpointing_func__call__r  r,  r'  r+  r  to_legacy_cachetupler   )rq   rD  r   r   rE  r?   r   rF  r   r   rG  rH  r   return_legacy_cache
seq_lengthpast_key_values_lengthr   r   token_type_embedsoutput_shapenext_decoder_cacheall_self_attentionsall_hidden_statesr  r/  r   r-  r.  
next_caches                                r5   r   zGPTJModel.forward  sg   , 2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]-t";< 	[YZZZ& 	"4= 	" "##p   "	  HHY//M $ 
	Z?? 
	"&&"..."."@"Q"Q##^   #(+
!IXId_%C%C%E%E%Ejk""\&(>(KTaTh  N )33A66L..M>?L]
 
 &&y$+2EFF	%%+00Z@@N $ 8 8),==M		-00J(:(:2(>(>?!$5?bb4"6@BBD!$&)) 5	O 5	OHAu" C
%%m&:;;; #.0?0I0L0L]Ma0b0bO-2A2M2P2PQ^Qe2f2fO/ *"-..1E"F"FKi66 C )]-A B BI# I$58H$H!* t} ;;N! aL%"
 
  %"/.#.!-'l'&7#1	 	 	 $AJMD  %,QZ"  ^&9W)EZQQYZ=[<]&]# " O O1133 O ODAqAbEzzgA&6$:J&J&J(5(8(83q1u::9M(N(N		-00%**<88 	E 1]4D D+4>''$
 	6#3355J 	  ):7HJ]^      '+&+*	
 
 
 	
r7   input_tensorc           
         | j         j        dk    r
|d|v r|S d S ||                                nd}t          |t                    }| j         j        dk    r#|s!|st          j        |||| j                  rd S |j        |j	        }	}|j
        d         }
|r|                                }n/t          |t          j                  r|j
        d         n||
z   dz   }|                     ||
|||	||j
        d                   }| j         j        dk    rB|@|j	        j        d	k    r0|s.t          j        |          j        }t          j        ||          }|S )
Nr   r   r   sdpa)rF  r_  is_trainingr   rC   )sequence_lengthtarget_lengthr)   r;   r   
batch_sizer$  )rU   r   rR  r   r   r   _ignore_causal_mask_sdpar   r)   r;   r=   get_max_cache_shaper+   r   5_prepare_4d_causal_attention_mask_with_cache_positiontypefinfor(  _unmask_unattended)rq   r   rf  r   r   r   past_seen_tokensusing_static_cacher)   r;   rj  rk  r   	min_dtypes                 r5   rS  zGPTJModel._update_causal_maskv  s    ;+/BBB)c^.C.C%%4
 @O?Z?99;;;`a'EE ;+v55>P5Yj5%>*'7 M	    t$*L,?v&,Q/ 	+??AAMM nel;;<$R((%7!;  PP+')#)!, Q 
 
 K,66*%*f44% 5 E**.I0CKQZ[[Kr7   rj  rk  r)   r;   rl  c                    | |                                  dk    r| }n+t          j        |          j        }	t          j        ||f|	||          }|dk    rt          j        |d          }|t          j        ||          |                    dd          k    z  }|ddddddf                             |ddd          }| |	                                }| j
        d         }
|ddddddd|
f         | ddddddf         z   }|dk    }|ddddddd|
f                             ||	          |ddddddd|
f<   |S )	a  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            device (`torch.device`):
                The device to plcae the 4D attention mask on.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nrv   )
fill_valuer)   r;   r   )diagonalrJ  rC   r   )r#   r+   rq  r(  fulltriur,   r   expandcloner=   masked_fill)r   rj  rk  r)   r;   r   rl  r   r   ru  mask_lengthpadding_masks               r5   ro  z?GPTJModel._prepare_4d_causal_attention_mask_with_cache_position  s   D %.*<*<*>*>!*C*C(KKE**.I* -0Ye\b  K !###jqAAA5<fEEEH^H^_acdHeHeeeK%dD!!!QQQ&67>>z1bRTUUK))//11,226*111aaaL[L+@ANSTSTSTVZ\`bcbcbcScDdd+q05@AAAqqq,;,AV5W5c5c )6 6AAAqqq!!!\k\12 r7   r   NNNNNNNNNNNN)$ra   r   r   rT   r   PARALLELIZE_DOCSTRINGr1  DEPARALLELIZE_DOCSTRINGr6  r:  r>  r   GPTJ_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_REAL_CHECKPOINT_FOR_DOCr   r+   r   r   r   r   r   r   r   r   rS  staticmethodintr)   r;   ro  r   r   s   @r5   r	  r	    s       
Y Y Y Y Y( /003 3 3 1032 122! ! 32!  " " " +*+@+G+GHe+f+fgg&+$0	   15NR6:59371559$(,0/3&*59]
 ]
E,-]
 "%uU5<5H/I(I"JK]
 !!23	]

 !!12]
 u/0]
 E-.]
   12]
 D>]
 $D>]
 'tn]
 d^]
 !!12]
 
u--	.]
 ]
 ]
  hg]
@?? l? 	?
 ?  ? ? ? ?B 555 5 {	5
 5 5 5 5 5 \5 5 5 5 5r7   r	  zK
    The GPT-J Model transformer with a language modeling head on top.
    c            !           e Zd ZdgZ fdZ ee          dd            Z ee          d             Z	d Z
d Z ee                    d                     eeeee	          	 	 	 	 	 	 	 	 	 	 	 	 	 dd
eej                 deeeeeej                          f                  deej                 deej                 deej                 deej                 deej                 deej                 dee         dee         dee         dee         deej                 deeef         fd                        Zedeeej                          dej        deeej                          fd            Z xZ S )GPTJForCausalLMzlm_head.weightc                     t                                          |           t          |          | _        t	          j        |j        |j                  | _        d| _	        d | _
        |                                  d S NF)rS   rT   r	  r   r   rk   r   r  lm_headr  r  r  r  s     r5   rT   zGPTJForCausalLM.__init__  sl       $V,,y0ABB $ 	r7   Nc                    t          j        dt                     |Pt          t	          | j        j                  t          t          j	        
                                                    n|| _        t          | j        t	          | j        j                             | j                            | j                   | j                            | j        j                  | _        d| _        d S )NaT  `GPTJForCausalLM.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'transformer.h.0': 0, 'transformer.h.1': 1, ...}T)r!  r"  r#  r   rz   r   r  r  r+   r$  r%  r  r   r1  r  r:   r)  r  )rq   r  s     r5   r1  zGPTJForCausalLM.parallelize  s    - 	
 	
 	
 ! 3t/122E%*:Q:Q:S:S4T4TUUU 	
 	$/3t/?/A+B+BCCC$$T_555|t'7'DEE"r7   c                 2   t          j        dt                     | j                                         | j                            d          | _        | j                            d          | _        d| _        t          j	        
                                 d S )Nr3  r  F)r!  r"  r#  r   r6  r:   r  r  r+   r$  r4  r9  s    r5   r6  zGPTJForCausalLM.deparallelize  s    j	
 	
 	
 	&&(((+..u55|u--#
     r7   c                     | j         S r   r  r9  s    r5   get_output_embeddingsz%GPTJForCausalLM.get_output_embeddings%  s
    |r7   c                     || _         d S r   r  r<  s     r5   set_output_embeddingsz%GPTJForCausalLM.set_output_embeddings(  s    %r7   r?  r@  rD  r   r   rE  r?   r   rF  labelsr   r   rG  rH  r   r$   c                    ||n| j         j        }|                     ||||||||	|
|||          }|d         }| j        rMt          j                            | j        j                   |                    | j	        j
        j                  }| 	                    |                              t          j                  }d}||                    |j                  }|dddddf                                         }|dddf                                         }t                      } ||                    d|                    d                    |                    d                    }|                    |j                  }|s|f|dd         z   }||f|z   n|S t%          |||j        |j        |j                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)r   r   rE  r?   r   rF  r   r   rG  rH  r   r   .rC   r   losslogitsr   r   rO  )rU   rP  r   r  r+   r$  rV  r)  r:   r  r   r;   rh   r   r   ry   rx   r)   r   r   r   rO  )rq   rD  r   r   rE  r?   r   rF  r  r   r   rG  rH  r   transformer_outputsr   	lm_logitsr  shift_logitsshift_labelsloss_fctoutputs                         r5   r   zGPTJForCausalLM.forward+  s   : &1%<kk$+B]"..+))%'/!5#) / 
 
 ,A.  	IJ!!$"2"?@@@),,T\-@-GHHM
 LL//225=AA	YYy/00F$S#2#qqq[1<<>>L!#qrr'?5577L'))H8L--b,2C2CB2G2GHH,J[J[\^J_J_``D77=.//D 	F\$7$;;F)-)9TGf$$vE%/?-;*5
 
 
 	
r7   beam_idxc                 :    t          fd| D                       S )a  
        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
        beam_idx at every generation step.
        c              3   N   K   | ]}t          fd |D                       V   dS )c              3   t   K   | ]2}|                     d                     |j                            V  3dS )r   N)index_selectr:   r;   )r  
past_stater  s     r5   rM  z;GPTJForCausalLM._reorder_cache.<locals>.<genexpr>.<genexpr>  sC      jjQ[*))!X[[9J-K-KLLjjjjjjr7   Nr\  )r  r   r  s     r5   rM  z1GPTJForCausalLM._reorder_cache.<locals>.<genexpr>  sU       
 
 jjjj_ijjjjj
 
 
 
 
 
r7   r  )r   r  s    `r5   _reorder_cachezGPTJForCausalLM._reorder_cache}  s=      
 
 
 
-
 
 
 
 
 	
r7   r   )NNNNNNNNNNNNN)!ra   r   r   _tied_weights_keysrT   r   r  r1  r  r6  r  r  r   r  r  r   r  r   r  r  r   r+   r   r   r   r   r   r   r   r   r  r  r   r   s   @r5   r  r    s        ++
 
 
 
 
 /00# # # 10#$ 122	! 	! 32	!  & & & +*+@+G+GHe+f+fgg&*$0	   15NR6:59371559-1$(,0/3&*59I
 I
E,-I
 "%uU5<5H/I(I"JKI
 !!23	I

 !!12I
 u/0I
 E-.I
   12I
 )*I
 D>I
 $D>I
 'tnI
 d^I
 !!12I
 
u,,	-I
 I
 I
  hgI
V 
uU\23
?D|
	uU\"	#
 
 
 \
 
 
 
 
r7   r  a  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       e Zd Z fdZ ee                    d                     edee	e
          	 	 	 	 	 	 	 	 	 	 	 	 ddeej                 deeeej                                   deej                 d	eej                 d
eej                 deej                 deej                 deej                 dee         dee         dee         dee         deeef         fd                        Z xZS )GPTJForSequenceClassificationc                    t                                          |           |j        | _        t          |          | _        t          j        |j        | j        d          | _        d| _	        d | _
        |                                  d S )NFrQ   )rS   rT   
num_labelsr	  r   r   rk   r   scorer  r  r  r  s     r5   rT   z&GPTJForSequenceClassification.__init__  sz        +$V,,Yv}doEJJJ
 $ 	r7   r?  z4ydshieh/tiny-random-gptj-for-sequence-classificationr@  NrD  r   r   rE  r?   r   rF  r  r   r   rG  rH  r$   c                 "   ||n| j         j        }|                     ||||||||	|
||          }|d         }|                     |          }||j        d         }n|j        d         }| j         j        |dk    rt          d          | j         j        d}n|rt          j        || j         j                  	                                
                    d          dz
  }||j        d         z  }|                    |j                  }n)d}t                              | j        j         d           |t          j        ||j                  |f         }d}|t|                    |j                  }| j         j        f| j        dk    rd	| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j	        k    rd
| j         _        nd| j         _        | j         j        d	k    rWt-                      }| j        dk    r1 ||                                |                                          }n |||          }n| j         j        d
k    rGt1                      } ||                    d| j                  |                    d                    }n*| j         j        dk    rt5                      } |||          }|s|f|dd         z   }||f|z   n|S t7          |||j        |j        |j                  S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)
r   r   rE  r?   r   rF  r   r   rG  rH  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.rC   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rJ  
regressionsingle_label_classificationmulti_label_classificationr  )rU   rP  r   r  r=   pad_token_idrf   r+   eqr  argmaxr:   r;   r^   r_   r`   ra   r,   problem_typer  r)   longr	   squeezer   ry   r   r   r   r   rO  )rq   rD  r   r   rE  r?   r   rF  r  r   r   rG  rH  r  r   r  rl  sequence_lengthspooled_logitsr  r  r  s                         r5   r   z%GPTJForSequenceClassification.forward  sR   8 &1%<kk$+B]"..+))%'/!5# / 
 
 ,A.M** "+JJ&,Q/J;#+
a\]]];#+!$#(8It{7O#P#P#T#T#V#V#]#]^`#a#ade#e #3iob6I#I #3#6#6v}#E#E  #% ##~. ^ ^ ^  
 u|Jv}MMMO__`YY}344F{'/?a''/;DK,,_q((flej.H.HFL\a\eLeLe/LDK,,/KDK,{'<77"99?a''#8M$9$9$;$;V^^=M=MNNDD#8M6::DD)-JJJ+--x 2 22t G GUWYY)-III,..xv66 	F#%(;ABB(??F)-)9TGf$$vE/ /?-;*5
 
 
 	
r7   r  )ra   r   r   rT   r   r  r  r   r   r  r  r   r+   r   r   r   r   r   r   r   r   r   s   @r5   r  r    s             +*+@+G+GHe+f+fggI4$0	   15@D6:59371559-1$(,0/3&*`
 `
E,-`
 "%el(;"<=`
 !!23	`

 !!12`
 u/0`
 E-.`
   12`
 )*`
 D>`
 $D>`
 'tn`
 d^`
 
u66	7`
 `
 `
  hg`
 `
 `
 `
 `
r7   r  z
    The GPT-J Model transformer with a span classification head on top for extractive question-answering tasks like
    SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       e Zd Z fdZ ee                    d                     eee	e
e          	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 deej                 d	eej                 d
eej                 deej                 deej                 dee         dee         dee         deee	f         fd                        Z xZS )GPTJForQuestionAnsweringc                    t                                          |           |j        | _        t          |          | _        t          j        |j        |j                  | _        d| _	        d | _
        |                                  d S r  )rS   rT   r  r	  r   r   rk   rb   
qa_outputsr  r  r  r  s     r5   rT   z!GPTJForQuestionAnswering.__init__  sw        +$V,,)F$68IJJ $ 	r7   r?  r@  NrD  r   rE  r?   r   rF  start_positionsend_positionsr   rG  rH  r$   c                    ||n| j         j        }|                     |||||||	|
|	  	        }|d         }|                     |          }|                    dd          \  }}|                    d                                          }|                    d                                          }d}||t          |                                          dk    r-|                    d          	                    |j
                  }t          |                                          dk    r-|                    d          	                    |j
                  }|                    d          }|                    d|          }|                    d|          }t          |          } |||          } |||          }||z   dz  }|s||f|dd         z   }||f|z   n|S t          ||||j        |j        	          S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        N)r   rE  r?   r   rF  r   rG  rH  r   r   rC   r*   )ignore_indexr'   )r  start_logits
end_logitsr   rO  )rU   rP  r   r  r   r  r   rz   rx   r:   r;   clampr   r   r   rO  )rq   rD  r   rE  r?   r   rF  r  r  r   rG  rH  r   sequence_outputr  r  r  
total_lossignored_indexr  
start_lossend_lossr  s                          r5   r   z GPTJForQuestionAnswering.forward(  s7   > &1%<kk$+B]""))%'/!5# # 

 

 "!*11#)<<r<#:#: j#++B//::<<''++6688

&=+D?''))**Q.."1"9"9""="="@"@AT"U"U=%%''((1,, - 5 5b 9 9 < <Z=N O O(--a00M-33A}EEO)//=AAM']CCCH!,@@Jx
M::H$x/14J 	R"J/'!""+=F/9/EZMF**6Q+%!!/)
 
 
 	
r7   )NNNNNNNNNNN)ra   r   r   rT   r   r  r  r   r  r   r  r  r   r+   r   r   r   r   r   r   r   r   s   @r5   r  r    s            +*+@+G+GHe+f+fgg&0$0	   156:593715596:48,0/3&*H
 H
E,-H
 !!23H
 !!12	H

 u/0H
 E-.H
   12H
 "%"23H
   01H
 $D>H
 'tnH
 d^H
 
u22	3H
 H
 H
  hgH
 H
 H
 H
 H
r7   r  )Kr   r!  typingr   r   r   r+   torch.fxtorch.utils.checkpointr   torch.nnr   r   r	   activationsr   cache_utilsr   r   r   
generationr   modeling_attn_mask_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   r   r   r   r   r   utils.model_parallel_utilsr   r   configuration_gptjr    modeling_flash_attention_utilsr!   
get_loggerra   r^   r  r  r  r  r   r6   fxwrapr@   rI   rM   ModulerO   r   r   r   r   r   GPTJ_START_DOCSTRINGr  r  r  r	  r  r  r  rK  r7   r5   <module>r     s@      ) ) ) ) ) ) ) ) ) )             A A A A A A A A A A ! ! ! ! ! ! ; ; ; ; ; ; ; ; ; ; ) ) ) ) ) ) > > > > > >            . - - - - -                  L K K K K K K K * * * * * *  KJJJJJJ 
	H	%	%< 0 P P3 P5< P P P P W W W     = =EL =u| =X]Xd = = = =n n n n nBI n n nbN N N N N- N N Nd ,      bi   &* * * * *	 * * *Z#* #* #* #* #*/ #* #* #*L	 E N : ( e d d d d d# d d	 dN  	 R
 R
 R
 R
 R
)? R
 R
 R
j   u
 u
 u
 u
 u
$7 u
 u
 u
p   ]
 ]
 ]
 ]
 ]
2 ]
 ]
 ]
 ]
 ]
r7   