gdZddlZddlmZddlmZmZmZddlZddl Zddlm Z ddl m Z m Z mZddlmZdd lmZmZmZmZmZmZdd lmZdd lmZmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$d dl%m&Z&e$j'e(Z)dZ*dZ+da,dZ-dZ.dZ/dZ0Gddej1j2Z3Gddej1j2Z4Gdde j5Z6Gdde j5Z7Gdde j5Z8Gdd e j5Z9Gd!d"e j5Z:Gd#d$e j5Z;Gd%d&e j5Z<Gd'd(e j5Z=Gd)d*e j5Z>Gd+d,e j5Z?Gd-d.e j5Z@Gd/d0eZAd1ZBd2ZCe d3eBGd4d5eAZDe d6eBGd7d8eAZEGd9d:e j5ZFe d;eBGd<d=eAZGe d>eBGd?d@eAZHe dAeBGdBdCeAZIe dDeBGdEdFeAZJdS)GzPyTorch YOSO model.N)Path)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)"BaseModelOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_ninja_availableis_torch_cuda_availablelogging) YosoConfigzuw-madison/yoso-4096rcVddlm}d}|gd}|d|dddladS)Nr)loadcttjjjdz dz fd|DS)Nkernelsyosocg|]}|z Sr%).0file src_folders b/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/yoso/modeling_yoso.py z:load_cuda_kernels..append_root..?s444d T!444)r__file__resolveparent)filesr(s @r) append_rootz&load_cuda_kernels..append_root=sH(^^++--4;BYNQWW 4444e4444r+)zfast_lsh_cumulation_torch.cppzfast_lsh_cumulation.cuzfast_lsh_cumulation_cuda.cufast_lsh_cumulationT)verbose)torch.utils.cpp_extensionr r1lsh_cumulation)r r0 src_filess r)load_cuda_kernelsr69se......555 vvvwwID  48888000000r+c t|trFg}|D]?}|s|}||@|S|s|}|SN) isinstancelist is_contiguous contiguousappend input_tensorsouttensors r) to_contiguousrBHs-&& #  F'')) -**,, JJv     **,, 7)4466Mr+ct|tr>g}|D]7}|tj|dd8|Stj|ddS)N)pdim)r9r:r=r functional normalizer>s r)rIrIVsy-&&C# E EF JJr}..v.CC D D D D }&&}r&BBBr+ct|dkrtdt|dkrtdtj|d|d||z|j}dtj||jz}tj|||d|d||}tj|||d|d||}|dk }|dk } tj ||zd } tj | |zd } | | fS) Nr zQuery has incorrect size.zKey has incorrect size.rrDdevicerrErG) lensize ValueErrortorchrandnrLarangematmulreshapeintsum) querykeynum_hashhash_lenrmat raise_powquery_projectionkey_projection query_binary key_binary query_hashs r)hashingrc`s 5::<<A4555 388::!2333 ;uzz!}}ejjmmX5HQVQ] ^ ^ ^DU\(5<@@@@I|E40088A ST W_aijj\#t,,44SXXa[[#((1++xYabbN$q(--//L 1$))++J<)3<<>  Z^^-- --r+c:eZdZedZedZdS)YosoCumulationc b|d}dtjtj||ddtjz z |z}||dddddfz|dddddfz}tj||} |||||||||_| S)N hash_code_lenrrE)rQacosrT transposemathpisave_for_backwardconfig) ctx query_maskkey_maskrXrYvaluernrg expectationcumulation_values r)forwardzYosoCumulation.forwardts/ 5:el5#--B:O:O&P&PQQTXT[[[`mm !Jqqq!!!Tz$::Xaaaqqqj=QQ  < U;; j(KUSSS r+ct|}|j\}}}}}}|j}|d} tj||dd|z} tj| | dz |z} tj| dd| dz |z} tj|dd|} dd| | | dfS)NrgrErhrD)rB saved_tensorsrnrQrTrj)rogradrprqrsrXrYrrrnrg weighted_exp grad_querygrad_key grad_values r)backwardzYosoCumulation.backwardsT""?B?P< Hk5#u/ |D%//"b*A*ABB[P \,1Bc0IJJ < 6 6r2 > >QRARV[@[\\\+"7"7B"?"?FF T:xTAAr+N__name__ __module__ __qualname__ staticmethodrur}r%r+r)reressM   \   B B\ B B Br+rec:eZdZedZedZdS)YosoLSHCumulationc Z|d|dkrtd|d|dkrtd|d|dkrtd|d|dkrtd|d|dkrtd|d|dkrtd t|||||g\}}}}}|j}|d }|d } t d| z} |d r%t |||||| |d\} } nt|||| \} } t || || || |d} |||| | |||||_ | S) Nrz6Query mask and Key mask differ in sizes in dimension 0z3Query mask and Query differ in sizes in dimension 0z1Query mask and Key differ in sizes in dimension 0z8Query mask and Value mask differ in sizes in dimension 0rz,Key and Value differ in sizes in dimension 1rDz,Query and Key differ in sizes in dimension 2rZrg use_fast_hash) rOrPrBis_cudarVr4 fast_hashrcrmrn)rorprqrXrYrrrnuse_cudarZrghashtable_capacityquery_hash_code key_hash_coderts r)ruzYosoLSHCumulation.forwards' ??1  q!1!1 1 1UVV V ??1  A . .RSS S ??1  ! , ,PQQ Q ??1  A . .WXX X 88A;;%**Q-- ' 'KLL L ::a==CHHQKK ' 'KLL L2?XW\^ach@i2j2j/ HeS%%*%/  M!122 / " Z-;-E-EE8S(M8UV.. *O]].5UC=-Y-Y *O])88 =%I[]egh   j(O]TY[^`efff r+c t|}|j\}}}}}}}|j} |j} | d} t d| z} | drut |||||| | d} t ||||||| dz |z| | d }t ||||||| dz |z| | d }ndtjtj || ddtj z z | z}||dddddfz|dddddfz}tj || dd|z}tj || dz |z}tj | dd| dz |z}tj | dd|} dd||| dfS)NrgrD lsh_backwardrrErh) rBrwrnrrVr4lsh_weighted_cumulationrQrirTrjrkrl)rorxrprqrrrXrYrrrnrrgrr|rzr{rsrys r)r}zYosoLSHCumulation.backwards T""RURcO Ho}eS%</  M!122 . !" K'66-_dL^`hjkJ(??"c)"  J&=="e+"  HHuz%,ucmmBPR>S>S*T*TUUX\X___dqqK% 111aaa:(>>!!!TSTSTST*AUUK <eoob".E.EFFTLl]Q5F#4MNNJ|L$:$:2r$B$B]UVEVZ_D_``Hk&;&;B&C&CTJJJT:xTAAr+Nr~r%r+r)rrsN# # \# J.B.B\.B.B.Br+rc*eZdZdZfdZddZxZS)YosoEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c ttj|j|j|j|_tj|jdz|j|_ tj|j |j|_ tj |j|j |_ tj|j|_|dt%j|jddzdt+|dd |_|d t%j|jt$j|jj ddS) N) padding_idxrDeps position_ids)rrEF) persistentposition_embedding_typeabsolutetoken_type_idsdtyperL)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrQrSexpandgetattrrzerosrrOlongrLselfrn __class__s r)rzYosoEmbeddings.__init__sb !|F,=v?Q_e_rsss#%<0NQR0RTZTf#g#g %'\&2H&J\%]%]"f&8f>STTTz&"<==   EL)GHHOOPWXX[\\in    (/v7PR\']']$   K)..00 4K\Kc d d d      r+Nc||}n|dd}|d}||jddd|f}|mt|dr2|jddd|f}||d|}|}n+t j|t j|jj}|| |}| |} || z} |j dkr| |} | | z } | | } || } | S)NrErrrrr)rOrhasattrrrrQrrrLrrrrrr) r input_idsrr inputs_embeds input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr embeddingsrs r)ruzYosoEmbeddings.forwardsb  #..**KK',,..ss3K ^  ,QQQ  ^ J J"%::  ': 5 5"&":":<"H"H  - -J^^J// \\*-- r+)NNNNrrr__doc__rru __classcell__rs@r)rrsRQQ     ,        r+rc.eZdZdfd ZdZddZxZS)YosoSelfAttentionNct|j|jzdkr0t |ds t d|jd|jdt du}trTtrF|sD tn4#t$r'}t d|Yd}~nd}~wwxYw|j|_t|j|jz |_|j|jz|_t!j|j|j|_t!j|j|j|_t!j|j|j|_t!j|j|_||n|j|_|j|_|j|_|jdu|_|j|_|j|_|j|_|j|j|j|jd|_ |jAt!j!|j|j|jdf|jd zdfd |j |_"dSdS) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zGCould not load the custom kernel for multi-scale deformable attention: )rgrrZrrrDF) in_channels out_channels kernel_sizepaddingbiasgroups)#rrrnum_attention_headsrrPr4rrr6 ExceptionloggerwarningrVattention_head_size all_head_sizerLinearrXrYrrrattention_probs_dropout_probrruse_expectationrg conv_windowuse_convrrZr lsh_configConv2dconv)rrnr kernel_loadeders r)rzYosoSelfAttention.__init__)s    : :a ? ?PVXhHiHi ?8F$688 4888 'd2 " $ $ n);)=)= nm n n!#### n n nlijllmmmmmmmm n$*#= #&v'9F'J # #PVPn $ &5#1*$6 #1 "/"/!/  -      ) "6#7#/3+q0!41 DIII * )s B C &CC c|dd|j|jfz}|j|}|ddddS)NrErrDrr )rOrrviewpermute)rlayernew_layer_shapes r)transpose_for_scoresz&YosoSelfAttention.transpose_for_scores\sN**,,ss+t/GIa.bb O,}}Q1a(((r+Fc|||}|||}|||}||}|jr&|||ddddddfz}|\} } } } || | z| | }|| | z| | }|| | z| | }d|dz z}|d | d| | z|  }d} |j s| | kr| | z| | | z f}tj |tj||jgd}tj |tj||jgd}tj |tj||jgd}|j s|jrt#||g\}}|j r%t$||||||j}n$t*||||||j}|j s| | kr|ddddd| f}t#|}|| | | | }|jr||z }|dd dd }|dd |jfz}|j|}|r||fn|f}|S) N?g@rrM rKrErrDr rh)rXrrYrrrrrOrU unsqueezerepeat_interleaverVrrQcatrrLtrainingrIreapplyrrrr<rr)r hidden_statesattention_maskoutput_attentionsmixed_query_layer key_layer value_layer query_layerconv_value_layer batch_size num_headsseq_lenhead_dim gpu_warp_sizepad_size context_layernew_context_layer_shapeoutputss r)ruzYosoSelfAttention.forwardas JJ}55--dhh}.E.EFF // =0I0IJJ //0ABB = Y#yy~aaaqqqRVFV7W)WXX 3>3C3C3E3E0 Iw!))*y*@'8TT %%j9&S"S* *,CD 4E[=-00MK[r+r8NF)rrrrrrurrs@r)rr(si111111f))) QQQQQQQQr+rcPeZdZfdZdejdejdejfdZxZS)YosoSelfOutputcttj|j|j|_tj|j|j|_tj|j |_ dSNr) rrrrrdenserrrrrrs r)rzYosoSelfOutput.__init__sf Yv163EFF f&8f>STTTz&"<== r+r input_tensorreturnc||}||}|||z}|Sr8rrrrrrs r)ruzYosoSelfOutput.forward@ =11  ]33 }|'CDD r+rrrrrQTensorrurrs@r)rri>>>>> U\RWR^r+rc.eZdZdfd ZdZddZxZS) YosoAttentionNctt|||_t ||_t |_dS)N)r)rrrrroutputset pruned_heads)rrnrrs r)rzYosoAttention.__init__sO %fF]^^^ $V,, EEr+ct|dkrdSt||jj|jj|j\}}t |jj||j_t |jj||j_t |jj ||j_ t |j j |d|j _ |jjt|z |j_|jj|jjz|j_ |j ||_dS)NrrrM)rNrrrrrrrXrYrrrrrunion)rheadsindexs r) prune_headszYosoAttention.prune_headss u::?? F7 490$)2OQUQb  u -TY_eDD *49=%@@ ,TY_eDD .t{/@%QOOO )- (EE (R %"&)"?$)B_"_  -33E::r+Fc||||}||d|}|f|ddz}|S)Nrr)rr)rrrr self_outputsattention_outputrs r)ruzYosoAttention.forwardsKyy@QRR ;;|A FF#% QRR(88r+r8r)rrrrrrurrs@r)r r s`"""""" ;;;$r+r cBeZdZfdZdejdejfdZxZS)YosoIntermediatecttj|j|j|_t|jtrt|j|_ dS|j|_ dSr8) rrrrrintermediate_sizerr9 hidden_actstrr intermediate_act_fnrs r)rzYosoIntermediate.__init__sn Yv163KLL f' - - 9'-f.?'@D $ $ $'-'8D $ $ $r+rrcZ||}||}|Sr8)rr rrs r)ruzYosoIntermediate.forwards, =11 00?? r+r rs@r)rrs^99999U\elr+rcPeZdZfdZdejdejdejfdZxZS) YosoOutputcttj|j|j|_tj|j|j|_tj |j |_ dSr) rrrrrrrrrrrrrs r)rzYosoOutput.__init__sf Yv79KLL f&8f>STTTz&"<== r+rrrc||}||}|||z}|Sr8rrs r)ruzYosoOutput.forwardrr+r rs@r)r$r$r r+r$c,eZdZfdZddZdZxZS) YosoLayerct|j|_d|_t ||_|j|_t||_t||_ dSNr) rrchunk_size_feed_forward seq_len_dimr  attentionadd_cross_attentionr intermediater$rrs r)rzYosoLayer.__init__si '-'E$&v..#)#= ,V44 (( r+NFc||||}|d}|dd}t|j|j|j|}|f|z}|S)N)rrr)r-rfeed_forward_chunkr+r,)rrrrself_attention_outputsrr layer_outputs r)ruzYosoLayer.forward si!% ~ar!s!s1!4(,0  #T%A4CSUe   /G+r+c\||}|||}|Sr8)r/r)rrintermediate_outputr3s r)r1zYosoLayer.feed_forward_chunks2"//0@AA{{#68HII r+r)rrrrrur1rrs@r)r(r(s[)))))    r+r(c0eZdZfdZ ddZxZS) YosoEncoderct|_tjfdt jD|_d|_dS)Nc.g|]}tSr%)r()r&_rns r)r*z(YosoEncoder.__init__..$s!#_#_#_!If$5$5#_#_#_r+F) rrrnr ModuleListrangenum_hidden_layersrgradient_checkpointingrs `r)rzYosoEncoder.__init__!s`  ]#_#_#_#_uVE]?^?^#_#_#_`` &+###r+NFTcl|rdnd}|rdnd}t|jD]\\} } |r||fz}|jr%|jr|| j|||} n | |||} | d}|r || dfz}]|r||fz}|st d|||fDSt|||S)Nr%rrc3K|]}||V dSr8r%)r&vs r) z&YosoEncoder.forward..Is(mmq_`_l_l_l_l_lmmr+)last_hidden_stater attentions) enumeraterr>r_gradient_checkpointing_func__call__tupler ) rrr head_maskroutput_hidden_states return_dictall_hidden_statesall_self_attentionsi layer_module layer_outputss r)ruzYosoEncoder.forward's7#7@BBD$5?bb4(44 P POA|# I$58H$H!* _t} _ $ A A )!"% !! !- ]NL] ^ ^ )!,M  P&9]1=MSTTTr+rrc||}||}||}|Sr8)rrTrr"s r)ruz#YosoPredictionHeadTransform.forward\s= =11 --m<< }55 r+r rs@r)rRrRRscUUUUUU\elr+rRc*eZdZfdZdZdZxZS)YosoLMPredictionHeadc>tt||_t j|j|jd|_t j tj |j|_ |j |j_ dS)NF)r) rrrR transformrrrrdecoder ParameterrQrrrs r)rzYosoLMPredictionHead.__init__esz 4V<<y!3V5FUSSS LV->!?!?@@ !I r+c(|j|j_dSr8)rrZrs r) _tie_weightsz!YosoLMPredictionHead._tie_weightsrs I r+cZ||}||}|Sr8)rYrZr"s r)ruzYosoLMPredictionHead.forwardus*}55  ]33 r+)rrrrr^rurrs@r)rWrWdsV & & & & &&&&r+rWcBeZdZfdZdejdejfdZxZS)YosoOnlyMLMHeadcptt||_dSr8)rrrW predictionsrs r)rzYosoOnlyMLMHead.__init__}s/ /77r+sequence_outputrc0||}|Sr8)rc)rrdprediction_scoress r)ruzYosoOnlyMLMHead.forwards ,,_==  r+r rs@r)rara|s^88888!u|! !!!!!!!!r+rac$eZdZdZeZdZdZdZdS)YosoPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. r#Tct|tjrT|jjd|jj|j |jj dSdSt|tj r_|jjd|jj|j +|jj|j  dSdSt|tj r?|jj |jj ddSdS)zInitialize the weightsg)meanstdNr)r9rrweightdatanormal_rninitializer_rangerzero_rrrfill_)rmodules r) _init_weightsz!YosoPreTrainedModel._init_weightss) fbi ( ( * M  & &CT[5R & S S S{&  &&((((('&  - - * M  & &CT[5R & S S S!- "6#56<<>>>>>.-  - - * K  " " $ $ $ M  $ $S ) ) ) ) ) * *r+N) rrrrr config_classbase_model_prefixsupports_gradient_checkpointingrsr%r+r)rhrhs@ L&*#*****r+rhaG This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`YosoConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a5 Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z^The bare YOSO Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZdZdZdZee de e e e  ddeejd eejd eejd eejd eejd eejdeedeedeedeee ffdZxZS) YosoModelct|||_t||_t ||_|dSr8)rrrnrrr7encoder post_initrs r)rzYosoModel.__init__sX     (00"6**  r+c|jjSr8rrr]s r)get_input_embeddingszYosoModel.get_input_embeddingss ..r+c||j_dSr8r})rrrs r)set_input_embeddingszYosoModel.set_input_embeddingss*/'''r+c|D]/\}}|jj|j|0dS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrzrr-r)rheads_to_prunerrs r) _prune_headszYosoModel._prune_headssU +0022 C CLE5 L u % / ; ;E B B B B C Cr+batch_size, sequence_length checkpoint output_typertNrrrrrIrrrJrKrc ||n |jj}||n |jj}| | n |jj} ||t d|+||||} n.||dd} nt d| \} } ||jn|j} |tj | | f| }|gt|j dr1|j j ddd| f}| | | }|}n!tj| tj| }|||jj}| ||||}||||||| }|d }| s|f|d dzSt'||j|j|j S) NzDYou cannot specify both input_ids and inputs_embeds at the same timerEz5You have to specify either input_ids or inputs_embedsrKrr)rrrr)rrIrrJrKrr)rCrrDcross_attentions)rnrrJuse_return_dictrP%warn_if_padding_and_no_attention_maskrOrLrQonesrrrrrr get_head_maskr=rzr rrDr)rrrrrrIrrrJrKrrrrLrrembedding_outputencoder_outputsrds r)ruzYosoModel.forwards4$2C1N--TXT_Tq$8$D $+Jj &1%cdd d  "  6 6y. Q Q Q#..**KK  &',,..ss3KKTUU U!, J%.%:!! @T  !"Z*j)A6RRRN  !t(899 [*./*HKZK*X'3J3Q3QR\^h3i3i0!A!&[ SY!Z!Z!Z&&y$+2OPP ??%)' +   ,, )/!5# '  *!, <#%(;; ;1-)7&1,=     r+) NNNNNNNNN)rrrrr~rrrYOSO_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrQr boolrrrurrs@r)rxrxs ///000CCC+*+@+G+GHe+f+fgg&6$-11515/3,004,0/3&*I I EL)I !.I !. I u|, I EL) I   -I $D>I 'tnI d^I  u88 9I I I hg I I I I I r+rxz2YOSO Model with a `language modeling` head on top.ceZdZddgZfdZdZdZee de e e e  dd eejd eejd eejd eejd eejdeejdeejdeedeedeedeee ffdZxZS)YosoForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasct|t||_t ||_|dSr8)rrrxr#raclsr{rs r)rzYosoForMaskedLM.__init__QsQ    f%% "6** r+c$|jjjSr8)rrcrZr]s r)get_output_embeddingsz%YosoForMaskedLM.get_output_embeddingsZsx#++r+cT||jj_|j|jj_dSr8)rrcrZr)rnew_embeddingss r)set_output_embeddingsz%YosoForMaskedLM.set_output_embeddings]s%'5$$2$7!!!r+rrNrrrrrIrlabelsrrJrKrc | | n |jj} ||||||||| |  } | d} || } d}|Kt }|| d|jj|d}| s| f| ddz}||f|zn|St|| | j| j S)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. NrrrrIrrrJrKrrErlosslogitsrrD) rnrr#rr rrrrrD)rrrrrrIrrrrJrKrrdrfmasked_lm_lossloss_fctrs r)ruzYosoForMaskedLM.forwardas2&1%2 'tn2 d^2  un$ %2 2 2 hg 2 2 2 2 2 r+rc(eZdZdZfdZdZxZS)YosoClassificationHeadz-Head for sentence-level classification tasks.c"ttj|j|j|_tj|j|_tj|j|j |_ ||_ dSr8) rrrrrrrrr num_labelsout_projrnrs r)rzYosoClassificationHead.__init__sj Yv163EFF z&"<==  &"4f6GHH  r+c |dddddf}||}||}t|jj|}||}||}|S)Nr)rrr rnrr)rfeatureskwargsxs r)ruzYosoClassificationHead.forwardst QQQ111W  LLOO JJqMM 4;) *1 - - LLOO MM!  r+rrs@r)rrsM77r+rzYOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks.ceZdZfdZeedeee e  dde e j de e j de e j de e j d e e j d e e j d e e j d e ed e ede edeee ffdZxZS)YosoForSequenceClassificationct||j|_t||_t ||_|dSr8)rrrrxr#r classifierr{rs r)rz&YosoForSequenceClassification.__init__s[     +f%% 088 r+rrNrrrrrIrrrrJrKrc | | n |jj} ||||||||| |  } | d} || } d}|Z|jjf|jdkr d|j_nN|jdkr7|jtjks|jtj kr d|j_n d|j_|jjdkrWt}|jdkr1|| | }n|| |}n|jjdkrGt}|| d|j| d}n*|jjdkrt}|| |}| s| f| ddz}||f|zn|St|| | j| j S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrrr regressionsingle_label_classificationmulti_label_classificationrEr)rnrr#r problem_typerrrQrrVr squeezer rrrrrD)rrrrrrIrrrrJrKrrdrrrrs r)ruz%YosoForSequenceClassification.forwards 2&1%C 'tnC d^C  u.. /C C C hg C C C C C r+rzYOSO Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.ceZdZfdZeedeee e  dde e j de e j de e j de e j d e e j d e e j d e e j d e ed e ede edeee ffdZxZS)YosoForMultipleChoicec t|t||_t j|j|j|_t j|jd|_| dSr*) rrrxr#rrrpre_classifierrr{rs r)rzYosoForMultipleChoice.__init__sr    f%%  i(:FG>SINN2y~~b'9'9:::Y] M[Mg,,R1D1DR1H1HIIImqM[Mg,,R1D1DR1H1HIIImqGSG_|((\->->r-B-BCCCei (   r=#5#5b#9#9=;M;Mb;Q;Q R R R  )) ))%'/!5#   qz $QQQT* ++M:: ! -00 // ++b+66  '))H8OV44D F%''!""+5F)-)9TGf$$v E("!/)     r+r)rrrrrrrrrrrrrQr rrrrurrs@r)rr s +*+@+G+GHr+s+stt&-$-11515/3,004)-,0/3&*B B EL)B !.B !. B u|, B EL) B   -B &B $D>B 'tnB d^B  u// 0B B B ut B B B B B r+rzYOSO Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.ceZdZfdZeedeee e  dde e j de e j de e j de e j d e e j d e e j d e e j d e ed e ede edeee ffdZxZS)YosoForTokenClassificationc6t||j|_t||_t j|j|_t j |j |j|_ | dSr8) rrrrxr#rrrrrrrr{rs r)rz#YosoForTokenClassification.__init__msy     +f%% z&"<== )F$68IJJ r+rrNrrrrrIrrrrJrKrc | | n |jj} ||||||||| |  } | d} || } || } d}|t }||ddk}| d|j}tj ||dtj |j  |}|||}n8|| d|j|d}| s| f| ddz}||f|zn|St|| | j| jS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. NrrrErr)rnrr#rrr rrrQwhererA ignore_indextype_asrrrD)rrrrrrIrrrrJrKrrdrrr active_loss active_logits active_labelsrs r)ruz"YosoForTokenClassification.forwardxs.&1%; 'tn; d^;  u++ ,; ; ; hg ; ; ; ; ; r+rzYOSO Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).ceZdZfdZeedeee e  dde e j de e j de e j de e j d e e j d e e j d e e j d e e j d e ede ede edeee ffdZxZS)YosoForQuestionAnsweringct|d|_|j|_t||_t j|j|j|_| dS)NrD) rrrrxr#rrr qa_outputsr{rs r)rz!YosoForQuestionAnswering.__init__sm     +f%% )F$68IJJ r+rrNrrrrrIrstart_positions end_positionsrrJrKrc h| | n |jj} |||||||| | |  } | d} || }|dd\}}|d}|d}d}||t |dkr|d}t |dkr|d}|d}|d|}|d|}t|}|||}|||}||zdz }| s||f| ddz}||f|zn|St|||| j | j S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. NrrrrErM)rrD)r start_logits end_logitsrrD) rnrr#rsplitrrNrOclampr rrrD)rrrrrrIrrrrrJrKrrdrrr total_loss ignored_indexr start_lossend_lossrs r)ruz YosoForQuestionAnswering.forwards<&1%H 'tnH d^H  u22 3H H H hg H H H H H r+r)KrrkpathlibrtypingrrrrQtorch.utils.checkpointrtorch.nnrr r activationsr modeling_outputsr rrrrrmodeling_utilsr pytorch_utilsrrrutilsrrrrrrconfiguration_yosor get_loggerrrrrr4r6rBrIrcautogradFunctionrerModulerrrr rr$r(r7rRrWrarhYOSO_START_DOCSTRINGrrxrrrrrrr%r+r)rs )))))))))) AAAAAAAAAA!!!!!!.-----llllllllll+*****  H % %, 1 1 1   CCC...&BBBBBU^,BBB>VBVBVBVBVB/VBVBVBt99999RY999xJJJJJ JJJ\     RY   BIB     ry            :. . . . . "). . . d")$290!!!!!bi!!!*****/***6 /ddh h h h h #h h h VNPdeeK K K K K )K K feK \RY*/ S S S S S $7S S  S lH S S S S S /S S  S lP M M M M M !4M M  M `h [ [ [ [ [ 2[ [  [ [ [ r+