gn dZddlZddlZddlmZmZmZmZddlZddl m cm Z ddl Zddlm Z ddlmZmZmZddlmZddlmZmZdd lmZdd lmZmZdd lmZmZmZm Z d d l!m"Z"e j#e$Z%dZ&dZ'gdZ(dZ)dZ*d3dej+de,de-dej+fdZ.Gdde j/Z0Gdde j/Z1Gdde j/Z2Gdd e j/Z3Gd!d"e j/Z4Gd#d$e j/Z5Gd%d&e j/Z6Gd'd(e j/Z7Gd)d*eZ8d+Z9d,Z:ed-e9Gd.d/e8Z;ed0e9Gd1d2e8Z??? r, embeddingsheightwidthrc||z}tjs$||jj|jjzkr|jS|d||ddddd}tj |||fd}|dd||zddd}|S)Nrrr bilinear)sizemode) r#jit is_tracingrGrHrZreshapepermuteF interpolate)r3rdrerfrWinterpolated_embeddingss r*interpolate_pos_encodingz+PvtPatchEmbeddings.interpolate_pos_encoding|sun y##%% ,+9ORVR]Rh9h*h*h+ +''65"==EEaAqQQ "#- &%Wa"b"b"b"9"A"A!RRW"X"X"`"`abdegh"i"i&&r, pixel_valuesc|j\}}}}||jkrtd||}|j^}}}|ddd}||}|j|j|dd} tj | |fd}| |j ddddf||} tj |j ddddf| fd} n| |j ||} | || z}|||fS)NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rirrhdim)r!rK ValueErrorr]flatten transposer`rMexpandr#catrtrZrc) r3ru batch_sizerKrerf patch_embed_rdrMrZs r*r8zPvtPatchEmbeddings.forwardso2>2D/ L&% 4, , ,w ool33 '-FE!))!,,66q!<< __[11 > %--j"bAAIIz#:BBBJ"&"?"?@XYZYZYZ\]\^\^Y^@_agin"o"o "')T-Eaaa!e-LNa,bhi"j"j"j  "&"?"?@XZ`bg"h"h \\*/B"BCC 65((r,F)r<r=r>r?rrintrboolr2r#rArtrr8rCrDs@r*rFrFYs( @@@#x},-@#x},- @  @  @@@@@@@@8 '5< ' 'UX ']b]i ' ' ' ')EL)U5<c;Q5R))))))))r,rFcLeZdZdedeffd ZdejdejfdZxZ S) PvtSelfOutputrGrLcttj|||_tj|j|_dSr0)r1r2rLineardenserarbrc)r3rGrLr4s r*r2zPvtSelfOutput.__init__sD Y{K88 z&"<== r,r5rcZ||}||}|Sr0)rrcr7s r*r8zPvtSelfOutput.forwards* =11  ]33 r,) r<r=r>rrr2r#rAr8rCrDs@r*rrsq>y>s>>>>>> U\elr,rc eZdZdZdedededeffd Zdedej fd Z ddej d ed ed e de ej f dZ xZS)PvtEfficientSelfAttentionzpEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://arxiv.org/abs/2102.12122).rGrLnum_attention_headssequences_reduction_ratioc t||_||_|j|jzdkr t d|jd|jdt |j|jz |_|j|jz|_tj |j|j|j |_ tj |j|j|j |_ tj |j|j|j |_ tj|j|_||_|dkr?tj|||||_tj||j|_dSdS) NrzThe hidden size (z6) is not a multiple of the number of attention heads ())biasrrOrQ)r1r2rLrryrattention_head_size all_head_sizerrqkv_biasquerykeyvalueraattention_probs_dropout_probrcrr\sequence_reductionr^r_r`r3rGrLrrr4s r*r2z"PvtEfficientSelfAttention.__init__s~ &#6  d6 6! ; ;6D$4662666  $'t'7$:R'R#S#S !58PPYt/1C&/ZZZ 9T-t/AXXXYt/1C&/ZZZ z&"EFF )B& $q ( (&(iXq'''D #!l;FS"S%**+BCC 6G]=/22mM]r,r)r<r=r>r?rrr@r2r#rArrrr8rCrDs@r*rrszzSS.1SHKShmSSSSSS:1#1%,1111#( **|** *  * u|  ********r,rc veZdZdedededeffd ZdZ ddej d ed ed e d e ej f d Z xZ S) PvtAttentionrGrLrrctt|||||_t |||_t |_dS)N)rLrr)rL)r1r2rr3rr)set pruned_headsrs r*r2zPvtAttention.__init__se - # 3&?     $F DDD EEr,ct|dkrdSt||jj|jj|j\}}t |jj||j_t |jj||j_t |jj ||j_ t |j j |d|j _ |jjt|z |j_|jj|jjz|j_ |j ||_dS)Nrrrw)lenrr3rrrrrrrr)rrunion)r3headsindexs r* prune_headszPvtAttention.prune_heads s u::?? F7 490$)2OQUQb  u -TY_eDD *49=%@@ ,TY_eDD .t{/@%QOOO )- (EE (R %"&)"?$)B_"_  -33E::r,Fr5rerfrrc|||||}||d}|f|ddz}|S)Nrr)r3r))r3r5rerfr self_outputsattention_outputrs r*r8zPvtAttention.forwardsMyy?PQQ ;;|A77#% QRR(88r,r)r<r=r>rrr@r2rr#rArrr8rCrDs@r*rrs " ".1 "HK "hm " " " " " ";;;&_d"\36?BW[ u| r,rc reZdZ d dededeedeeffd Zdejdejfd Z xZ S) PvtFFNNrG in_featureshidden_features out_featuresctt||n|}tj|||_t |jtrt|j|_ n |j|_ tj|||_ tj |j |_ dSr0)r1r2rrdense1rT hidden_actrBr intermediate_act_fndense2rarbrc)r3rGrrrr4s r*r2zPvtFFN.__init__&s '3'?||[ i _== f' - - 9'-f.?'@D $ $'-'8D $i>> z&"<== r,r5rc||}||}||}||}||}|Sr0)rrrcrr7s r*r8zPvtFFN.forward7s_ M22 00??  ]33  M22  ]33 r,)NN) r<r=r>rrrr2r#rAr8rCrDs@r*rr%s *.&* >>>>"# > sm >>>>>>"U\elr,rc \eZdZdedededededef fd Zdd ejd ed ed e fd Z xZ S)PvtLayerrGrLrr+r mlp_ratiocttj||j|_t |||||_|dkrt|ntj |_ tj||j|_ t||z}t||||_dS)NrQ)rGrLrrr)rGrr)r1r2rr^r_ layer_norm_1r attentionr.Identityr+ layer_norm_2rrmlp) r3rGrLrr+rrmlp_hidden_sizer4s r*r2zPvtLayer.__init__As L&:OPPP%# 3&?     4=s??Y/// L&:OPPPkI566[Rabbbr,Fr5rerfrcF||||||}|d}|dd}||}||z}|||}||}||z} | f|z}|S)N)r5rerfrrr)rrr+rr) r3r5rerfrself_attention_outputsrr mlp_output layer_outputs r*r8zPvtLayer.forwardWs!%++M::/ "0" "  2!4(,>>*:;;(=8 XXd// >>?? ^^J// $z1 /G+r,r) r<r=r>rrr@r2r#rArr8rCrDs@r*rr@sccc! c  c $) ccccccc,U\3s_cr,rc eZdZdeffd Z d dejdeedeedeed e e e ff d Z xZ S) PvtEncoderrGct||_tjd|jt |j}g}t|j D]}| t||dkr|j n|jj d|dzzz|j||j||dkr|jn|j|dz |j|||j dz kt%j||_g}d}t|j D]}g}|dkr||j|dz z }t|j|D]_}| t+||j||j||||z|j||j|`| t%j|t%j||_t%j|jd|j|_dS)Nrrir)rGrHrIrJrKrLrM)rGrLrr+rrrhrQ)r1r2rGr#linspacedrop_path_ratesumdepthstolistrangenum_encoder_blocksappendrFrH patch_sizesstridesrK hidden_sizesr ModuleListpatch_embeddingsrrsequence_reduction_ratios mlp_ratiosblockr^r_r`) r3rGdrop_path_decaysrdiblockscurlayersjr4s r*r2zPvtEncoder.__init__os[  !>!V-BC DVDVWW^^`` v011  A   "!45FFv00 @V[\abefaf[g@h%1!4!>!,89Q!4!4FDWXY\]X]D^ & 3A 66#FDYZZZr,FTruroutput_hidden_states return_dictrcl|rdnd}|rdnd}|jd}t|j}|} tt |j|jD]\} \} } | | \} } }| D].}|| | ||}|d} |r ||dfz}|r|| fz}/| |dz kr@| || |ddddd} | | } |r|| fz}|std| ||fDSt| ||S) Nrrrhr ric3K|]}||V dSr0r).0vs r* z%PvtEncoder.forward..s(mmq_`_l_l_l_l_lmmr,last_hidden_stater5 attentions) r!rr enumerateziprrorprr`tupler )r3rurrrall_hidden_statesall_self_attentionsr~ num_blocksr5idxembedding_layer block_layerrerfr layer_outputss r*r8zPvtEncoder.forwards#7@BBD$5?bb4!'* __ $ 3rr2r# FloatTensorrrrrr r8rCrDs@r*rrns0[y0[0[0[0[0[0[j-2/4&* # # '# $D># 'tn # d^ # uo% & # # # # # # # # r,rcbeZdZdZeZdZdZgZde e j e j e j fddfdZdS)PvtPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. pvtrumodulerNct|tjritj|jjd|jj|j_|j |j j dSdSt|tj r?|j j |jj ddSt|trtj|jjd|jj|j_|jDtj|jjd|jj|j_dSdSdS)zInitialize the weightsr)meanstdNg?)rTrrinit trunc_normal_weightdatarGinitializer_rangerzero_r^fill_rFrZrM)r3r s r* _init_weightsz PvtPreTrainedModel._init_weightss` fbi ( ( "$!6!6v}7IPSY]YdYv!6!w!wFM {&  &&((((('&  - -  K  " " $ $ $ M  $ $S ) ) ) ) )  2 3 3 .0g.C.C*/K1/D//F & + +(*(=(=$) 5)>)) %%%   ,+r,)r<r=r>r?r config_classbase_model_prefixmain_input_name_no_split_modulesrrrr\r^rrr,r*rrsk L$OE")RY *L$MRVr,raG This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`~PvtConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a  Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`PvtImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zSThe bare Pvt encoder outputting raw hidden-states without any specific head on top.ceZdZdeffd ZdZeede e e e de  ddejd eed eed eed eee ff d ZxZS)PvtModelrGct|||_t||_|dSr0)r1r2rGrencoder post_initr3rGr4s r*r2zPvtModel.__init__ sK     "&))  r,c|D]/\}}|jj|j|0dS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrlayerrr)r3heads_to_pruner!rs r* _prune_headszPvtModel._prune_headssU +0022 C CLE5 L u % / ; ;E B B B B C Cr,%(batch_size, channels, height, width)vision) checkpoint output_typermodalityexpected_outputNrurrrrc||n |jj}||n |jj}||n |jj}|||||}|d}|s|f|ddzSt ||j|jS)Nrurrrrrr)rGrruse_return_dictrr r5r)r3rurrrencoder_outputssequence_outputs r*r8zPvtModel.forwards2C1N--TXT_Tq$8$D $+Jj &1%rr2r#rPVT_INPUTS_DOCSTRINGr:r_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr#rrrrrr8rCrDs@r*rrs yCCC+*+?+F+FGn+o+opp&#$. -1/3&*   ' $D> 'tn  d^  uo% &    qp     r,rz Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. c$eZdZdeddffd Zeedee e e e  d de ejde ejd e ed e ed e edeee ff d ZxZS)PvtForImageClassificationrGrNcBt||j|_t||_|jdkr%t j|jd|jnt j|_ | dS)Nrrh) r1r2 num_labelsrrrrrr classifierrrs r*r2z"PvtForImageClassification.__init__Ls     +F##FLEVYZEZEZBIf)"-v/@ A A A`b`k`m`m  r,r$)r&r'rr)rulabelsrrrc||n |jj}|||||}|d}||dddddf}d} |Z|jjf|jdkr d|j_nN|jdkr7|jtjks|jtj kr d|j_n d|j_|jjdkrWt} |jdkr1| | | } n| ||} n|jjdkrGt} | | d|j| d} n*|jjdkrt} | ||} |s|f|ddz} | | f| zn| St| ||j|j S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr+rr regressionsingle_label_classificationmulti_label_classificationrh)losslogitsr5r)rGr,rr7 problem_typer6rr#longrr squeezer rrrr5r) r3rur8rrrrr.r>r=loss_fctr)s r*r8z!PvtForImageClassification.forwardZs*&1%rr2rr/r:r_IMAGE_CLASS_CHECKPOINTrr1_IMAGE_CLASS_EXPECTED_OUTPUTrr#rArrrr8rCrDs@r*r4r4Ds% y T      +*+?+F+FGn+o+opp*)$4 *.,0/3&* ; ; u|,; &; $D> ; 'tn ; d^ ;  u++ ,; ; ; qp; ; ; ; ; r,r4)rF)=r?rUrtypingrrrrr#torch.nn.functionalrrrqtorch.utils.checkpointtorch.nnrr r activationsr modeling_outputsr rmodeling_utilsr pytorch_utilsrrutilsrrrrconfiguration_pvtr get_loggerr<loggerr1r0r2rCrDrAr@rr+Moduler.rFrrrrrrrPVT_START_DOCSTRINGr/rr4rr,r*rSs%" 333333333333 AAAAAAAAAA!!!!!!FFFFFFFF------QQQQQQQQ )(((((  H % %-%11U\eTV[Vb* - - - - -") - - -A)A)A)A)A)A)A)A)H     BI   OOOOO OOOd'''''29'''TRY6+++++ry+++\V V V V V V V V r!!!!!!!!H   Y7 7 7 7 7 !7 7 7 t Q Q Q Q Q 2Q Q Q Q Q r,