gYdZddlZddlZddlmZmZmZmZm Z m Z ddl Z ddl Z ddl m Z ddlmZmZmZddlmZddlmZmZmZmZdd lmZdd lmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$d d l%m&Z&e"j'e(Z)dZ*dZ+gdZ,dZ-dZ.Gdde j/Z0Gdde j/Z1Gdde j/Z2Gdde2Z3Gdde j/Z4Gdde j/Z5Gdd e5Z6Gd!d"e j/Z7Gd#d$e j/Z8e5e6d%Z9Gd&d'e j/Z:Gd(d)e j/Z;Gd*d+eZe d.e=Gd/d0e<Z?Gd1d2e j/Z@e d3e=Gd4d5e<ZAe d6e=Gd7d8e<ZBdS)9zPyTorch ViT model.N)DictListOptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedImageModelingOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings torch_int) ViTConfigrz!google/vit-base-patch16-224-in21k)rizgoogle/vit-base-patch16-224z Egyptian catc eZdZdZddededdffd Zdejd e d e dejfd Z dd ejd e ej dedejfdZ xZS) ViTEmbeddingszb Construct the CLS token, position and patch embeddings. Optionally, also the mask token. Fconfiguse_mask_tokenreturnNc$ttjt jdd|j|_|r-tjt jdd|jnd|_ t||_ |j j }tjt jd|dz|j|_ tj|j|_|j|_||_dS)Nr)super__init__r Parametertorchrandn hidden_size cls_tokenzeros mask_tokenViTPatchEmbeddingspatch_embeddings num_patchesposition_embeddingsDropouthidden_dropout_probdropout patch_sizer!)selfr!r"r0 __class__s `/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/vit/modeling_vit.pyr&zViTEmbeddings.__init__As ek!Q8J&K&KLLQ_i",u{1a9K'L'LMMMei 26 : :+7 #%< A{QPVPb0c0c#d#d z&"<==  + embeddingsheightwidthc|jddz }|jjddz }tjs||kr ||kr|jS|jddddf}|jddddf}|jd}||jz} ||jz} t |dz} |d| | |}|dddd}tj || | fdd }|dddd dd|}tj ||fd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rN?rr bicubicF)sizemode align_cornersdim)shaper1r(jit is_tracingr5rreshapepermuter functional interpolateviewcat) r6r:r;r<r0 num_positionsclass_pos_embedpatch_pos_embedrF new_height new_widthsqrt_num_positionss r8interpolate_pos_encodingz&ViTEmbeddings.interpolate_pos_encodingMsr!&q)A- 06q9A= y##%% ,+*F*F6UZ??+ +2111bqb592111abb59r"t. T_, &}c'9::)11!5GI[]`aa)11!Q1==-33 i( 4  *11!Q1==BB1b#NNy/?;CCCCr9 pixel_valuesbool_masked_posrVc|j\}}}}|||}|_|jd} |j|| d} |d| } |d| z z| | zz}|j|dd} tj| |fd}|r|| |||z}n ||j z}| |}|S)N)rVrr>?rE) rGr/r-expand unsqueezetype_asr+r(rOrVr1r4) r6rWrXrV batch_size num_channelsr;r<r: seq_length mask_tokensmask cls_tokenss r8forwardzViTEmbeddings.forwardus 3?2D/ L&%**> Y J7Q???  $ ?#d&C&CJPVX]&^&^^JJ#d&>>J\\*-- r9FNF)__name__ __module__ __qualname____doc__rboolr&r(TensorintrVr BoolTensorrd __classcell__r7s@r8r r <s  y $ 4      &D5<&D&DUX&D]b]i&D&D&D&DV7;). l"%"23#'   r9r cLeZdZdZfdZddejdedejfdZxZ S) r.z This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a Transformer. ct|j|j}}|j|j}}t |tjj r|n||f}t |tjj r|n||f}|d|dz|d|dzz}||_||_||_||_ tj |||||_ dS)Nrr) kernel_sizestride)r%r& image_sizer5r_r* isinstance collectionsabcIterabler0r Conv2d projection)r6r!rur5r_r*r0r7s r8r&zViTPatchEmbeddings.__init__s !'!2F4EJ $*$79Kk #-j+/:R#S#SqZZZdfpYq #-j+/:R#S#SqZZZdfpYq !!} 1 5*Q-:VW=:XY $$(&)L+:^hiiir9FrWrVr#c |j\}}}}||jkrtd|jd|d|sT||jdks||jdkr2td|d|d|jdd|jdd ||d dd }|S) NzoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z but got .rrzInput image size (*z) doesn't match model (z).r@)rGr_ ValueErrorrur{flatten transpose)r6rWrVr^r_r;r<r:s r8rdzViTPatchEmbeddings.forwards*2>2D/ L&% 4, , ,I!.II9EIII ( +++u8J/J/J EEE%EE+EE.2oa.@EEE__\22::1==GG1MM r9re) rgrhrirjr&r(rlrkrdrorps@r8r.r.s{ j j j j jELD]b]ir9r.c eZdZdeddffd ZdejdejfdZ d deejd e de e ejejfe ejffd Z xZ S) ViTSelfAttentionr!r#Nct|j|jzdkr1t |ds!t d|jfd|jd|j|_t |j|jz |_|j|jz|_tj |j|j|j |_ tj |j|j|j |_ tj |j|j|j |_tj|j|_dS)Nrembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r})bias)r%r&r*num_attention_headshasattrrrmattention_head_size all_head_sizer Linearqkv_biasquerykeyvaluer2attention_probs_dropout_probr4r6r!r7s r8r&zViTSelfAttention.__init__s1    : :a ? ?PVXhHiHi ?76#5"7773777  $*#= #&v'9Frr@rr )rBrrrNrK)r6r new_x_shapes r8transpose_for_scoresz%ViTSelfAttention.transpose_for_scoressPffhhssmt'?AY&ZZ FF;  yyAq!$$$r9F head_maskoutput_attentionsc||}|||}|||}||}t j||dd}|tj|j z }tj |d} | | } || |z} t j| |} | dddd} | dd|jfz} | | } |r| | fn| f} | S)Nr>rErr@rr )rrrrr(matmulrmathsqrtrr rLsoftmaxr4rK contiguousrBrrN) r6 hidden_statesrrmixed_query_layer key_layer value_layer query_layerattention_scoresattention_probs context_layernew_context_layer_shapeoutputss r8rdzViTSelfAttention.forwardsr!JJ}55--dhh}.E.EFF // =0I0IJJ //0ABB !< Y5H5HR5P5PQQ+di8P.Q.QQ-//0@b/II,,77  - 9O _kBB %--aAq99DDFF "/"4"4"6"6ss";t?Q>S"S%**+BCC 6G]=/22mM]r9rf)rgrhrirr&r(rlrrrkrrrdrorps@r8rrsGyGTGGGGGG$%el%u|%%%% bg!!(0(>!Z^! uU\5</0% 2EE F!!!!!!!!r9rc eZdZdeddffd Z d dejdeejde de e ejejfe ejfffd Z xZ S) ViTSdpaSelfAttentionr!r#Ncbt||j|_dSN)r%r&rrs r8r&zViTSdpaSelfAttention.__init__s,    ,2,O)))r9Frrrc |s|>tdt|||S||}|||}|||}||}tj j |||||j r|j nddd}|dddd }|dd |jfz} || }|dfS) Na`ViTSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rrrF) is_causalscalerr@rr r)logger warning_oncer%rdrrrrr(r rLscaled_dot_product_attentiontrainingrrKrrBrrN) r6rrrrrrrrrr7s r8rdzViTSdpaSelfAttention.forwardsp   5   w    77??+#"3#  !JJ}55--dhh}.E.EFF // =0I0IJJ //0ABB +HH    15 GD - -CI  &--aAq99DDFF "/"4"4"6"6ss";t?Q>S"S%**+BCC d""r9rf)rgrhrirr&r( FloatTensorrrlrkrrrdrorps@r8rrsPyPTPPPPPP-1"' '#'#('#EL)'# '# uU\5</0% 2EE F '#'#'#'#'#'#'#'#'#'#r9rc^eZdZdZdeddffd ZdejdejdejfdZxZ S) ViTSelfOutputz The residual connection is defined in ViTLayer instead of here (as is the case with other models), due to the layernorm applied before each block. r!r#Ncttj|j|j|_tj|j|_dSr) r%r&r rr*denser2r3r4rs r8r&zViTSelfOutput.__init__,sJ Yv163EFF z&"<== r9r input_tensorcZ||}||}|Srrr4r6rrs r8rdzViTSelfOutput.forward1s* =11  ]33 r9) rgrhrirjrr&r(rlrdrorps@r8rr&s >y>T>>>>>> U\RWR^r9rc eZdZdeddffd ZdeeddfdZ d dej d e ej d e de e ej ej fe ej ffd ZxZS) ViTAttentionr!r#Nctt||_t ||_t |_dSr)r%r&r attentionroutputset pruned_headsrs r8r&zViTAttention.__init__9sI )&11#F++ EEr9headsct|dkrdSt||jj|jj|j\}}t |jj||j_t |jj||j_t |jj ||j_ t |j j |d|j _ |jjt|z |j_|jj|jjz|j_ |j ||_dS)NrrrE)lenrrrrrrrrrrrrunion)r6rindexs r8 prune_headszViTAttention.prune_heads?s u::?? F7 4>5t~7Y[_[l  u 2$.2FNN/0BEJJ1$.2FNN.t{/@%QOOO .2^-ORUV[R\R\-\*'+~'IDNLn'n$ -33E::r9Frrrc||||}||d|}|f|ddz}|S)Nrr)rr)r6rrr self_outputsattention_outputrs r8rdzViTAttention.forwardQsM ~~mY@QRR ;;|A FF#% QRR(88r9rf)rgrhrirr&rrmrr(rlrrkrrrdrorps@r8rr8s"y"T"""""" ;S;d;;;;*-1"'  | EL)   uU\5</0% 2EE F        r9rc(eZdZdeddffd ZxZS)ViTSdpaAttentionr!r#Ncrt|t||_dSr)r%r&rrrs r8r&zViTSdpaAttention.__init__`s.    -f55r9)rgrhrirr&rorps@r8rr_sK6y6T6666666666r9rcLeZdZdeddffd ZdejdejfdZxZS)ViTIntermediater!r#Ncttj|j|j|_t|jtrt|j|_ dS|j|_ dSr) r%r&r rr*intermediate_sizerrv hidden_actstrrintermediate_act_fnrs r8r&zViTIntermediate.__init__fsn Yv163KLL f' - - 9'-f.?'@D $ $ $'-'8D $ $ $r9rcZ||}||}|Sr)rr)r6rs r8rdzViTIntermediate.forwardns, =11 00?? r9 rgrhrirr&r(rlrdrorps@r8rresq9y9T999999U\elr9rcZeZdZdeddffd ZdejdejdejfdZxZS) ViTOutputr!r#Ncttj|j|j|_tj|j|_ dSr) r%r&r rrr*rr2r3r4rs r8r&zViTOutput.__init__vsJ Yv79KLL z&"<== r9rrcd||}||}||z}|Srrrs r8rdzViTOutput.forward{s4 =11  ]33 % 4 r9rrps@r8rrus|>y>T>>>>>> U\RWR^r9r)eagersdpac eZdZdZdeddffd Z d dejdeejd e de e ejejfe ejffd Z xZ S) ViTLayerz?This corresponds to the Block class in the timm implementation.r!r#Nct|j|_d|_t |j||_t||_t||_ tj |j |j|_tj |j |j|_dS)Nreps)r%r&chunk_size_feed_forward seq_len_dimVIT_ATTENTION_CLASSES_attn_implementationrr intermediaterrr LayerNormr*layer_norm_epslayernorm_beforelayernorm_afterrs r8r&zViTLayer.__init__s '-'E$.v/JKFSS+F33'' " V-?VEZ [ [ [!|F,>FDYZZZr9Frrrc|||||}|d}|dd}||z}||}||}|||}|f|z}|S)N)rrr)rrrrr)r6rrrself_attention_outputsrr layer_outputs r8rdzViTLayer.forwards "&  ! !- 0 0 /"0" "  2!4(,)=8 ++M:: ((66 {{<?? /G+r9rf)rgrhrirjrr&r(rlrrkrrrdrorps@r8rrsII[y[T[[[[[[-1"' |EL)  uU\5</0% 2EE F r9rceZdZdeddffd Z d dejdeejd ed ed ede e e ff d Z xZ S) ViTEncoderr!r#Nct|_tjfdt jD|_d|_dS)Nc.g|]}tS)r).0_r!s r8 z'ViTEncoder.__init__..s!#^#^#^HV$4$4#^#^#^r9F) r%r&r!r ModuleListrangenum_hidden_layerslayergradient_checkpointingrs `r8r&zViTEncoder.__init__s`  ]#^#^#^#^eFD\>]>]#^#^#^__ &+###r9FTrrroutput_hidden_states return_dictc|rdnd}|rdnd}t|jD]h\}} |r||fz}|||nd} |jr%|jr|| j|| |} n | || |} | d}|r || dfz}i|r||fz}|st d|||fDSt|||S)Nrrrc3K|]}||V dSrr)rvs r8 z%ViTEncoder.forward..s(mmq_`_l_l_l_l_lmmr9)last_hidden_stater attentions) enumeraterrr_gradient_checkpointing_func__call__tupler) r6rrrrrall_hidden_statesall_self_attentionsi layer_modulelayer_head_mask layer_outputss r8rdzViTEncoder.forwardsI#7@BBD$5?bb4(44 P POA|# I$58H$H!.7.CillO* `t} ` $ A A )!#% !! !- ]OM^ _ _ )!,M  P&9]1=M