Ng2T dZddlZddlZddlZddlmZddlZddlm cm Z ddlm Z ddl m Z mZddlmZmZmZmZmZmZddlmZmZmZmZmZdd lmZdd lmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%d gZ&ej'e(Z)Gdde j*Z+Gdde j*Z,Gdde j*Z-de.fdZ/ede.fdZ0Gdde j*Z1Gdd e j*Z2d3de j*de3de4fdZ5d Z6d!Z7d4d#Z8d5d$Z9e$e9e9e9e9d%&e9d%&e9d%&d'Z:e#d4d(e2fd)Z;e#d4d(e2fd*Ze#d4d(e2fd-Z?e#d4d(e2fd.Z@e%e(d/d0d1d2dS)6a Nested Transformer (NesT) in PyTorch A PyTorch implement of Aggregating Nested Transformers as described in: 'Aggregating Nested Transformers' - https://arxiv.org/abs/2105.12723 The official Jax code is released and available at https://github.com/google-research/nested-transformer. The weights have been converted with convert/convert_nest_flax.py Acknowledgments: * The paper authors for sharing their research, code, and model weights * Ross Wightman's existing code off which I based this Copyright 2021 Alexander Soare N)partial)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPathcreate_classifier trunc_normal__assert) create_conv2d create_pool2d to_ntupleuse_fused_attn LayerNorm)build_model_with_cfg)register_notrace_function)checkpoint_seq named_apply)register_modelgenerate_default_cfgsregister_model_deprecationsNestcVeZdZUdZejjeed<dfd Z dZ xZ S) Attentionz This is much like `.vision_transformer.Attention` but uses *localised* self attention by accepting an input with an extra "image block" dim fused_attnFcpt||_||z}|dz|_t |_t j|d|z||_t j ||_ t j|||_ t j ||_ dS)Ng)bias) super__init__ num_headsscalerrrLinearqkvDropout attn_dropproj proj_drop)selfdimr&qkv_biasr+r-head_dim __class__s L/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/nest.pyr%zAttention.__init__/s ")#% (**9S!C%h777I..Ic3'' I..c |j\}}}}|||||d|j||jzdddddd}|d\}}} |jr,tj||| |j r |j j nd}nS||j z}|| d d z} | d } | | } | | z}|ddddd||||}||}||}|S) zm x is shape: B (batch_size), T (image blocks), N (seq length per image block), C (embed dim) r"rrr ) dropout_p)r/)shaper)reshaper&permuteunbindrFscaled_dot_product_attentiontrainingr+pr' transposesoftmaxr,r-) r.xBTNCr)qkvattns r3forwardzAttention.forward;sSW 1ahhqkk!!!Q1dna4>>QRRZZ[\^_abdeghjkll**Q--1a ? .q!QVZVcBk$.BRBRiklllAADJAq{{2r***D<>$''DqA IIaAq! $ $ , ,Q1a 8 8 IIaLL NN1  r4)rFr r ) __name__ __module__ __qualname____doc__torchjitFinalbool__annotations__r%rO __classcell__r2s@r3rr(so %%%% / / / / / /r4rcLeZdZdZdddddejejffd ZdZxZ S)TransformerLayerz This is much like `.vision_transformer.Block` but: - Called TransformerLayer here to allow for "block" as defined in the paper ("non-overlapping image blocks") - Uses modified Attention layer that handles the "block" dimension @Fr c ht| ||_t||||||_|dkrt |nt j|_| ||_ t||z} t|| |||_ dS)N)r&r0r+r-r ) in_featureshidden_features act_layerdrop) r$r%norm1rrNr rIdentity drop_pathnorm2intr mlp) r.r/r& mlp_ratior0r-r+rera norm_layermlp_hidden_dimr2s r3r%zTransformerLayer.__init__Zs Z__       1:B),,,BKMMZ__ S9_--*    r4c||}||||z}|||||z}|SN)rcrerNrhrf)r.rFys r3rOzTransformerLayer.forwardysb JJqMM tyy||,, , txx 1 6677 7r4) rPrQrRrSrGELUrr%rOrYrZs@r3r\r\Tsqg|      >r4r\c&eZdZdfd ZdZxZS)ConvPoolctt||d|d|_|||_t ddd||_dS)Nr"T) kernel_sizepaddingr#maxr7)rtstrideru)r$r%rconvnormrpool)r. in_channels out_channelsrjpad_typer2s r3r%zConvPool.__init__sb !+|T\cghhh J|,, !%Qq(SSS r4cdt|jddzdkdt|jddzdkd||}||dddddddd}||}|S)z: x is expected to have shape (B, C, H, W) r:r7rz1BlockAggregation requires even input spatial dimsr;r"r)r r<rxryr>rzr.rFs r3rOzConvPool.forwards  a1$&YZZZ a1$&YZZZ IIaLL IIaii1a++ , , 4 4Q1a @ @ IIaLLr4rr)rPrQrRr%rOrYrZs@r3rqrqsRTTTTTT       r4rq block_sizec |j\}}}}t||zdkdt||zdkd||z}||z}|||||||}|dd|||zd|}|S)zimage to blocks Args: x (Tensor): with shape (B, H, W, C) block_size (int): edge length of a single square block in units of H, W rz,`block_size` must divide input height evenlyz+`block_size` must divide input width evenlyr7r"r;)r<r r=rD)rFrrGHWrJ grid_height grid_widths r3blockifyrs 'JAq!Q A Na !OPPP A Na !NOOOz/KjJ ![*j*aHHA Aq!!![:%=r1EEA Hr4c|j\}}}}ttj|}||zx}}|||||||}|dd||||}|S)zblocks to image Args: x (Tensor): with shape (B, T, N, C) where T is number of blocks and N is sequence size per block block_size (int): edge length of a single square block in units of desired H, W r7r")r<rgmathsqrtr=rD) rFrrGrH_rJ grid_sizeheightwidths r3 deblockifyrs~JAq!QDIaLL!!I++FU !Y :z1EEA Aq!!!VUA66A Hr4c <eZdZdZdddddgdddf fd ZdZxZS) NestLevelz7 Single hierarchical level of a Nested Transformer Nr]Tr rrc  t||_d|_t jt jd|||_|t| ||_ nt j |_ t rt |ks Jdt j    f dt|D|_dS)NFr)rjr}zDMust provide as many drop path rates as there are transformer layersc L g|] }t | !S)) r/r&rir0r-r+rerjra)r\) .0irar+re embed_dimrirjr&r-r0s r3 z&NestLevel.__init__..sY 3# 3# 3# ##!###A,%#    3# 3# 3#r4)r$r%rgrad_checkpointingr ParameterrTzeros pos_embedrqrzrdlen Sequentialrangetransformer_encoder)r. num_blocksr seq_lengthr&depthrprev_embed_dimrir0r-r+rerjrar}r2s ` ` ``````` r3r%zNestLevel.__init__s$ $"'ek!ZY&W&WXX  % z\deeeDII DI y>> sy>>U***,r***#%= 3# 3# 3# 3# 3# 3# 3# 3# 3# 3# 3# 3#5\\ 3# 3# 3# $$   r4c||}|dddd}t||j}||jz}|jr4t jst|j |}n| |}t||j}|ddddS)z+ expects x as (B, C, H, W) rr7r"r) rzr>rrrrrTrU is_scriptingrrrrs r3rOzNestLevel.forwards IIaLL IIaAq ! ! Q ( (    " ,59+A+A+C+C ,t7;;AA((++A q$/ * *yyAq!$$$r4)rPrQrRrSr%rOrYrZs@r3rrst !,$,$,$,$,$,$\%%%%%%%r4rc^eZdZdZ dfd Zejjd dZejjdZ ejjd!dZ ejjd"dZ ejjde j fdZd#dedefdZdZd!defdZdZxZS)$rz Nested Transformer (NesT) A PyTorch impl of : `Aggregating Nested Transformers` - https://arxiv.org/abs/2105.12723 r"r6ir6rr7r7r]Tr ?NrravgctdD]V}t|}t|tjjr!t||ksJd|dWt||}t||}t||}||_ |dx|_ |_ g|_ |pt}|p tj}| |_||_t|tjjr$|d|dks Jd|d}||zdks Jd||_d t'j|zd|_||zt1j|jdzdks Jd t5||zt1j|jdz|_t9||||dd |_|jj|_|j|jdz|_g}d t'j d|tC|"|D}d}d }tGt|jD]}||}|$tK|j||j|j||||||| | | | ||||||xj tM||d|gz c_ |}|dz}tj'||_(||d|_)tU|j |j |\}}||_+tj,| |_-||_.|/|dS)a Args: img_size (int, tuple): input image size in_chans (int): number of input channels patch_size (int): patch size num_levels (int): number of block hierarchies (T_d in the paper) embed_dims (int, tuple): embedding dimensions of each level num_heads (int, tuple): number of attention heads for each level depths (int, tuple): number of transformer layers for each level num_classes (int): number of classes for classification head mlp_ratio (int): ratio of mlp hidden dim to embedding dim for MLP of transformer layers qkv_bias (bool): enable bias for qkv if True drop_rate (float): dropout rate for MLP of transformer layers, MSA final projection layer, and classifier attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate norm_layer: (nn.Module): normalization layer for transformer layers act_layer: (nn.Module): activation layer in MLP of transformer layers pad_type: str: Type of padding to use '' for PyTorch symmetric, 'same' for TF SAME weight_init: (str): weight init scheme global_pool: (str): type of pooling operation to apply to final feature map Notes: - Default values follow NesT-B from the original Jax code. - `embed_dims`, `num_heads`, `depths` should be ints or tuples with length `num_levels`. - For those following the paper, Table A1 may have errors! - https://github.com/google-research/nested-transformer/issues/2  embed_dimsr&depthsz Require `len(z) == num_levels`r;rrz Model only handles square inputsz*`patch_size` must divide `img_size` evenlyr6zUFirst level blocks don't fit evenly. Check `img_size`, `patch_size`, and `num_levels`F)img_size patch_sizein_chansrflattenc6g|]}|S)tolist)rrFs r3rz!Nest.__init__..[s eee1AHHJJeeer4N)rir0r-r+rerjrar}zlevels.)num_chs reductionmoduler7 pool_type)0r$r%locals isinstance collectionsabcSequencerr num_classes num_featureshead_hidden_size feature_inforrro drop_rate num_levelsrrTarangefliprrrrrgrr patch_embed num_patchesrlinspacesumsplitrappendrdictrlevelsryr global_poolr* head_drophead init_weights)r.rrrrrr&rrrir0rproj_drop_rateattn_drop_ratedrop_path_raterjrar} weight_initr param_name param_valuerdp_ratesprev_dim curr_striderr/rr2s r3r%z Nest.__init__sb ? d dJ ((:.K+{'?@@ d;'':5557cz7c7c7c555*Yz**:66 )Ij)))44 &:&&v..&4>rNBD1,9 ( "$ h  8 9 9 #A;(1+---/Q---{H*$)))+W)))$ Z 8 88>>qAAHHJJJ&$)DOA4F*G*GG1LLL dMLL x:5$)DOTUDV:W:WWXX&! m     +7*doa.@@eeq.#f++(V(V(\(\]c(d(deee s4?++,,  AQ-C MM)"! q #!(("1+%#!   "   $skR_\]R_R_"`"`"`!a a  H 1 KKmV, Jz"~.. .d.?AQ]hiii T&I..  +&&&&&r4c|dvsJd|vrtj|j nd}|jD]}t |jdddt tt||dS) N)nlhbrrrr {Gz?r:r7stdab) head_bias) rlogrrr rrr_init_nest_weights)r.moderlevels r3rzNest.init_weightss|####39T>>TXd.////r [ ? ?E %/sbA > > > > >G.)DDDdKKKKKr4cXdtt|jDS)Nch|]}d|d S)zlevel.z .pos_embedr)rrs r3 z'Nest.no_weight_decay..s$HHH1&&&&HHHr4)rrrr.s r3no_weight_decayzNest.no_weight_decays(HHc$+6F6F0G0GHHHHr4Fc:td|rdnddfddg}|S)Nz ^patch_embedz^levels\.(\d+)z*^levels\.(\d+)\.transformer_encoder\.(\d+))z"^levels\.(\d+)\.(?:pool|pos_embed))r)z^norm)i)stemblocks)r)r.coarsematchers r3 group_matcherzNest.group_matchers@ &,_""2_aef=$   r4c(|jD] }||_ dSrm)rr)r.enablels r3set_grad_checkpointingzNest.set_grad_checkpointings( * *A#)A  * *r4returnc|jSrm)rrs r3get_classifierzNest.get_classifiers yr4rrcf||_t|j|j|\|_|_dS)Nr)rr rrr)r.rrs r3reset_classifierzNest.reset_classifiers<&&7  t/;'H'H'H#$)))r4c||}||}||dddddddd}|S)Nrr7r"r)rrryr>rs r3forward_featureszNest.forward_featuress`   Q   KKNN IIaii1a++ , , 4 4Q1a @ @r4 pre_logitsc||}||}|r|n||Srm)rrr)r.rFrs r3 forward_headzNest.forward_heads?   Q   NN1  0qqDIIaLL0r4cZ||}||}|Srm)rrrs r3rOz Nest.forwards-  ! !! $ $   a r4)rr"r6r"rrrrr]Tr r r rNNrrrrrrF)T)r)rPrQrRrSr%rTrUignorerrrrrModulerrgstrrrrWrrOrYrZs@r3rrs& )C'C'C'C'C'C'J YLLLL YIII Y     Y**** Y HHCHcHHHH 11$1111 r4rrr rnamerc2t|tjr|dr?t |jdddtj|j|dSt |jddd|j&tj |jdSdSt|tj rEt |jddd|j(tj |jdSdSdS)zn NesT weight initialization Can replicate Jax implementation. Otherwise follows vision_transformer.py rrr:r7rN) rrr( startswithr weightinit constant_r#zeros_Conv2d)rrrs r3rrs &")$$ ( ??6 " " , &-SB! < < < < G  fk9 5 5 5 5 5 &-SB! < < < <{&v{+++++'& FBI & &(fma8888 ; " GNN6; ' ' ' ' '(( " "r4c td|j|j|jd}|jdd\}}tt j||z}t |tt j|dddd}tj |||gdd}t|ddddtt j|}|S) z Rescale the grid of position embeddings when loading from state_dict Expected shape of position embeddings is (1, T, N, C), and considers only square images z$Resized position embedding: %s to %sr7rr"rbicubicF)sizer align_corners) _loggerinfor<rgrrrr>r@ interpolater)posemb posemb_newseq_length_oldnum_blocks_newseq_length_newsize_news r3resize_pos_embedr#s  LL7zGWXXX\!_N%/%5ac%:"NN49^N:;;< > ? ? G G1aQR S SF ]68(<9\a b b bF fnnQ1a00#di6O6O2P2P Q QF Mr4cd|D}|D]M}||jt||jkr't||t||||<N|S)z4 resize positional embeddings of pretrained weights c<g|]}|d|S) pos_embed_)r)rrLs r3rz(checkpoint_filter_fn..s)QQQAall<6P6PQaQQQr4)keysr<getattrr#) state_dictmodelpos_embed_keysrLs r3checkpoint_filter_fnr,swQQ!2!2QQQN OO a= '%"3"3"9 9 9,Z]GE1ZPZee^deeL  M M* M M ME Lr4c Btddddd|}tdd|i|}|S)  Nest-T @ 224x224 rKrNr7r7rr nest_tinyr3r)rUrFrGs r3rUrUs?d>ZPYdd]cddL  L L L| L LE Lr4c n|ddtd dddd|}td d|i|}|S) rDr}samerrrr nest_base_jxr3r)rX setdefaultrr5rGs r3rXrX$sd j&)))W"jWWOUWWL  O OJ O, O OE Lr4c n|ddtd dddd|}td d|i|}|S) rJr}rWrKrNrr nest_small_jxr3r)r\rYrGs r3r\r\/sU j&)))e>ZPZee^deeL  P PZ P< P PE Lr4c n|ddtd dddd|}td d|i|}|S) rSr}rWrKrNrTr nest_tiny_jxr3r)r^rYrGs r3r^r^9sU j&)))d>ZPYdd]cddL  O OJ O, O OE Lr4rXr\r^) jx_nest_base jx_nest_small jx_nest_tiny)rrr r r)ArScollections.abcrloggingr functoolsrrTtorch.nn.functionalr functionalr@ timm.datarr timm.layersrr r r r r rrrrr_builderr _features_fxr _manipulaterr _registryrrr__all__ getLoggerrPrr rr\rqrgrrrrr floatrr#r,r5rA default_cfgsrErQrUrXr\r^rr4r3rqs"  AAAAAAAA\\\\\\\\\\\\\\\\ZZZZZZZZZZZZZZ******33333344444444YYYYYYYYYY ( ' H % %))))) )))X)))))ry)))Xry(  C       c      ?%?%?%?%?% ?%?%?%DAAAAA29AAAH((ry((U(((($"    %$466 DFF466"dW555#tg666"dW555&& TdTH"$"''r4