NgRdZddlZddlmZmZmZmZmZddlZddl m Z ddl m cm Z ddlmcmZddlmZmZddlmZmZmZmZmZmZmZmZmZmZmZddl m!Z!ddl"m#Z#dd l$m%Z%dd l&m'Z'm(Z(m)Z)d gZ*ee+ee+e+ffZ,d ej-d ee+e+fdej-fdZ.e%dej-d ee+e+fdee+e+fdej-fdZ/Gdde j0Z1Gdde j0Z2Gdde j0Z3Gdde j0Z4Gdd e j0Z5dZ6dKdZ7dLd Z8e'e8d!d"#e8d!d$d%d&d'(e8d!d)#e8d!d*d%d&d'(e8d!d+#e8d!d,#e8d!d-#e8d!d.#e8d!d/#e8d!d0#e8d!d1d2d3d45e8d!d6d2d3d45d7 Z9e(dKde5fd8Z:e(dKde5fd9Z;e(dKde5fd:Ze(dKde5fd=Z?e(dKde5fd>Z@e(dKde5fd?ZAe(dKde5fd@ZBe(dKde5fdAZCe(dKde5fdBZDe(dKde5fdCZEe)eFdDdEdFdGdHdIdJdS)MaK Swin Transformer V2 A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 Code/weights from https://github.com/microsoft/Swin-Transformer, original copyright/license info below Modifications and additions for timm hacked together by / Copyright 2022, Ross Wightman N)CallableListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPath to_2tuple trunc_normal__assertClassifierHeadresample_patch_embedndgrid get_act_layer LayerType)build_model_with_cfg)feature_take_indices)register_notrace_function)generate_default_cfgsregister_modelregister_model_deprecationsSwinTransformerV2x window_sizereturnc*|j\}}}}||||dz|d||dz|d|}|ddddddd|d|d|}|S)z Args: x: (B, H, W, C) window_size (int): window size Returns: windows: (num_windows*B, window_size, window_size, C) rrshapeviewpermute contiguous)rrBHWCwindowss [/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/swin_transformer_v2.pywindow_partitionr2$sJAq!Q q!{1~%{1~qKN7JKXYN\]^^Aii1aAq))4466;;B AP[\]P^`abbG Nr0img_sizec|\}}|jd}|d||dz||dz|d|d|}|ddddddd|||}|S)z Args: windows: (num_windows * B, window_size[0], window_size[1], C) window_size (Tuple[int, int]): Window size img_size (Tuple[int, int]): Image size Returns: x: (B, H, W, C) r&rrr"r#r$r%r')r0rr4r-r.r/rs r1window_reverser63s DAq bA Rk!n,a;q>.A;q>S^_`SacdeeA !Q1a##..0055b!QBBA Hr3ceZdZdZ ddedeeefded ed ed ed ed eeefddffd ZdZ deeefddfdZ dde j de e j de j fdZxZS)WindowAttentiona Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 proj_drop (float, optional): Dropout ratio of output. Default: 0.0 pretrained_window_size (tuple[int]): The height and width of the window in pre-training. TFrrdimr num_headsqkv_biasqkv_bias_separate attn_drop proj_droppretrained_window_sizer Nc @t||_||_t ||_||_||_tj tj dtj |ddfz|_ tjtjdddtjdtjd|d|_tj||d zd|_|rtj tj||_|d tj|d tj tj||_nd|_d|_d|_tj||_tj|||_tj||_tjd |_|dS)N rr#iTbias)inplaceFr"k_bias persistentr&r;)super__init__r;rrrAr<r>nn Parametertorchlogones logit_scale SequentialLinearReLUcpb_mlpqkvzerosq_biasregister_bufferv_biasrGDropoutr?projr@Softmaxsoftmax"_make_pair_wise_relative_positions) selfr;rr<r=r>r?r@rA __class__s r1rLzWindowAttention.__init__Ss &&/0F&G&G#"!2< "uz9aQRBS7T7T2T(U(UVV} Ia4 ( ( ( GD ! ! ! Ic95 1 1 1   9S#'666  ,u{3'7'788DK  5;s+;+; N N N,u{3'7'788DKKDKDKDKI..Ic3'' I..zb)))  //11111r3ctj|jddz |jdtj}tj|jddz |jdtj}tjt ||}|ddd d}|j ddkrQ|dddddddfxx|j ddz zcc<|dddddddfxx|j ddz zcc<nP|dddddddfxx|jddz zcc<|dddddddfxx|jddz zcc<|dz}tj |tj tj |dzztj dz }|d|dtj|jd}tj|jd}tjt ||}tj|d}|dddddf|dddddfz }|ddd}|dddddfxx|jddz z cc<|dddddfxx|jddz z cc<|dddddfxxd|jdzdz zcc<|d } |d | ddS) Nrrr#?relative_coords_tableFrHr&relative_position_index)rOarangertofloat32stackrr*r+ unsqueezerAsignlog2absmathrZflattensum) rarelative_coords_hrelative_coords_wrfcoords_hcoords_wcoordscoords_flattenrelative_coordsrgs r1r`z2WindowAttention._make_pair_wise_relative_positions~s!L4+;A+>+B)CTEUVWEXYY\\]b]jkk!L4+;A+>+B)CTEUVWEXYY\\]b]jkk % F3DFW,X,X Y Y 5 = =aA F F Q Q S S ] ]^_ ` `  &q )A - - !!!!QQQ1* - - -$2Ma2PST2T U - - - !!!!QQQ1* - - -$2Ma2PST2T U - - - - !!!!QQQ1* - - -$2B12E2I J - - - !!!!QQQ1* - - -$2B12E2I J - - -" % +@ A AEJ I+ , ,s 2E4E4!46:ill!C 46KX]^^^< 0 344< 0 344VHh7788vq11(AAAt4~aaaqqqj7QQ)11!Q::EEGG111a   D$4Q$7!$;;   111a   D$4Q$7!$;;   111a   A(8(;$;a$??   "1"5"5b"9"9 68O\abbbbbr3ctt|}||jkr||_|dSdS)zzUpdate window size & interpolate position embeddings Args: window_size (int): New window size N)rrr`)rars r1set_window_sizezWindowAttention.set_window_sizesG  ,, $* * **D   3 3 5 5 5 5 5 + *r3rmaskc|j\}}}|j||}nitj|j|j|jf}|jr||}||z }n!tj ||jj |}| ||d|j d ddddd}|d\}} } tj|d tj| d d dz} tj|jt'jd  } | | z} ||jd|j } | |jd|jd|jdz|jd|jdzd}| ddd}d tj|z}| |dz} ||jd}| d||j |||ddz} | d|j ||} || } n|| } || } | | zdd |||}| |}|!|}|S)z Args: x: input features with shape of (num_windows*B, N, C) mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None N)weightrEr"r&r#rrr$rJgY@)max)"r(rYrWrOcatrGr[r>Flinearr~reshaper<r*unbind normalize transposeclamprRrprPexprVrfr)rgrr+sigmoidrlr_r?r]r@)rarr|B_Nr/rWr=qkvattnrRrelative_position_bias_tablerelative_position_biasnum_wins r1forwardzWindowAttention.forwards 7Aq ; ((1++CCy$+t{DK!HIIH% IhhqkkxhqxHHHkk"aDNB77??1aANN**Q--1a A2&&&QB)?)?)?)I)I"b)Q)QQk$"28K8KLLLPPRR k!'+||D4N'O'O'T'TUWY]Yg'h'h$!=d>Z>_>_`b>c>c!d!i!i  Q $"21"5 5t7G7JTM]^_M`7`bd"f"f!7!?!?1a!H!H!S!S!U!U!#em4J&K&K!K,66q999  jmG99R$.!Q??$..QRBSBSB]B]^_B`B``D99RA66D<<%%DD<<%%D~~d## AX A & & . .r1a 8 8 IIaLL NN1  r3)TFr9r9r:N)__name__ __module__ __qualname____doc__intrboolfloatrLr`r{rOTensorrr __classcell__rbs@r1r8r8Es;  $"&+!!6<)2)2)2sCx)2 )2  )2 $ )2)2)2%*#s(O)2 )2)2)2)2)2)2Vccc<65c?6t6666---Xel-C-u|--------r3r8ceZdZdZddddddddddejdf d ed ed ed ed ededede dede de de de dej deffd Z d$de ejde ejfdZ d$dede edeeeefeeefffdZ d$d eeefd eeefde efd!Zdejdejfd"Zdejdejfd#ZxZS)%SwinTransformerV2Blockz Swin Transformer Block. rF@Tr9gelur;input_resolutionr<r shift_sizealways_partition dynamic_mask mlp_ratior=r@r? drop_path act_layer norm_layerrAc t||_t||_||_t||_||_||_| ||\|_ |_ |j d|j dz|_ ||_ t| } t|t|j || | | t||_|||_| dkrt%| nt'j|_t-|t/||z| | |_|||_| dkrt%| nt'j|_|d|jrdn|d dS) a Args: dim: Number of input channels. input_resolution: Input resolution. num_heads: Number of attention heads. window_size: Window size. shift_size: Shift size for SW-MSA. always_partition: Always partition into full windows and shift mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Dropout rate. attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer. norm_layer: Normalization layer. pretrained_window_size: Window size in pretraining. rr)rr<r=r?r@rAr9) in_featureshidden_featuresrdrop attn_maskNFrH)rKrLr;rrr<target_shift_sizerr_calc_window_shiftrr window_arearrr8rnorm1r rMIdentity drop_path1r rmlpnorm2 drop_path2rZ get_attn_mask)rar;rr<rrrrrr=r@r?rrrrArbs r1rLzSwinTransformerV2Block.__init__sF  )*: ; ;"!*:!6!6 0(,0,C,CKQ[,\,\)$/+A.1A!1DD"!),, # !$"233#,-C#D#D     Z__ 1:R(9---R[]]i00      Z__ 1:R(9---R[]]  % ?DD4+=+=+?+?      r3Nrr c t|jr| tjdg|jdR}n;tjd|jd|jddf|j|j}d}d|jd f|jd |jd f|jd dffD]n}d|jd f|jd |jd f|jd dffD]0}||dd|d|d|d|dddf<|dz }1ot||j}| d|j }| d| dz }| |dktd |dktd}nd}|S)Nrr#)dtypedevicerr&gYr9)anyrrOrXrr(rrrr2r)rrl masked_fillr)rarimg_maskcnthw mask_windowsrs r1rz$SwinTransformerV2Block.get_attn_mask$s t   y ;'ED,A'E1'E'EFF ;171:qwqz1'EQW]^]efffC)!,,-&q))DOA,>+>?oa(($/   T-a001*1--0B/BC/!,,d3A z=SwinTransformerV2Block._calc_window_shift..Ts(eeedaAFFqqeeer3c*g|]\}}}||krdn|Srr)rrrss r1rz=SwinTransformerV2Block._calc_window_shift..Us*sssWQ1166aaqsssr3)rrrrziprtuple)rarrrrs r1rz)SwinTransformerV2Block._calc_window_shift@s ''9::  $ $ 6 $%% ]%7%:a%?ASTUAVZ[A[$\! )*; < <    9%'88 8&'9::%&788eec$:OQc6d6deee ssD'>>$BRSTBUU!!$q4+;A+>'>>$BRSTBUUH'++I1a57QRR   2r1%Y0@AA NN2t'7;;  4 / / '**955IIIyyy;; $((T-=a-@$BRSTBUWXYY "<1AB8LL aaa!RaRl+6688    9T_6JJJAAAr3ct|j\}}}}|||||z}||d|}|||||z}|||||}|S)Nr&)r(rrrrrrr)rarr,r-r.r/s r1rzSwinTransformerV2Block.forwardsW 1a  4::a== 9 9:: : IIaQ    488A;; 7 788 8 IIaAq ! !r3r)rrrrrM LayerNormr_int_or_tuple_2_trrrModulerLrrOrrrrrrrrrs@r1rrsL./,-%*!&!!!!!#)$&L89!H H H 0H  H + H * H #H H H H H H H !H  H %6!H H H H H H Tx 5%,AW>>B55 15 ((9:5 uS#Xc3h/ 0 5555804   S#X sCx 'tn     4$u|$ $$$$L%,r3rczeZdZdZdejfdedeedejffd Z de j de j fd Z xZ S) PatchMergingz Patch Merging Layer. Nr;out_dimrct||_|pd|z|_t jd|z|jd|_||j|_dS)z Args: dim (int): Number of input channels. out_dim (int): Number of output channels (or 2 * dim if None) norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm r#r$FrDN)rKrLr;rrMrT reductionnorm)rar;rrrbs r1rLzPatchMerging.__init__sg )!c' 1s7DLuEEEJt|,, r3rr c||j\}}}}ddd|dzd|dzf}tj||}|j\}}}}|||dzd|dzd|ddddddd}||}||}|S)Nrr#rr"r$r%) r(rMrrrr*rqrr)rarr,r-r.r/ pad_valuesrs r1rzPatchMerging.forwardsW 1aAq1uaQ/ M  a , ,W 1a IIaaAFAq 1 1 9 9!Q1a K K S STU V V NN1   IIaLLr3)rrrrrMrrrrrLrOrrrrs@r1rrs &*$&L ---c]- ------$  %,        r3rc'eZdZdZdddddddddejddf ded ed ed ed ed edededede dede de de de e e fdej dededdf&fd Z d"deeefd edeefdZdejdejfd Zd#d!ZxZS)$SwinTransformerV2Stagez" A Swin Transformer V2 Stage. FrTr9rrr;rrdepthr<rrr downsamplerr=r@r?rrrrA output_nchwr Nc4 t|_|_| rt d|Dn|_|_|_d_tt dD| rt|_ n |ksJtj _ tj   fdt|D_dS)a  Args: dim: Number of input channels. out_dim: Number of output channels. input_resolution: Input resolution. depth: Number of blocks. num_heads: Number of attention heads. window_size: Local window size. always_partition: Always partition into full windows and shift dynamic_mask: Create attention mask in forward based on current input size downsample: Use downsample layer at start of the block. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Projection dropout rate attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer type. norm_layer: Normalization layer. pretrained_window_size: Local window size in pretraining. output_nchw: Output tensors on NCHW format instead of NHWC. c3 K|] }|dzV dSr#Nrris r1 z2SwinTransformerV2Stage.__init__..s&&H&H!qAv&H&H&H&H&H&Hr3Fcg|]}|dzSr#r)rrs r1rz3SwinTransformerV2Stage.__init__..s888qAF888r3)r;rrcg|]M}t j |dzdkrdn ttr|n NS)r#r)r;rr<rrrrrr=r@r?rrrrA)routput_resolution isinstancelist)rrrrr?rrrrr<rrAr@r=rarrs r1rz3SwinTransformerV2Stage.__init__..s%#%#%#$# #!%!7#'!"Q!11*!1)#!##*4Y*E*ET)A,,9#%'=   %#%#%#r3N)rKrLr;rrrrrgrad_checkpointingrrrrMr ModuleListrangeblocks)rar;rrrr<rrrrrr=r@r?rrrrArrrbs` ` ```` ```````` @r1rLzSwinTransformerV2Stage.__init__soT  0LV!l&H&H7G&H&H&H!H!H!H\l &"' ,, 88K88899   ,*sGPZ[[[DOO'>>>> kmmDOm%#%#%#%#%#%#%#%#%#%#%#%#%#%#%#%#%#%#$5\\%%#%#%#$$ r3rc$||_t|jtjr||_n:t|jt sJtd|D|_|jD]}| |j|| dS)a Updates the resolution, window size and so the pair-wise relative positions. Args: feat_size: New input (feature) resolution window_size: New window size always_partition: Always partition / shift the window c3 K|] }|dzV dSrrrs r1rz8SwinTransformerV2Stage.set_input_size..*s&*E*Ea16*E*E*E*E*E*Er3rrrN) rrrrMrrrrrr)rarrrblocks r1rz%SwinTransformerV2Stage.set_input_sizes!* dor{ 3 3 F%.D " "do|<< < < <%**E*E9*E*E*E%E%ED "[  E  0'!1 !      r3rc||}|jD]H}|jr4tjst j||}=||}I|Sr)rrrrOjit is_scripting checkpoint)rarblks r1rzSwinTransformerV2Stage.forward2sl OOA  ;  C& uy/E/E/G/G )#q11CFFr3cj|jD]}tj|jjdtj|jjdtj|jjdtj|jjddS)Nr)rrMinit constant_rrEr~r)rars r1_init_respostnormz(SwinTransformerV2Stage._init_respostnorm<s; 3 3C G  cina 0 0 0 G  ci. 2 2 2 G  cina 0 0 0 G  ci. 2 2 2 2  3 3r3r)r N)rrrrrMrrrrrrstrrrrLrrrrOrrrrrs@r1rrs&+!&$!!!!!.4$&L89 %'N$N$N$N$0 N$  N$  N$+N$#N$N$N$N$N$N$N$N$ S(]+!N$" #N$$%6%N$&'N$( )N$N$N$N$N$N$h04 S#X'tn 4%,33333333r3rc+feZdZdZdddddddd d d d d d dddddejdfdedededededede edfde edfdede de de de d e d!e d"e d#e d$e ee fd%e d&e edff(fd' Zd(Z dCdee eefdee eefdee eefd+eedee f d,Zejjd-ZejjdDd.ZejjdEd/Zejjd0ejfd1ZdFdedeefd2Z dGd4ejd5ee eeefd6e d7e d8ed9e d0e eeje ejeejfffd:Z dHd5e eeefdZd?ZdDd@e fdAZ dBZ!xZ"S)Irz Swin Transformer V2 A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 r$r"avg`r#r#r#r"r rFTrr9g?r)rrrrr4 patch_sizein_chans num_classes global_pool embed_dimdepths.r<rrstrict_img_sizerr= drop_rateproj_drop_rateattn_drop_ratedrop_path_raterrpretrained_window_sizesc t||_|dvsJ||_d|_t ||_|_td|jdz zzx|_ |_ g|_ tttfs fdt|jDt!|||d|| d|_|jj}dt'jd|t+||D}g}d}d}t|jD]}|}|t/did |d |d |d|z|d|zfd ||d |dkd||d| d| d| d| d| d|d|d||d|d|d||gz }|}|dkr|dz}|xj t1|d|zd|gz c_ t3j||_||j |_t;|j ||||j|_||j |jD]}|!dS) a] Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of input image channels. num_classes: Number of classes for classification head. embed_dim: Patch embedding dimension. depths: Depth of each Swin Transformer stage (layer). num_heads: Number of attention heads in different layers. window_size: Window size. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. drop_rate: Head dropout rate. proj_drop_rate: Projection dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. norm_layer: Normalization layer. act_layer: Activation layer type. patch_norm: If True, add normalization after patch embedding. pretrained_window_sizes: Pretrained window sizes of each layer. output_fmt: Output tensor format if not None, otherwise output 'NHWC' by default. )rNHWCr#rc:g|]}td|zzSr)r)rrr$s r1rz.SwinTransformerV2.__init__..s*QQQQYa/00QQQr3r)r4r r!r$rr& output_fmtc6g|]}|Sr)tolist)rrs r1rz.SwinTransformerV2.__init__..s ```aqxxzz```r3r;rrrrr<rrrrr=r@r?rrrrAr$layers.)num_chsrmodule) pool_typer' input_fmtNr)"rKrLr"r#r0len num_layersr$r num_featureshead_hidden_size feature_inforrrrr patch_embed grid_sizerOlinspacerrsplitrdictrMrSlayersrrheadapply _init_weightsr)rar4r r!r"r#r$r%r<rrr&rr=r'r(r)r*rrr+kwargsr>dprrBin_dimscalerrblyrbs ` r1rLzSwinTransformerV2.__init__Ksp\ &k))))& f++"47 A$/\]J]D^8^4_4__D1)eT]33 RQQQQ%:P:PQQQI&!l!+   $. ``5>!^S[[#Q#Q#W#WX^#_#_```1t'' d dAlG -F#,A,%"719N!O!OQii  q55  $A,, (K"2!1"100$)").).a&&$) &:!"(?q'A'A# F&F1uu    $w!e)Ta^_TaTa"b"b"b!c c   mV, Jt011 "   !o      4%&&&; $ $C  ! ! # # # # $ $r3ct|tjr^t|jdt|tjr0|j+tj|jddSdSdSdS)Ng{Gz?)stdr)rrMrTrr~rErr)rams r1rEzSwinTransformerV2._init_weightss a # # - !( , , , ,!RY'' -AF,>!!!&!,,,,, - - - -,>,>r3Nrd window_ratiocV||(|j|||jj}|tfd|D}t |jD]G\}}dt |dz dz} ||d| z|d| zf||HdS)aTUpdates the image resolution, window size, and so the pair-wise relative positions. Args: img_size (Optional[Tuple[int, int]]): New input resolution, if None current resolution is used patch_size (Optional[Tuple[int, int]): New patch size, if None use current patch size window_size (Optional[int]): New window size, if None based on new_img_size // window_div window_ratio (int): divisor for calculating window size from patch grid size always_partition: always partition / shift windows even if feat size is < window N)r4r cg|]}|zSrr)rrrNs r1rz4SwinTransformerV2.set_input_size..s F F Fql!2 F F Fr3r#rrr )r=rr>r enumeraterBr) rar4r rrNrr>indexstage stage_scales ` r1rz SwinTransformerV2.set_input_sizes"  :#9   + +X* + U U U(2I  <#; F F F FI F F FGGK%dk22  LE5s519a000K  $Q<;6 ! 8ST'!1 !      r3ct}|D]5\}tfddDr|6|S)Ncg|]}|vSrr)rkwns r1rz5SwinTransformerV2.no_weight_decay..sAAAB!GAAAr3)rVrR)set named_modulesradd)ranodrMrXs @r1no_weight_decayz!SwinTransformerV2.no_weight_decaysfee&&((  DAqAAAA&@AAABB   r3c0td|rdngdS)Nz^absolute_pos_embed|patch_embedz^layers\.(\d+)))z^layers\.(\d+).downsampler)z^layers\.(\d+)\.\w+\.(\d+)N)z^norm)i)stemr)rA)racoarses r1 group_matcherzSwinTransformerV2.group_matchers93(.$$555    r3c(|jD] }||_ dSr)rBr)raenablels r1set_grad_checkpointingz(SwinTransformerV2.set_grad_checkpointings( * *A#)A  * *r3r c|jjSr)rCfc)ras r1get_classifierz SwinTransformerV2.get_classifiers y|r3cJ||_|j||dSr)r"rCreset)rar"r#s r1reset_classifierz"SwinTransformerV2.reset_classifiers&&  [11111r3NCHWrindicesr stop_earlyr0intermediates_onlycx|dvs Jdg}tt|j|\}} ||}t|j} tjs|s|j} n|jd| dz} t| D]v\} } | |}| |vrb|r| | dz kr||}n|}| dddd }| |w|r|S||}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )rlzOutput shape must be NCHW.Nrrr"r#) rr8rBr=rOr rrQrr*r+append)rarrmrrnr0ro intermediates take_indices max_index num_stagesstagesrrSx_inters r1forward_intermediatesz'SwinTransformerV2.forward_intermediatess^*Y&&&(D&&& "6s4;7G7G"Q"Q i   Q  %% 9 ! ! # # 1: 1[FF[)a-0F!&)) . .HAuaAL   Aa//"iillGGG!//!Q155@@BB$$W---  ! IIaLL-r3r prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|dd|S)z@ Prune layers not required for specified intermediates. Nrrr-)rr8rBrMrrrk)rarmryrzrsrts r1prune_intermediate_layersz+SwinTransformerV2.prune_intermediate_layers7sq#7s4;7G7G"Q"Q ik.9q=.1  & DI  )  ! !!R ( ( (r3c||}||}||}|Sr)r=rBrrars r1forward_featuresz"SwinTransformerV2.forward_featuresGs8   Q   KKNN IIaLLr3 pre_logitsc^|r||dn||S)NT)r)rC)rarrs r1 forward_headzSwinTransformerV2.forward_headMs,0:Ltyyty,,, ! Lr3cZ||}||}|Sr)rrr~s r1rzSwinTransformerV2.forwardPs-  ! !! $ $   a r3)NNNrdNF)Tr)NFFrlF)rFT)#rrrrrMrrrrrrrrrrLrErrrOr ignorer]rarerrhrkrrrxr|rrrrrs@r1rrDs'+.#$&2)7-.%*$(!!!$&$&$'.4#%<7C+r$r$'r$r$ r$  r$  r$r$#s(Or$S#Xr$+r$#r$"r$r$r$r$ "!r$""#r$$"%r$&S(]+'r$(!)r$*&+38_+r$r$r$r$r$r$h---374859*+/3 uS#X/!sCx1"%S/2  #3-  'tn @ Y Y     Y**** Y 22C2hsm22228<$$',0 0 |0 eCcN340  0  0  0 !%0  tEL!5tEL7I)I#JJ K0 0 0 0 h./$# 3S >*   MM$MMMMr3c |d|}|d|}d|v}i}ddl}|D]\ }t fddDr!d vrS|jjjj\}}}}|jd|ks|jd |krt|||fd d d }|s.| d d  dd || <|S)Nmodel state_dictzhead.fc.weightrcg|]}|vSrr)rrXrs r1rz(checkpoint_filter_fn..]sbbb1Qbbbr3)rgrfrzpatch_embed.proj.weightrr&bicubicT) interpolation antialiasverbosezlayers.(\d+).downsamplecTdt|ddzdS)Nr3rz .downsample)rgroup)rs r1z&checkpoint_filter_fn..ms)=gs177ST::YZGZ=g=g=gr3zhead.zhead.fc.) getreitemsrr=r]r~r(rsubreplace) rrnative_checkpointout_dictrrrr-r.rs @r1checkpoint_filter_fnrVs>44J j99J(J6H III  ""1 bbbb abbb c c   $ ) )*/6.us&\\da\\\\\\r3r%)rrrr out_indicesT)flatten_sequentialr)pretrained_filter_fn feature_cfg)rrQrpoprrrrA)variant pretrainedrFdefault_out_indicesrrs r1_create_swin_transformer_v2rts\\i 8\8Z8Z.[.[\\\\\**],?@@K 7J 1DkJJJ      E Lr3r-c 8|ddddddttddd d |S) Nr)r"r)rdrdg?rTzpatch_embed.projzhead.fcmit) urlr" input_size pool_sizecrop_pctrfixed_input_sizemeanrL first_conv classifierlicenser)rrFs r1_cfgrs:=v%.B(     #  r3ztimm/z{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to16_192to256_22kto1k_ft.pth) hf_hub_idrz{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to24_192to384_22kto1k_ft.pth)r"r)rrre)rrrrrz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to16_192to256_22kto1k_ft.pthz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to24_192to384_22kto1k_ft.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window16_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window8_256.pthzhhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window16_256.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window16_256.pthzkhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12_192_22k.pthiQU)r"r)rr)rrr"rrzlhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12_192_22k.pth) 2swinv2_base_window12to16_192to256.ms_in22k_ft_in1k2swinv2_base_window12to24_192to384.ms_in22k_ft_in1k3swinv2_large_window12to16_192to256.ms_in22k_ft_in1k3swinv2_large_window12to24_192to384.ms_in22k_ft_in1kzswinv2_tiny_window8_256.ms_in1kz swinv2_tiny_window16_256.ms_in1kz swinv2_small_window8_256.ms_in1kz!swinv2_small_window16_256.ms_in1kzswinv2_base_window8_256.ms_in1kz swinv2_base_window16_256.ms_in1k!swinv2_base_window12_192.ms_in22k"swinv2_large_window12_192.ms_in22kc \tdddd}t dd|it|fi|S)  rrrrrr$r%r<swinv2_tiny_window16_256r)rrArrrF model_argss r1rrs\"B:>X>XQW>X>X Z ZZr3c \tdddd}t dd|it|fi|S) rrdrrrrswinv2_small_window8_256r)rrrs r1rrs\!r-SabbbJ &" Y Y/9 Y=A*=W=WPV=W=W Y YYr3c \tdddd}t dd|it|fi|S) rrrr$rdr rswinv2_base_window16_256r)rrrs r1rr\"MUcdddJ &" Y Y/9 Y=A*=W=WPV=W=W Y YYr3c \tdddd}t dd|it|fi|S) rrdrrrrswinv2_base_window8_256r)rrrs r1rrs[!s=TbcccJ &! X X.8 X<@B:>X>XQW>X>X Z ZZr3c ^tddddd}t d d|it|fi|S) rrrrrrr"swinv2_large_window12to16_192to256r)rrrs r1rr#g#m /111J ', c c9C cGKJGaGaZ`GaGa c ccr3c ^tddddd}t d d|it|fi|S) rrrrrrr"swinv2_large_window12to24_192to384r)rrrs r1rr.rr3rrrrrr)swinv2_base_window12_192_22k)swinv2_base_window12to16_192to256_22kft1k)swinv2_base_window12to24_192to384_22kft1kswinv2_large_window12_192_22k*swinv2_large_window12to16_192to256_22kft1k*swinv2_large_window12to24_192to384_22kft1kr)r-)GrrptypingrrrrrrOtorch.nnrMtorch.nn.functionalrrtorch.utils.checkpointutilsr timm.datar r timm.layersr r r rrrrrrrr_builderr _featuresr _features_fxr _registryrrr__all__rrrr2r6rr8rrrrrrr default_cfgsrrrrrrrrrrrrrrr3r1rs 99999999999999 +++++++++AAAAAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;******++++++333333YYYYYYYYYY  #uS#X./  5c? u|     EL uS#X RWX[]`X`Ra fkfr    "NNNNNbiNNNbHHHHHRYHHHV     29   F}3}3}3}3}3RY}3}3}3@OOOOO OOOd<    %$:>$ J;;;;?$ J Hs;;; <@4 K<<<<@4 K Hs<<< (,t t((()- u))))- u)))*. v***(,t t((()- u))) *. ymv*** +/$ zmv+++e7&7&77 tYY