NgnHdZddlZddlmZddlmZmZmZmZm Z ddl Z ddl m Z ddl m cmZddlZ ddlmZmZmZmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#ddl$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z/m0Z0dgZ1ej2e3Z4de5de5de j6de j6fdZ7e+e7de j6de j6de j6dee5e5fdee5e5fde j6f dZ8Gdde j9Z:Gdde j9Z;Gdde j9ZGd%de j9Z?d&Z@d=d(ZAe/eAd)d*d+eedd,d-.eAd/d*d+eedd,d-.eAd0d*d+eedd,d-.eAeed1d2d34d5ZBd>d7ZCe0d>de?fd8ZDe0d>de?fd9ZEe0d>de?fd:ZFe0d>de?fd;ZGdS)?a+ Vision Transformer (ViT) in PyTorch A PyTorch implement of Vision Transformers as described in: 'Exploring Plain Vision Transformer Backbones for Object Detection' - https://arxiv.org/abs/2203.16527 'Segment Anything Model (SAM)' - https://github.com/facebookresearch/segment-anything/ N)partial)CallableListOptionalTupleUnion)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STD) PatchEmbedMlpDropPath PatchDropout LayerNorm2dClassifierHeadNormMlpClassifierHeadFormatresample_abs_pos_embed_nhwcRotaryEmbeddingCatapply_rot_embed_cat to_2tupleuse_fused_attn)Final)build_model_with_cfg)feature_take_indices)register_notrace_function)checkpoint_seq)generate_default_cfgsregister_modelVisionTransformerSAMq_sizek_sizerel_posreturnctdt||zdz }|jd|krwtj|d|jddddd|d}|d|dd}n|}tj|dddft||z dz}tj|dddft||z dz}||z |dz t||z dzz}|| S) a\ Get relative positional embeddings according to the relative positions of query and key sizes. Args: q_size (int): size of query q. k_size (int): size of key k. rel_pos (Tensor): relative position embeddings (L, C). Returns: Extracted positional embeddings according to relative positions. rrlinear)sizemodeN?) intmaxshapeF interpolatereshapepermutetorcharangelong)r#r$r% max_rel_distrel_pos_resizedq_coordsk_coordsrelative_coordss ^/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/vision_transformer_sam.py get_rel_posr>&sKq3vv...233L}Q<''- OOAw}Q/ 4 4 < bhwkzbhwc,wkc->bhwkNr))r>r0r3r5einsum)r@rArBr#r$q_hq_wk_hk_wRhRwB_dimr_qrel_hrel_w attn_biass r=get_decomposed_rel_pos_biasrRIs(HCHC S#y ) )B S#y ) )BIAq# ))AsC % %C L)3 3 3E L)3 3 3EaaaAAAqqq$&'%111aaaqqq0@*AAI   RsC#I 6 66r?c eZdZUeeed<dddddejdddf dedee e e fd eej ffd Z d Z xZS) Attention fused_attnTFN use_rel_pos input_sizeropec t||zdks Jd||_||z|_|jdz|_t |_tj||dz||_ |r||jntj |_ |r||jntj |_ tj ||_tj|||_tj ||_||_|jr| J| Jdtjt'jd| dzdz |j|_tjt'jd| dzdz |j|_| |_dS) Nrz$dim should be divisible by num_headsg)biaszBInput size must be provided if using relative positional encoding.r(r)super__init__ num_headshead_dimscalerrUnnLinearqkvIdentityq_normk_normDropout attn_dropproj proj_droprX Parameterr5zerosrArBrZ) selfrMr`qkv_biasqk_normrjrl norm_layerrXrYrZ __class__s r=r_zAttention.__init__ns Y!###%K###"y( ]d* (**9S#'9993:Mjj///  3:Mjj///  I..Ic3'' I..&   7<<<&&S'&& \%+JqM!A%t}+6+677DN\%+JqM!A%t}+6+677DN r?c|j\}}}}||z}|||d}||||d|jdddddd}|d||jz|dd\}} } |||| } }|j r"t||j |j ||f||f} nhd} |j _|j } t|| | }t| | | } |jr=t$jj|| | | |jr |jjnd}nZ||jz}|| d dz} | | | z} | d } || } | | z}|||j|ddd||d}||}||}||||d}|S) Nr)r\r(rrrW) attn_mask dropout_p)rM)r0r3reviewr`r4unbindrgrhrXrRrArBrZ get_embedrtype_asrUr5rc functionalscaled_dot_product_attentiontrainingrjprb transposesoftmaxrkrl)roxrKHWrLNrer@kvrQrZattns r=forwardzAttention.forwardsmW 1a E IIaB  hhqkkq!Q;;CCAq!QPQRR++aT^!3Q;;BB1EE1a{{1~~t{{1~~1   <3At~t~XY[\W]`acd_effIIIy$y**,,'40088;;'40088;; ? #@@1a#.2mC$.**AAA DJAq{{2r***D$i'<>$''DqA FF1dna , , 6 6q! < < D DQ2 N N IIaLL NN1   FF1aB  r?)__name__ __module__ __qualname__rbool__annotations__rc LayerNormrrr.Moduler_r __classcell__rss@r=rTrTksd  | %48(,%%%!sCx1%29%%%%%%%N&&&&&&&r?rTc&eZdZdfd ZdZxZS) LayerScaleh㈵>Fct||_tj|t j|z|_dSN)r^r_inplacercrmr5onesgamma)rorM init_valuesrrss r=r_zLayerScale.__init__sB  \+ 3"?@@ r?cX|jr||jn ||jzSr)rmul_rrors r=rzLayerScale.forwards(%)\Eqvvdj!!!q4:~Er?)rF)rrrr_rrrs@r=rrsYAAAAAA FFFFFFFr?rcVeZdZdddddddejejeddddffd ZdZxZ S) Block@TFrWNrc t||_| ||_t ||||||| | |dkr|n||f| |_|rt ||ntj|_ | dkrt| ntj|_ | ||_ | |t||z| ||_|rt ||ntj|_| dkrt| ntj|_dS)Nr) r`rprqrjrlrrrXrYrZ)rrW) in_featureshidden_features act_layerdrop)r^r_ window_sizenorm1rTrrrcrfls1r drop_path1norm2r.mlpls2 drop_path2)rorMr` mlp_ratiorprqrlrjr drop_pathrrr mlp_layerrXrrYrZrss r=r_zBlock.__init__sX& &Z__  !#%0A%5%5zzK;U     @K]:c{;;;;PRP[P]P]1:R(9---R[]]Z__ 9i00     @K]:c{;;;;PRP[P]P]1:R(9---R[]]r?c d|j\}}}}|}||}d}|jdkrt||j\}}||||}|jdkrt||j||f|}||z}||||zd}|| | | | |z}||||d}|S)Nrr)) r0rrwindow_partitionrrrwindow_unpartitionr3rrrr)rorrKrrrLshortcutpad_hws r=rz Block.forwardsW 1a JJqMM,0  a  (D,<==IAv OODHHTYYq\\22 3 3  a  "1d&6AGGA qL IIaQ # # $**Q--)@)@ A ABB B IIaAr " "r?) rrrrcGELUrrr_rrrs@r=rrs g|#-S-S-S-S-S-S^r?rrrc `|j\}}}}|||zz |z}|||zz |z}tj|ddd|d|f}||z||z} }||||z|| |z||}|ddddddd|||} | || ffS)aU Partition into non-overlapping windows with padding if needed. Args: x (tensor): input tokens with [B, H, W, C]. window_size (int): window size. Returns: windows: windows after partition with [B * num_windows, window_size, window_size, C]. (Hp, Wp): padded height and width before partition rrr\r(rur))r0r1padryr4 contiguous) rrrKrrCpad_hpad_wHpWpwindowss r=rrsJAq!Q 1{? *k 9E 1{? *k 9E a!Q5!U+,,A YE B q" #[" 2C[RSTTAii1aAq))4466;;B [Z[\\G RH r?rhwrcd||n|\}}|\}}|jd||z|z|zz}||||z||z||d} | dddddd|||d} | ddd|d|ddf} | S) a Window unpartition into original sequences and removing padding. Args: windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. window_size (int): window size. pad_hw (Tuple): padded height and width (Hp, Wp). hw (Tuple): original height and width (H, W) before padding. Returns: x: unpartitioned sequences with [B, H, W, C]. Nrr)rr\r(rur)r0ryr4r) rrrrrrrrrKrs r=rr)s)VVrFB DAq aR"W 3{BCA Qk)2+ed?ed@edAed8ee ejeeje ejfffdBZ! dNd=eee e e fdCedDefdEZ"dFZ#dJdGefdHZ$dIZ%xZ&S)Or"z Vision Transformer for Segment-Anything Model(SAM) A PyTorch impl of : `Exploring Plain Vision Transformer Backbones for Object Detection` or `Segment Anything Model (SAM)` - https://arxiv.org/abs/2010.11929 r\ rTFNrW) output_fmtstrict_img_sizeavgimg_size patch_sizein_chans num_classes embed_dimdepthr`rrprqrpre_norm drop_rate pos_drop_ratepatch_drop_rateproj_drop_rateattn_drop_ratedrop_path_rate weight_init embed_layerrrrblock_fnr use_abs_posrXuse_roperglobal_attn_indexes. neck_chans global_poolhead_hidden_sizeref_feat_shapec" $%&tpttjdp tj|_|_x_x_ _ d_ ||||| _ j j %tj drj n|&|r;tjt#jd%d%d_nd_tj| _|dkrt-|d _ntj_| r ntj_|rr Jd |!@t5|!d ksJt7|!d}"t7|!d}#ndx}"}#t9zd%|" _t9zdt7|# _nd_d_dt#jd||D$tj $%  fdtC|D_"&fdtC|D_#|rktj tj$|ddtK|tj$||dddtK|_&|_n1| rtj_&ntK_&}| rtO||| || _(dStS|||| _(dS)a Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of image input channels. num_classes: Mumber of classes for classification head. global_pool: Type of global pooling for final sequence (default: 'token'). embed_dim: Transformer embedding dimension. depth: Depth of transformer. num_heads: Number of attention heads. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: Enable bias for qkv projections if True. init_values: Layer-scale init values (layer-scale enabled if not None). drop_rate: Head dropout rate. pos_drop_rate: Position embedding dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. weight_init: Weight initialization scheme. embed_layer: Patch embedding layer. norm_layer: Normalization layer. act_layer: MLP activation layer. block_fn: Transformer block layer. use_abs_pos: If True, use absolute positional embeddings. use_rel_pos: If True, add relative positional embeddings to the attention map. use_rope: If True, add rotary position embeddings to q/k in attention block. window_size: Window size for window attention blocks. If 0, not use window attention. global_attn_indexes: Indexes for blocks using global attention. Used when window_size > 0. global_pool: Global pooling type. head_hidden_size: If set, use NormMlpHead ref_feat_shape: Tuple of reference feature shapes for ROPE, (global, local) gư>)epsF)rrrrr] feat_ratiorrN)r)num_prefix_tokenszCROPE and relative pos embeddings should not be enabled at same timer() in_pixels feat_shaperc6g|]}|Sr)item).0rs r= z1VisionTransformerSAM.__init__..s JJJAqvvxxJJJr?c g|]T}didd d ddd ddd|d d d d d |vrnddd|vrjnjUS)rMr`rrprqrrlrjrrrrrrXrrrYrZr) rope_window rope_global)rirrrdprrr grid_sizerrrrrr`rrqrprorXrs r=rz1VisionTransformerSAM.__init__..s&#&#&#&% H   I #) $) "       (K  ). ). a&& &: $) $) (K ,-4G+G+GKKQ %9  *+2E)E)ET%%4K[! &#&#&#r?c:g|]}td|S)zblocks.)modulenum_chs reductiondict)rrrrs r=rz1VisionTransformerSAM.__init__..sA^^^KLD ! yA F F F^^^r?) kernel_sizer]r\)rpaddingr]) hidden_size pool_typer)rr)*r^r_rrcrrrr num_featuresrrgrad_checkpointing patch_embedrhasattrrrmr5rn pos_embedripos_dropr patch_droprfnorm_prelenrrrrlinspace Sequentialrangeblocks feature_infoConv2drneckrheadr)(rorrrrrrr`rrprqrrrrrrrrrrrrrrrrrXrrrrrrrref_feat_shape_globalref_feat_shape_windowrrrrss(` ` ````` `` ```` ` `` @@@r=r_zVisionTransformerSAM.__init__GstF B72**a////(1.2C(D(D%(1.2C(D(D%%@DD%(=1Y&$4    D  2Y&$[114    D   $D #D KJ>5!I!IJJJm&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&#&5\\'&#&#&#$ (^^^^^PUV[P\P\^^^  #   !  J''  ! J''DI"!+D   3KMM ( 22 "J  -,%# DIII'%# DIIIr?c ddhS)Nr dist_tokenrros r=no_weight_decayz$VisionTransformerSAM.no_weight_decay s \**r?c(tdddgS)Nz^pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i)stemrr)rocoarses r= group_matcherz"VisionTransformerSAM.group_matcher s%*-/CD    r?c||_dSr)r)roenables r=set_grad_checkpointingz+VisionTransformerSAM.set_grad_checkpointings"(r?r&c|jSrrrs r=get_classifierz#VisionTransformerSAM.get_classifiers yr?c<|j||dSr)rreset)rorrs r=reset_classifierz%VisionTransformerSAM.reset_classifiers  [11111r?NCHWrindicesnorm stop_earlyrintermediates_onlyc j|dks Jdg}tt|j|\}} ||}|j%|t |j|jddz}||}||}| |}tj s|s|j} n|jd| dz} t| D]\} } | |}| |vrl|r?|||ddddU||dddd|r|S||dddd}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to all intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: r'z&Output shape for ViT-SAM must be NCHW.Nrr\rr()rr rrrrr0rr r r5jit is_scripting enumerateappendrr4) rorr(r)r*rr+ intermediates take_indices max_indexrrblks r=forward_intermediatesz*VisionTransformerSAM.forward_intermediatess*V###%M### "6s4;7G7G"Q"Q i   Q   > %/! MMMA MM!   OOA   MM!   9 ! ! # # 1: 1[FF[)a-0F'' @ @FAsAAL  @"((199Q1a3H3H)I)IJJJJ!((1aA)>)>???  ! IIaii1a++ , ,-r? prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|dd|S)z@ Prune layers not required for specified intermediates. Nrrr)rr rrcrfrr&)ror(r6r7r2r3s r=prune_intermediate_layersz.VisionTransformerSAM.prune_intermediate_layersWsq#7s4;7G7G"Q"Q ik.9q=.1  & DI  )  ! !!R ( ( (r?c||}|j%|t|j|jddz}||}||}||}|jr4tj st|j |}n| |}| |dddd}|S)Nrr\rr()rrrr0rr r rr5r-r.rrrr4rs r=forward_featuresz%VisionTransformerSAM.forward_featureshs   Q   > %/! MMMA MM!   OOA   MM!    " 59+A+A+C+C t{A..AA AA IIaii1a++ , ,r? pre_logitsc^|r||dn||S)NT)r<r")rorr<s r= forward_headz!VisionTransformerSAM.forward_headws,0:Ltyyty,,, ! Lr?cZ||}||}|Sr)r;r>rs r=rzVisionTransformerSAM.forwardzs-  ! !! $ $   a r?F)Tr)NFFr'F)NFT)'rrr__doc__rr rNHWCrcrrrrr.floatrrstrrrr_r5r-ignorerrr rr#r&Tensorrrr5r9r;r>rrrs@r=r"r"@s~! " !!!+/"!#%%'$&$&$&!$+GJ6;`e$f$f$f-/\,.G!&"% $ %"!35!$.2PTE@@@@ @  @  @@@@@@"%@@@!@ #!@""#@$"%@&"'@()@*"+@,!*-@. )/@01@2 3@45@67@89@:;@<"'sCx=@>?@@A@B'smC@D%U5c?E#s(O+K%LME@@@@@@D Y+++ Y     Y)))) Y 22C2hsm2222 8<$$',6 6 |6 eCcN346  6  6  6 !%6  tEL!5tEL7I)I#JJ K6 6 6 6 t8<$# eCcN34 "   MM$MMMMr?cd|v}i}|D]C\}}|dr!|dd}|dd}n|r>|||<D|S)z Remap SAM checkpoints -> timm z%image_encoder.patch_embed.proj.weightzimage_encoder.rNzmlp.linzmlp.fc)items startswithreplace) state_dictmodelsam_checkpointout_dictrrs r=checkpoint_filter_fnrOs = JNH  ""1 <<( ) ) "##A )X..AA  Or?rc 6|ddddddttddd |S) Nr\rr?bicubicTzpatch_embed.projzhead.fc) urlrrY pool_sizecrop_pct interpolationfixed_input_sizemeanstd first_conv classifier)r r )rUkwargss r=_cfgr_s7?'0F(      r?zDhttps://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pthztimm/z apache-2.0rRr-)rU hf_hub_idlicenserZr[rrYrWzDhttps://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pthzDhttps://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pthrQ)r\rbrS)rZr[rrYrW)zsamvit_base_patch16.sa1bzsamvit_large_patch16.sa1bzsamvit_huge_patch16.sa1bsamvit_base_patch16_224Fc |dd}tt||ftt |dd|S)N out_indicesr\getter)re feature_cls)pretrained_filter_fn feature_cfg)poprr"rOr)variant pretrainedr^res r=_create_vision_transformerrmsY**]A..K  2[hGGG       r?c ltddddgdddd}t d d |it|fi|}|S) z# ViT-B/16 for Segment-Anything rrrr(rrV rTrrrrr`rrrXrsamvit_base_patch16rl)rrrrmrlr^ model_argsrLs r=rrrrstB"R_R_R_D4J ' T T*4 T8rRrrTrrrrr"rOr_ default_cfgsrmrrrzrrcrr?r=rs  99999999999999 rrrrrrrrrrrrllllllllllllllllllllllllllllll******++++++333333''''''<<<<<<<< " " ' H % %33S35<3EL3333@+&&&7 |7|7|7 #s(O 7 #s(O 7  \ 7777DPPPPP PPPfFFFFFFFFGGGGGBIGGGT 35uUXZ]U]A^;_0gk  \ (+ 16sCx JRSXY\^aYaSbJc  \    .}}}}}29}}}@ $%$!% R "(