
    Ngj]                        d Z dgZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlm
c mZ ddlmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZm Z   G d dej
        j!                  Z" G d de
j#                  Z$ G d de
j#                  Z% G d de
j#                  Z& G d de
j#                  Z' G d de
j#                  Z( G d dej
        j#                  Z) G d de
j#                  Z* ee*            G d de
j#                  Z+ G d de
j#                  Z,d  Z-d7d"Z. e  e.d#d$%           e.d#&           e.d#&           e.d#d$%           e.d#&           e.d#&           e.d#d$%           e.d#&           e.d#&           e.d#d'd(d)*           e.d#d+d,d)d-.          d/          Z/d8d1Z0ed8d2            Z1ed8d3            Z2ed8d4            Z3ed8d5            Z4ed8d6            Z5dS )9z TinyViT

Paper: `TinyViT: Fast Pretraining Distillation for Small Vision Transformers`
    - https://arxiv.org/abs/2207.10666

Adapted from official impl at https://github.com/microsoft/Cream/tree/main/TinyViT
TinyVit    N)partial)DictOptionalIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)LayerNorm2dNormMlpClassifierHeadDropPathtrunc_normal_resize_rel_pos_bias_table_levituse_fused_attn   )build_model_with_cfg)register_notrace_module)checkpoint_seq)register_modelgenerate_default_cfgsc                   P     e Zd Zd fd	Z ej                    d             Z xZS )ConvNormr   r   c	           
      x   t                                                       t          j        |||||||d          | _        t          j        |          | _        t          j        j        	                    | j        j
        |           t          j        j        	                    | j        j        d           d S )NF)biasr   )super__init__nnConv2dconvBatchNorm2dbntorchinit	constant_weightr   )
selfin_chsout_chsksstridepaddilationgroupsbn_weight_init	__class__s
            P/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/tiny_vit.pyr   zConvNorm.__init__   s    Ifgr63&W\]]]	.))???a00000    c           	      n   | j         | j        }}|j        |j        |j        z   dz  z  }|j        |d d d d d f         z  }|j        |j        |j        z  |j        |j        z   dz  z  z
  }t          j        	                    |
                    d          | j         j        z  |
                    d          |j        dd          | j         j        | j         j        | j         j        | j         j                  }|j        j                            |           |j        j                            |           |S )Ng      ?r   r      )r)   paddingr+   r,   )r   r    r$   running_varepsr   running_meanr!   r   r   sizer,   shaper)   r3   r+   datacopy_)r%   cr    wbms         r/   fusezConvNorm.fuse$   s   	472I"&0S88HqD$,--Gbo	1^bf$,- -HOOFF1II	((!&&))QWQRR[9#TY->I[dhdmdt  v v 	
A	!r0   )r   r   r   r   r   r   )__name__
__module____qualname__r   r!   no_gradr?   __classcell__r.   s   @r/   r   r      s`        1 1 1 1 1 1 U]__  _    r0   r   c                   $     e Zd Z fdZd Z xZS )
PatchEmbedc                     t                                                       d| _        t          ||dz  ddd          | _         |            | _        t          |dz  |ddd          | _        d S )N   r2      r   )r   r   r)   r   conv1actconv2)r%   r&   r'   	act_layerr.   s       r/   r   zPatchEmbed.__init__4   sg    fglAq!<<
9;;glGQ1==


r0   c                     |                      |          }|                     |          }|                     |          }|S N)rK   rL   rM   r%   xs     r/   forwardzPatchEmbed.forward;   s4    JJqMMHHQKKJJqMMr0   r@   rA   rB   r   rS   rD   rE   s   @r/   rG   rG   3   sG        > > > > >      r0   rG   c                   $     e Zd Z fdZd Z xZS )MBConvc                    t                                                       t          ||z            }t          ||d          | _         |            | _        t          ||ddd|          | _         |            | _        t          ||dd          | _         |            | _	        |dk    rt          |          nt          j                    | _        d S )Nr   )r(   rJ   r(   r)   r*   r,           )r(   r-   )r   r   intr   rK   act1rM   act2conv3act3r   r   Identity	drop_path)r%   r&   r'   expand_ratiorN   r`   mid_chsr.   s          r/   r   zMBConv.__init__C   s    f|+,,fg!444
IKK	gw1QAgVVV
IKK	gw1SIII
IKK	09B),,,BKMMr0   c                 :   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }||z  }|                     |          }|S rP   )rK   r[   rM   r\   r]   r`   r^   )r%   rR   shortcuts      r/   rS   zMBConv.forwardN   s    JJqMMIIaLLJJqMMIIaLLJJqMMNN1	XIIaLLr0   rT   rE   s   @r/   rV   rV   B   sL        	R 	R 	R 	R 	R
 
 
 
 
 
 
r0   rV   c                   $     e Zd Z fdZd Z xZS )PatchMergingc                    t                                                       t          ||ddd          | _         |            | _        t          ||ddd|          | _         |            | _        t          ||ddd          | _        d S )Nr   r   rJ   r2   )r,   )r   r   r   rK   r[   rM   r\   r]   )r%   dimout_dimrN   r.   s       r/   r   zPatchMerging.__init__\   s    c7Aq!44
IKK	gw1aHHH
IKK	gw1a88


r0   c                     |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|S rP   )rK   r[   rM   r\   r]   rQ   s     r/   rS   zPatchMerging.forwardd   sR    JJqMMIIaLLJJqMMIIaLLJJqMMr0   rT   rE   s   @r/   rf   rf   [   sG        9 9 9 9 9      r0   rf   c                   *     e Zd Z	 	 d fd	Zd Z xZS )	ConvLayerrY         @c                     t                                                       | _        || _        t	          j        fdt          |          D              | _        d S )Nc                 p    g | ]2}t          t          t                    r|         n          3S  )rV   
isinstancelist).0irN   conv_expand_ratiorh   r`   s     r/   
<listcomp>z&ConvLayer.__init__.<locals>.<listcomp>y   sY     &
 &
 &

 	 S+Y *9d ; ;J	! &
 &
 &
r0   )r   r   rh   depthr   
Sequentialrangeblocks)r%   rh   rw   rN   r`   ru   r.   s    ` ```r/   r   zConvLayer.__init__n   s|     	
m &
 &
 &
 &
 &
 &
 &

 5\\&
 &
 &
 r0   c                 0    |                      |          }|S rP   )rz   rQ   s     r/   rS   zConvLayer.forward   s    KKNNr0   )rY   rm   rT   rE   s   @r/   rl   rl   m   sT               &      r0   rl   c                   D     e Zd Zddej        ej        df fd	Zd Z xZS )NormMlpNrY   c                 b   t                                                       |p|}|p|} ||          | _        t          j        ||          | _         |            | _        t          j        |          | _        t          j        ||          | _	        t          j        |          | _
        d S rP   )r   r   normr   Linearfc1rL   Dropoutdrop1fc2drop2)r%   in_featureshidden_featuresout_features
norm_layerrN   dropr.   s          r/   r   zNormMlp.__init__   s     	#2{)8[J{++	9[/::9;;Z%%
9_l;;Z%%


r0   c                    |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|S rP   )r   r   rL   r   r   r   rQ   s     r/   rS   zNormMlp.forward   sa    IIaLLHHQKKHHQKKJJqMMHHQKKJJqMMr0   )	r@   rA   rB   r   	LayerNormGELUr   rS   rD   rE   s   @r/   r}   r}      sa         !|g& & & & & &&      r0   r}   c                        e Zd ZU ej        j        e         ed<   ee	ej
        f         ed<   	 	 	 d fd	Z ej                    d fd	            Zd	ej        d
ej
        fdZd Z xZS )	Attention
fused_attnattention_bias_cache   rI      r   c           	         t                                                       t          |t                    rt	          |          dk    sJ || _        |dz  | _        || _        t          ||z            | _	        | j	        |z  | _
        || _        || _        t                      | _        t          j        |          | _        t          j        ||| j	        d|z  z   z            | _        t          j        | j
        |          | _        t+          t-          j        t1          |d                   t1          |d                                       }t	          |          }i }g }	|D ]t}
|D ]o}t3          |
d         |d         z
            t3          |
d         |d         z
            f}||vrt	          |          ||<   |	                    ||                    put6          j                            t7          j        |t	          |                              | _        |                     dt7          j         |	          !                    ||          d           i | _"        d S )Nr2   g      r   r   attention_bias_idxsF)
persistent)#r   r   rq   tuplelen	num_headsscalekey_dimrZ   val_dimri   
attn_ratio
resolutionr   r   r   r   r   r   qkvprojrr   	itertoolsproductry   absappendr!   	Parameterzerosattention_biasesregister_buffer
LongTensorviewr   )r%   rh   r   r   r   r   pointsNattention_offsetsidxsp1p2offsetr.   s                r/   r   zAttention.__init__   s/    	*e,,EZA1E1E1E1E"_
:/00|i/$$(**L%%	9S)t|a'k/I"JKKIdlC00	i'jm(<(<eJqM>R>RSSTTKK 	7 	7B 7 7bebem,,c"Q%"Q%-.@.@A!222034E0F0F%f--f56666	7
 !& 2 25;y#N_J`J`3a3a b b2E4DT4J4J4O4OPQST4U4Ubghhh$&!!!r0   Tc                 r    t                                          |           |r| j        ri | _        d S d S d S rP   )r   trainr   )r%   moder.   s     r/   r   zAttention.train   sM    d 	+D- 	+(*D%%%	+ 	+ 	+ 	+r0   devicereturnc                     t           j                                        s| j        r| j        d d | j        f         S t          |          }|| j        vr| j        d d | j        f         | j        |<   | j        |         S rP   )r!   jit
is_tracingtrainingr   r   strr   )r%   r   
device_keys      r/   get_attention_biaseszAttention.get_attention_biases   s    9!! 	9T] 	9(D,D)DEEVJ!:::8<8MaaaQUQiNi8j)*5,Z88r0   c                    |                      |j                  }|j        \  }}}|                     |          }|                     |          }|                    ||| j        d                              | j        | j        | j	        gd          \  }}}	|
                    dddd          }|
                    dddd          }|	
                    dddd          }	| j        rt          j        |||	|          }nC|| j        z  }||                    dd          z  }
|
|z   }
|
                    d          }
|
|	z  }|                    dd                              ||| j                  }|                     |          }|S )	NrJ   )rh   r   r2   r   )	attn_mask)r   r   r8   r   r   r   r   splitr   r   permuter   Fscaled_dot_product_attentionr   	transposesoftmaxreshaperi   r   )r%   rR   	attn_biasBr   _r   qkvattns              r/   rS   zAttention.forward   sr   --ah77	'1aIIaLLhhqkk((1a44::DL$,X\Xd;ekl:mm1aIIaAq!!IIaAq!!IIaAq!!? 	.q!Q)LLLAADJAq{{2r***D)#D<<B<''DqAKK1%%aDL99IIaLLr0   )r   rI   r   T)r@   rA   rB   r!   r   Finalbool__annotations__r   r   Tensorr   rC   r   r   r   rS   rD   rE   s   @r/   r   r      s         	%%%%sEL01111 #' #' #' #' #' #'J U]__+ + + + + _+
95< 9EL 9 9 9 9      r0   r   c                   L     e Zd ZdZdddddej        f fd	Zd Zdefd	Z	 xZ
S )
TinyVitBlocka5   TinyViT Block.

    Args:
        dim (int): Number of input channels.
        num_heads (int): Number of attention heads.
        window_size (int): Window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        drop (float, optional): Dropout rate. Default: 0.0
        drop_path (float, optional): Stochastic depth rate. Default: 0.0
        local_conv_size (int): the kernel size of the convolution between
                               Attention and MLP. Default: 3
        act_layer: the activation function. Default: nn.GELU
       rm   rY   rJ   c	                 L   t                                                       || _        || _        |dk    s
J d            || _        || _        ||z  dk    s
J d            ||z  }	||f}
t          ||	|d|
          | _        |dk    rt          |          nt          j
                    | _        t          |t          ||z            ||          | _        |dk    rt          |          nt          j
                    | _        |dz  }t!          |||d||	          | _        d S )
Nr   z"window_size must be greater than 0z"dim must be divisible by num_headsr   )r   r   rY   )r   r   rN   r   r2   rX   )r   r   rh   r   window_size	mlp_ratior   r   r   r   r_   
drop_path1r}   rZ   mlp
drop_path2r   
local_conv)r%   rh   r   r   r   r   r`   local_conv_sizerN   head_dimwindow_resolutionr*   r.   s               r/   r   zTinyVitBlock.__init__  s;    	"Q D&"Y!###%I###)#(+6c8Y1Qbccc	1:R(9---R[]] i00	
 
 
 2;R(9---R[]]""3s[^___r0   c           	         |j         \  }}}}||z  }|}|| j        k    rQ|| j        k    rF|                    |||          }|                     |          }|                    ||||          }nc| j        || j        z  z
  | j        z  }| j        || j        z  z
  | j        z  }	|dk    p|	dk    }
|
rt          j        |ddd|	d|f          }||z   ||	z   }}|| j        z  }|| j        z  }|                    ||| j        || j        |                              dd                              ||z  |z  | j        | j        z  |          }|                     |          }|                    |||| j        | j        |                              dd                              ||||          }|
r#|d d d |d |f                                         }|| 	                    |          z   }|
                    dddd          }|                     |          }|                    |||                              dd          }||                     |                     |                    z   }|                    ||||          S Nr   r2   rJ   r   )r8   r   r   r   r   r   r*   r   
contiguousr   r   r   r   r   )r%   rR   r   HWCLrd   pad_bpad_rr3   pHpWnHnWs                  r/   rS   zTinyVitBlock.forward*  s   W
1aE   Q$*:%:%:		!Q""A		!Aq!Q""AA%D,<(<<@PPE%D,<(<<@PPEai,519G 9E!aAua788 YE	Bt''Bt''Bq"d.D4DaHHRRSTVWXX``BT-0@@! A 		!A q"b$"2D4DaHHRRSTVWXX``abdfhjlmnnA .aaa!RaRiL++--tq)))IIaAq!!OOAIIaA((A..,,,vvaAq!!!r0   r   c                 F    d| j          d| j         d| j         d| j         S )Ndim=z, num_heads=z, window_size=z, mlp_ratio=)rh   r   r   r   r%   s    r/   
extra_reprzTinyVitBlock.extra_reprR  sQ    Mdh M MDN M M".M M<@NM M 	Mr0   r@   rA   rB   __doc__r   r   r   rS   r   r   rD   rE   s   @r/   r   r      s         $ g#` #` #` #` #` #`J&" &" &"PMC M M M M M M M Mr0   r   c                   L     e Zd ZdZdddddej        f fd	Zd Zdefd	Z	 xZ
S )
TinyVitStagea   A basic TinyViT layer for one stage.

    Args:
        dim (int): Number of input channels.
        out_dim: the output dimension of the layer
        depth (int): Number of blocks.
        num_heads (int): Number of attention heads.
        window_size (int): Local window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
        drop (float, optional): Dropout rate. Default: 0.0
        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
        local_conv_size: the kernel size of the depthwise convolution between attention and MLP. Default: 3
        act_layer: the activation function. Default: nn.GELU
    rm   rY   NrJ   c           
      F  
 t                                                       || _        | _        |	 |	|          | _        n t          j                    | _        |k    sJ t          j        
fdt          |          D              | _	        d S )N)rh   ri   rN   c                 x    g | ]6}t          	t          t                    r|         n           7S ))rh   r   r   r   r   r`   r   rN   )r   rq   rr   )
rs   rt   rN   r   r`   r   r   r   ri   r   s
     r/   rv   z)TinyVitStage.__init__.<locals>.<listcomp>  si     &# &# &#  #'#*4Y*E*ET)A,,9 /#	 	 	&# &# &#r0   )
r   r   rw   ri   
downsampler   r_   rx   ry   rz   )r%   rh   ri   rw   r   r   r   r   r`   r   r   rN   r.   s     ` ````` ``r/   r   zTinyVitStage.__init__k  s     	
 !(j#  DOO !kmmDO'>>>> m &# &# &# &# &# &# &# &# &# &# &# 5\\&# &# &# $r0   c                     |                      |          }|                    dddd          }|                     |          }|                    dddd          }|S r   )r   r   rz   rQ   s     r/   rS   zTinyVitStage.forward  sU    OOAIIaAq!!KKNNIIaAq!!r0   r   c                 &    d| j          d| j         S )Nr   z, depth=)ri   rw   r   s    r/   r   zTinyVitStage.extra_repr  s    8dl88DJ888r0   r   rE   s   @r/   r   r   Z  s         . g*$ *$ *$ *$ *$ *$X  9C 9 9 9 9 9 9 9 9r0   r   c                   j    e Zd Zddddddddd	d
dddej        f fd	Zd Zej        j	        d             Z
ej        j	        d             Zej        j	        dd            Zej        j	        dd            Zej        j	        dej        fd            Zddedee         fdZd ZddefdZd Z xZS ) r   rJ     avg)`        i   r2   r2      r2   )rJ   r        r   r   r   r   rm   rY   皙?Fc                 v   t                                                       || _        || _        t	          |          | _        || _        || _        t          ||d         |          | _	        d t          j        d|
t          |                    D             }t          j                    | _        | j	        j        }|d         }g | _        t%          | j                  D ]}|dk    r)t'          |||         ||d ||                  |          }n||         }|t          |d |                   t          |d |dz                               }
t)          ||dz
           |||         ||         ||         | j        |	||
t*          |          }|}|dz  }| j                            |           | xj        t/          ||d| 	          gz  c_        |d
         x| _        | _        t5          t6          d          }t9          | j        |||          | _        |                     | j                   d S )Nr   )r&   r'   rN   c                 6    g | ]}|                                 S rp   )itemrs   rR   s     r/   rv   z$TinyVit.__init__.<locals>.<listcomp>  s     PPPAqvvxxPPPr0   )rh   rw   rN   r`   ru   r   )rh   ri   rw   r   r   r   r   r   r`   r   rN   r2   zstages.)num_chs	reductionmoduler   gh㈵>)r5   )	pool_typer   ) r   r   num_classesdepthsr   
num_stagesr   grad_checkpointingrG   patch_embedr!   linspacesumr   rx   stagesr)   feature_infory   rl   r   rf   r   dictnum_featureshead_hidden_sizer   r
   r   headapply_init_weights)r%   in_chansr  global_pool
embed_dimsr  r   window_sizesr   	drop_ratedrop_path_rateuse_checkpointmbconv_expand_ratior   rN   dprr)   prev_dim	stage_idxstageri   norm_layer_cfr.   s                         r/   r   zTinyVit.__init__  sq   " 	&f++""0%qM
 
 
 QP>3v;;!O!OPPP moo!(a=t// 	j 	jIA~~!  +'!"46)#4"45&9   %Y/!$S

);%<%<SUVAW=X=X%X!Y$"9q=1# +'	2 ,Y 7"n"$3,+'   #!Ku%%%$x6Rg\eRgRg"h"h"h!ii 5?rNBD1666)!$	
 
 
	 	

4%&&&&&r0   c                     t          |t          j                  r^t          |j        d           t          |t          j                  r0|j        +t          j                            |j        d           d S d S d S d S )Ng{Gz?)stdr   )rq   r   r   r   r$   r   r"   r#   )r%   r>   s     r/   r  zTinyVit._init_weights  s    a## 	-!(,,,,!RY'' -AF,>!!!&!,,,,,	- 	-- -,>,>r0   c                     dhS )Nr   rp   r   s    r/   no_weight_decay_keywordsz TinyVit.no_weight_decay_keywords  s    "##r0   c                 b    d |                                                                  D             S )Nc                     h | ]}d |v |	S )r   rp   r  s     r/   	<setcomp>z*TinyVit.no_weight_decay.<locals>.<setcomp>  s#    OOOa7IQ7N7N7N7N7Nr0   )
state_dictkeysr   s    r/   no_weight_decayzTinyVit.no_weight_decay   s,    OO4??,,1133OOOOr0   c                 4    t          d|rdnddg          }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)stemrz   )r  )r%   coarsematchers      r/   group_matcherzTinyVit.group_matcher  s9     (. $$455
 
 
 r0   Tc                     || _         d S rP   )r  )r%   enables     r/   set_grad_checkpointingzTinyVit.set_grad_checkpointing  s    "(r0   r   c                     | j         j        S rP   )r  fcr   s    r/   get_classifierzTinyVit.get_classifier  s    y|r0   Nr  r!  c                 L    || _         | j                            ||           d S )N)r  )r  r  reset)r%   r  r!  s      r/   reset_classifierzTinyVit.reset_classifier  s(    &	{;;;;;r0   c                     |                      |          }| j        r4t          j                                        st          | j        |          }n|                     |          }|S rP   )r  r  r!   r   is_scriptingr   r  rQ   s     r/   forward_featureszTinyVit.forward_features  s\    Q" 	59+A+A+C+C 	t{A..AAAAr0   
pre_logitsc                 b    |r|                      ||          n|                      |          }|S )N)rH  )r  )r%   rR   rH  s      r/   forward_headzTinyVit.forward_head#  s0    3=ODIIaJI///499Q<<r0   c                 Z    |                      |          }|                     |          }|S rP   )rG  rJ  rQ   s     r/   rS   zTinyVit.forward'  s-    !!!$$a  r0   Fr   rP   )r@   rA   rB   r   r   r   r  r!   r   ignorer0  r6  r;  r>  ModulerA  rZ   r   r   rD  rG  r   rJ  rS   rD   rE   s   @r/   r   r     s        *$&  #gQ' Q' Q' Q' Q' Q'f- - - Y$ $ $ YP P P Y    Y) ) ) ) Y	    < <C <hsm < < < <   $          r0   c                 <   d|                                  v r| d         } |                                }i }|                                 D ]R\  }}|                    d          rd|v r.t	          |j        ||         j        d d d                   j        }|||<   S|S )Nmodelr   r   r   )r5  r4  itemsendswithr   Tr8   )r4  rP  	target_sdout_dictr   r   s         r/   checkpoint_filter_fnrV  -  s    *//####(
  ""IH  ""  1::+,, 	""/Yq\5G"5MNNPAOr0    c           
      2    | dt           t          dddddd	|S )Nr   zpatch_embed.conv1.convzhead.fc)r   r   )rJ      rY  gffffff?)	urlr  meanr.  
first_conv
classifier	pool_size
input_sizecrop_pctr   )rZ  kwargss     r/   _cfgrb  <  s6    %#.#   r0   ztimm/iQU  )	hf_hub_idr  )rc  )rJ   r  r  )r  r  g      ?)rc  r_  r^  r`  )rJ      rd  )   re  squash)rc  r_  r^  r`  	crop_mode)ztiny_vit_5m_224.dist_in22kz"tiny_vit_5m_224.dist_in22k_ft_in1kztiny_vit_5m_224.in1kztiny_vit_11m_224.dist_in22kz#tiny_vit_11m_224.dist_in22k_ft_in1kztiny_vit_11m_224.in1kztiny_vit_21m_224.dist_in22kz#tiny_vit_21m_224.dist_in22k_ft_in1kztiny_vit_21m_224.in1kz#tiny_vit_21m_384.dist_in22k_ft_in1kz#tiny_vit_21m_512.dist_in22k_ft_in1kFc                     |                     dd          }t          t          | |ft          d|          t          d|}|S )Nout_indices)r   r   r2   rJ   T)flatten_sequentialri  )feature_cfgpretrained_filter_fn)popr   r   r  rV  )variant
pretrainedra  ri  rP  s        r/   _create_tiny_vitrp    s]    **]L99K  DkJJJ1   E Lr0   c                     t          g dg dg dg dd          }|                    |           t          d| fi |S )N)@         i@  r  )r2   rI      
   r  rY   r"  r  r   r#  r%  tiny_vit_5m_224r  updaterp  ro  ra  model_kwargss      r/   rx  rx    sf    &&&||--"]]  L -zJJ\JJJr0   c                     t          g dg dg dg dd          }|                    |           t          d| fi |S )N)rr  rs     i  r  )r2   rI   r   r   r  r  rw  tiny_vit_11m_224ry  r{  s      r/   r  r    sf    &&&||--"]]  L .
KKlKKKr0   c                     t          g dg dg dg dd          }|                    |           t          d| fi |S )Nr   r  r  i@  r  rJ   r  r     r  g?rw  tiny_vit_21m_224ry  r{  s      r/   r  r    sf    &&&|| .."]]  L .
KKlKKKr0   c                     t          g dg dg dg dd          }|                    |           t          d| fi |S )Nr  r  r  )r  r  r  r  r  rw  tiny_vit_21m_384ry  r{  s      r/   r  r    h    &&&|| ..%%%  L .
KKlKKKr0   c                     t          g dg dg dg dd          }|                    |           t          d| fi |S )Nr  r  r  )re  re      re  r  rw  tiny_vit_21m_512ry  r{  s      r/   r  r    r  r0   )rW  rL  )6r   __all__r   	functoolsr   typingr   r   r!   torch.nnr   torch.nn.functional
functionalr   	timm.datar   r	   timm.layersr
   r   r   r   r   r   _builderr   _features_fxr   _manipulater   	_registryr   r   rx   r   rN  rG   rV   rf   rl   r}   r   r   r   r   rV  rb  default_cfgsrp  rx  r  r  r  r  rp   r0   r/   <module>r     s    +           ! ! ! ! ! ! ! !                 A A A A A A A AC C C C C C C C C C C C C C C C * * * * * * 1 1 1 1 1 1 ' ' ' ' ' ' < < < < < < < <    ux"   .           RY   2    29   $    	   2    bi   <O O O O O O O Od^M ^M ^M ^M ^M29 ^M ^M ^MB   % % %E9 E9 E9 E9 E929 E9 E9 E9PH H H H Hbi H H HV      %$"&$# # #
 +/$+ + + !D   $(4$ $ $
 ,04, , , "T   $(4$ $ $
 ,04, , , "T   ,04 Hs, , ,
 ,04 Hsh, , ,[2& 2& 2 2j
 
 
 
 	K 	K 	K 	K 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	Lr0   