
    NgS              	          d Z ddlZddlmZ ddlmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddlm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' dgZ( G d dej)                  Z* G d dej)                  Z+ G d dej)                  Z, G d dej)                  Z- G d dej)                  Z.dee/e/f         fdZ0e"dee/e/f         dee/e/f         fd            Z1 G d dej)                  Z2 G d dej)                  Z3 G d d ej)                  Z4 G d! dej)                  Z5d3d#Z6d4d%Z7 e' e7d&'           e7d('           e7d)'           e7d*'           e7d+'          d,          Z8e&d3d-e5fd.            Z9e&d3d-e5fd/            Z:e&d3d-e5fd0            Z;e&d3d-e5fd1            Z<e&d3d-e5fd2            Z=dS )5a)   Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
    N)partial)CallableListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tuple	to_ntupleMlpClassifierHeadLayerNorm2dget_attnget_act_layerget_norm_layer
RelPosBias_assert   )build_model_with_cfg)register_notrace_function)named_apply)register_modelgenerate_default_cfgsGlobalContextVitc                   >     e Zd ZdZddddej        f fd	Zd Z xZS )MbConvBlockzR A depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ?seFc           	         t                                                       t          |          }t          |t                    r|dk    s|dk    r
d|d<   d|d<   t          |          }|p|}t          ||z            }t          j        ||dd	d	||
          | _	         |            | _
         ||fi || _        t          j        ||d	d	d|          | _        d S )N	act_layerr    ecag      ?rd_ratioFbias   r   )groupsr&   r   r&   )super__init__dict
isinstancestrr   intnnConv2dconv_dwactr    conv_pw)
selfin_chsout_chsexpand_ratio
attn_layerr&   r#   attn_kwargsmid_chs	__class__s
            M/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/gcvit.pyr+   zMbConvBlock.__init__,   s     	Y///j#&& 	(:+=+=uATAT&*K
#"'Kj))
#VlV+,,y!Q&tTTT9;;*W4444y'1aFFF    c                     |}|                      |          }|                     |          }|                     |          }|                     |          }||z   }|S N)r2   r3   r    r4   )r5   xshortcuts      r=   forwardzMbConvBlock.forwardC   sQ    LLOOHHQKKGGAJJLLOOLr>   )	__name__
__module____qualname____doc__r0   GELUr+   rC   __classcell__r<   s   @r=   r   r   )   so         
 gG G G G G G.      r>   r   c                   8     e Zd Zddej        ef fd	Zd Z xZS )Downsample2dNconvc                    t                                                       |p|}| ||          nt          j                    | _        t          ||          | _        |dv sJ |dk    r t          j        ||dddd          | _        nM|d	k    r%||k    sJ t          j	        ddd
          | _        n"||k    sJ t          j
        d          | _        | ||          nt          j                    | _        d S )Nr"   )rM   maxavgrM   r'      r   Fr)   rO   kernel_sizestridepaddingrS   )r*   r+   r0   Identitynorm1r   
conv_blockr1   	reduction	MaxPool2d	AvgPool2dnorm2)r5   dimdim_outrZ   r#   
norm_layerr<   s         r=   r+   zDownsample2d.__init__N   s    	.S(2(>ZZ___BKMM
%cY???22222YsGQ15IIIDNN%'>>>>\a1MMMDNN'>>>>\a888DN,6,BZZ(((


r>   c                     |                      |          }|                     |          }|                     |          }|                     |          }|S r@   )rX   rY   rZ   r]   r5   rA   s     r=   rC   zDownsample2d.forwardf   sG    JJqMMOOANN1JJqMMr>   )	rD   rE   rF   r0   rH   r   r+   rC   rI   rJ   s   @r=   rL   rL   M   sb         g"V V V V V V0      r>   rL   c                   6     e Zd Zddej        f fd	Zd Z xZS )FeatureBlockr   rO   c                    t                                                       |}t          d|          }|dk    rt          t          j        d          }nt          t          j        ddd          }t	          j                    | _        t          |          D ]b}| j        
                    d|dz    t          ||                     |r.| j        
                    d	|dz     |                       |dz  }cd S )
Nr   rP   rQ   rV   r'   rR   rM   r"   pool)r*   r+   rO   r   r0   r\   r[   
Sequentialblocksrange
add_moduler   )	r5   r^   levelsrZ   r#   
reductionspool_fnir<   s	           r=   r+   zFeatureBlock.__init__o   s     	
Qbl:::GGbl!QOOOGmoov 	  	 AK""<!A#<<SI1V1V1VWWW  &&|ac||WWYY???a
		  	 r>   c                 ,    |                      |          S r@   )rh   rb   s     r=   rC   zFeatureBlock.forward   s    {{1~~r>   )rD   rE   rF   r0   rH   r+   rC   rI   rJ   s   @r=   rd   rd   n   sY         g           *      r>   rd   c            	       J     e Zd Zddej        efdedededef fdZd Z	 xZ
S )	Stemr'   `   r6   r7   r#   r`   c                     t                                                       t          j        ||ddd          | _        t          |||          | _        d S )Nr'   rQ   r   rR   )r#   r`   )r*   r+   r0   r1   conv1rL   down)r5   r6   r7   r#   r`   r<   s        r=   r+   zStem.__init__   sR     	YvwAaQRSSS
 I*UUU			r>   c                 Z    |                      |          }|                     |          }|S r@   )rt   ru   rb   s     r=   rC   zStem.forward   s%    JJqMMIIaLLr>   )rD   rE   rF   r0   rH   r   r/   r   r+   rC   rI   rJ   s   @r=   rq   rq      s         "$'#.	V 	V	V 	V  		V
 !	V 	V 	V 	V 	V 	V      r>   rq   c                   z     e Zd Z	 	 	 	 ddededeeef         dededed	ef fd
Zddee	j
                 fdZ xZS )WindowAttentionGlobalT        r^   	num_headswindow_size
use_globalqkv_bias	attn_drop	proj_dropc                    t                                                       t          |          }|| _        || _        ||z  | _        | j        dz  | _        || _        t          ||          | _	        | j        r t          j        ||dz  |          | _        nt          j        ||dz  |          | _        t          j        |          | _        t          j        ||          | _        t          j        |          | _        d S )Ng      )r{   rz   rQ   r)   r'   )r*   r+   r   r{   rz   head_dimscaler|   r   rel_posr0   LinearqkvDropoutr~   projr   )	r5   r^   rz   r{   r|   r}   r~   r   r<   s	           r=   r+   zWindowAttentionGlobal.__init__   s     	,,&"y(]d*
$!kYOOO? 	>ycAgH===DHHycAgH===DHI..Ic3''	I..r>   Nq_globalc                 t   |j         \  }}}| j        r|t          |j         d         |j         d         k    d           |                     |          }|                    ||d| j        | j                                      ddddd          }|                    d          \  }}|	                    ||j         d         z  ddd          }	|	                    ||| j        | j                                      dddd          }	nf|                     |                              ||d| j        | j                                      ddddd          }
|
                    d          \  }	}}|	| j
        z  }	|	|                    dd                                          z  }|                     |          }|                    d	          }|                     |          }||z                      dd                              |||          }|                     |          }|                     |          }|S )
Nz*x and q_global seq lengths should be equalrQ   r   r'   r      )r^   )shaper|   r   r   reshaperz   r   permuteunbindrepeatr   	transpose
contiguousr   softmaxr~   r   r   )r5   rA   r   BNCkvkvqr   attns               r=   rC   zWindowAttentionGlobal.forward   s   '1a? 	$x3AGBK8>"#557cddd!BAq!T^T]CCKKAqRSUVXYZZB99Q<<DAqX^A%6 61a@@A		!Q>>FFq!QPQRRAA((1++%%aAt~t}MMUUVWYZ\]_`bcddCjjmmGAq!
N1;;r2&&11333||D!!|||##~~d##AX  A&&..q!Q77IIaLLNN1r>   )TTry   ry   r@   )rD   rE   rF   r/   r   boolfloatr+   r   torchTensorrC   rI   rJ   s   @r=   rx   rx      s          $!!!/ // / sCx	/
 / / / / / / / / /6 8EL#9        r>   rx   r{   c                 *   | j         \  }}}}|                     |||d         z  |d         ||d         z  |d         |          } |                     dddddd                                                              d|d         |d         |          }|S )Nr   r   r'   rQ   r      r   r   viewr   r   )rA   r{   r   HWr   windowss          r=   window_partitionr      s    JAq!Q	q!{1~%{1~qKN7JKXYN\]^^Aii1aAq))4466;;BAP[\]P^`abbGNr>   img_sizec                    |\  }}| j         d         }|                     d||d         z  ||d         z  |d         |d         |          }|                    dddddd                                                              d|||          }|S )Nr   r   r   r'   rQ   r   r   r   )r   r{   r   r   r   r   rA   s          r=   window_reverser      s    DAqbARk!n,a;q>.A;q>S^_`SacdeeA			!Q1a##..0055b!QBBAHr>   c                   &     e Zd Zd fd	Zd Z xZS )
LayerScaleh㈵>Fc                     t                                                       || _        t          j        |t          j        |          z            | _        d S r@   )r*   r+   inplacer0   	Parameterr   onesgamma)r5   r^   init_valuesr   r<   s       r=   r+   zLayerScale.__init__   sB    \+
3"?@@


r>   c                 X    | j         r|                    | j                  n	|| j        z  S r@   )r   mul_r   rb   s     r=   rC   zLayerScale.forward   s(    %)\Eqvvdj!!!q4:~Er>   )r   F)rD   rE   rF   r+   rC   rI   rJ   s   @r=   r   r      sY        A A A A A A
F F F F F F Fr>   r   c                        e Zd Zddddddddeej        ej        fdedeeef         ded	ed
e	de
de
dee	         de	de	de	dededef fdZddeej                 fdZddeej                 fdZ xZS )GlobalContextVitBlock         @TNry   r^   	feat_sizerz   r{   	mlp_ratior|   r}   layer_scaler   r~   	drop_pathr9   r#   r`   c           	         t                                                       t          |          }t          |          }|| _        t	          |d         |d         z  |d         |d         z  z            | _         ||          | _         |||||||
|	          | _        |t          ||          nt          j
                    | _        |dk    rt          |          nt          j
                    | _         ||          | _        t          |t	          ||z            ||	          | _        |t          ||          nt          j
                    | _        |dk    rt          |          nt          j
                    | _        d S )Nr   r   )rz   r{   r|   r}   r~   r   ry   )in_featureshidden_featuresr#   drop)r*   r+   r   r{   r/   num_windowsrX   r   r   r0   rW   ls1r   
drop_path1r]   r   mlpls2
drop_path2)r5   r^   r   rz   r{   r   r|   r}   r   r   r~   r   r9   r#   r`   r<   s                  r=   r+   zGlobalContextVitBlock.__init__   st   " 	i((	,,&	!A >9Q<S^_`SaCabccZ__
J#!
 
 
	 4?3J:c;///PRP[P]P]1:R(9---R[]]Z__
3C)O8L8LXahqrrr3>3J:c;///PRP[P]P]1:R(9---R[]]r>   r   c                    |j         \  }}}}t          || j                  }|                    d| j        d         | j        d         z  |          }|                     ||          }t          || j        ||f          }|S )Nr   r   r   )r   r   r{   r   r   r   )	r5   rA   r   r   r   r   r   x_winattn_wins	            r=   _window_attnz"GlobalContextVitBlock._window_attn  s}    W
1a D$455

2t/2T5Ea5HH!LL99UH--8T%51v>>r>   c           
      L   ||                      |                     |                     |                     |          |                              z   }||                     |                     |                     |                     |                                        z   }|S r@   )r   r   r   rX   r   r   r   r]   )r5   rA   r   s      r=   rC   zGlobalContextVitBlock.forward  s{    ):):4::a==()S)S T TUUU$**Q--)@)@ A ABBBr>   r@   )rD   rE   rF   rx   r0   rH   	LayerNormr/   r   r   r   r   r   r+   r   r   r   rC   rI   rJ   s   @r=   r   r      sc         !!#!+/!!!#8"$'#%<'S 'S'S S#X'S 	'S
 'S 'S 'S 'S "%'S 'S 'S 'S !'S  'S !'S 'S 'S 'S 'S 'SR (>     8EL#9        r>   r   c            !            e Zd Zdddddddddej        ej        efdededeeef         d	eeef         d
e	de	de	de
de	dee
         de
de
deee
         e
f         dededef  fdZd Z xZS )GlobalContextVitStageTFr   Nry   depthrz   r   r{   
downsampleglobal_norm
stage_normr   r}   r   r   r~   r   r#   r`   norm_layer_clc                 
  	
 t                                                       |r6t          dz  |          | _        dz  d         dz  d         dz  fnt	          j                    | _        | _        t                    t          t          j
        t                    t                    z                      }t          |          | _        |r           nt	          j                    | _        t	          j        	
fdt!          |          D                       | _        |r           nt	          j                    | _        | _        | _        d| _        d S )NrQ   )r^   r_   r`   r   r   c                     g | ]B}t          
|d z  dk    t          t                    r|         n	          CS )rQ   r   )r^   rz   r   r{   r   r}   r|   r   r   r~   r   r#   r`   )r   r-   list).0rn   r#   r~   r^   r   r   r   r   r   rz   r   r}   r{   s     r=   
<listcomp>z2GlobalContextVitStage.__init__.<locals>.<listcomp>K  s     %
 %
 %
   "##'#!EQJ'##*4Y*E*ET)A,,9#(  %
 %
 %
r>   F)r*   r+   rL   r   r0   rW   r   r   r/   mathlog2minrd   global_blockr   
ModuleListri   rh   normr^   grad_checkpointing)r5   r^   r   rz   r   r{   r   r   r   r   r}   r   r   r~   r   r#   r`   r   feat_levelsr<   s    ` ```   ``````` ` r=   r+   zGlobalContextVitStage.__init__%  s   ( 	 		,*a%  DO
 'C"1*IaLA,=>II kmmDO",,$)C	NNS5E5E$EFFGG(k::1<O==---"+--m %
 %
 %
 %
 %
 %
 %
 %
 %
 %
 %
 %
 %
 %
 %
  5\\!%
 %
 %
  $ +5GMM#&&&"+--	""'r>   c                     |                      |          }|                     |          }|                    dddd          }|                     |                    dddd                    }| j        D ]I}| j        r4t          j                                        st          j	        ||          }= |||          }J| 
                    |          }|                    dddd                                          }|S )Nr   rQ   r'   r   )r   r   r   r   rh   r   r   jitis_scripting
checkpointr   r   )r5   rA   global_queryblks       r=   rC   zGlobalContextVitStage.forwardb  s    OOA((++ IIaAq!!''(<(<Q1a(H(HII; 	) 	)C& )uy/E/E/G/G ))#q11C<((IIaLLIIaAq!!,,..r>   )rD   rE   rF   r0   rH   r   r   r/   r   r   r   r   r   r   r   r+   rC   rI   rJ   s   @r=   r   r   $  s>         $ %$!!+/!!36"$'#%<&1%;( ;( ;( 	;(
 S#X;( sCx;( ;( ;( ;( ;( ;( "%;( ;( ;( T%[%/0;(   !;(" !#;($ $%;( ;( ;( ;( ;( ;(z      r>   r   c            )       0    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d6dedededeeef         deedf         deedf         dedeedf         deedf         dededee         ded ed!ed"ed#ed$ed%ed&ef( fd'Z	d7d)Z
ej        j        d*             Zej        j        d8d,            Zej        j        d9d-            Zej        j        d.ej        fd/            Zd:dedee         fd0Zd1ej        d.ej        fd2Zd8d3efd4Zd1ej        d.ej        fd5Z xZS );r   r'     rP          r      r   N@   r'   r      r   rQ   r      r         @Try    gelulayernorm2d	layernormr   in_chansnum_classesglobal_poolr   window_ratio.r{   	embed_dimdepthsrz   r   r}   r   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater#   r`   r   norm_epsc                 &   t                                                       t          |          }t          t	          |          |          }t          t	          |          |          }t                    t          d D                       }|| _        || _        || _	        t          |          }t          |d|dz
  z  z            x| _        | _        | t          |          |          }n5|J t          fd t          |          |          D                       }t          ||||          | _        d t#          j        d|t'          |                                        |          D             }g }t+          |          D ]}||dz
  k    }dt-          |dz
  d          z  }|                    t1          di d	||z  d
||         d|	|         d|d         |z  |d         |z  fd||         d|dk    d|d|
d|d|d|d|d||         d|d|d|           t3          j        | | _        t9          | j        |||          | _        |r&t=          t          | j        |          |            d S d S )N)epsc              3       K   | ]	}|d z  V  
dS )r   N )r   ds     r=   	<genexpr>z,GlobalContextVit.__init__.<locals>.<genexpr>  s&      33Q!q&333333r>   rQ   r   c                 <    g | ]}d          |z  d         |z  fS )r   r   r  )r   rr   s     r=   r   z-GlobalContextVit.__init__.<locals>.<listcomp>  s0     s s s!(1+"2HQK14D!E s s sr>   )r6   r7   r#   r`   c                 6    g | ]}|                                 S r  )tolist)r   rA   s     r=   r   z-GlobalContextVit.__init__.<locals>.<listcomp>  s     ```aqxxzz```r>   r   r^   r   rz   r   r{   r   r   r   r}   r   r   r~   r   r#   r`   r   	pool_typer   )schemer  ) r*   r+   r   r   r   r   tupler   r   r   lenr/   num_featureshead_hidden_sizer   rq   stemr   linspacesumsplitri   rO   appendr   r0   rg   stagesr   headr   _init_weights)r5   r   r   r   r   r   r{   r   r   rz   r   r}   r   r   r   r  r  weight_initr#   r`   r   r  r   
num_stagesdprr  rn   
last_stagestage_scaler<   s       `                        r=   r+   zGlobalContextVit.__init__u  sP   0 	!),,	^J77XFFF
} = =8LLLX&&33(33333	&&"[[
47	A*WX.DY8Y4Z4ZZD1"/)J//<<KK+++ s s s sOdyYcOdOdeqOrOr s s sttK!	
 
 
	 a`5>!^S[[#Q#Q#W#WX^#_#_```z"" 	 	Aj1n,Js1q5!}},KMM/   ++Qii $A,, %Q<;6	!8STT	
 (NN 66 &: $) " (K ). ). a&& $) &:  ,m!    $ mV, #4#4k[dmnnn	 	O 2;GGGNNNNN	O 	Or>   vitc                 &   |dk    rt          |t          j                  r}t          j                            |j                   |j        Td|v r(t          j                            |j        d           d S t          j                            |j                   d S d S d S t          |t          j                  rSt          j                            |j        d           |j        (t          j                            |j                   d S d S d S )Nr"  r   gư>)stdg{Gz?)	r-   r0   r   initxavier_uniform_weightr&   normal_zeros_)r5   modulenamer  s       r=   r  zGlobalContextVit._init_weights  s    U??&"),, 4''666;*}}>>>>>v{333334 4** &"),, 03777;*GNN6;/////0 0**r>   c                 >    d |                                  D             S )Nc                 P    h | ]"\  }t          fd dD                        #S )c              3       K   | ]}|v V  	d S r@   r  )r   nr   s     r=   r	  z=GlobalContextVit.no_weight_decay.<locals>.<setcomp>.<genexpr>  s'      SSa16SSSSSSr>   )relative_position_bias_tablezrel_pos.mlp)any)r   _r   s     @r=   	<setcomp>z3GlobalContextVit.no_weight_decay.<locals>.<setcomp>  sX     U U U!QSSSS#RSSSSSUU U Ur>   )named_parametersr5   s    r=   no_weight_decayz GlobalContextVit.no_weight_decay  s5    U U//11U U U 	Ur>   Fc                 (    t          dd          }|S )Nz^stemz^stages\.(\d+))r  rh   )r,   )r5   coarsematchers      r=   group_matcherzGlobalContextVit.group_matcher  s#    $
 
 
 r>   c                 (    | j         D ]	}||_        
d S r@   )r  r   )r5   enabless      r=   set_grad_checkpointingz'GlobalContextVit.set_grad_checkpointing  s(     	* 	*A#)A  	* 	*r>   returnc                     | j         j        S r@   )r  fcr5  s    r=   get_classifierzGlobalContextVit.get_classifier  s    y|r>   c                 ~    || _         || j        j        j        }t	          | j        ||| j                  | _        d S )Nr  )r   r  r   r  r   r  r   )r5   r   r   s      r=   reset_classifierz!GlobalContextVit.reset_classifier  s@    &)/9K"4#4k[dhdrsss			r>   rA   c                 Z    |                      |          }|                     |          }|S r@   )r  r  rb   s     r=   forward_featuresz!GlobalContextVit.forward_features  s%    IIaLLKKNNr>   
pre_logitsc                 ^    |r|                      ||          n|                      |          S )N)rG  )r  )r5   rA   rG  s      r=   forward_headzGlobalContextVit.forward_head  s-    6@Rtyyzy222diiPQllRr>   c                 Z    |                      |          }|                     |          }|S r@   )rF  rI  rb   s     r=   rC   zGlobalContextVit.forward  s-    !!!$$a  r>   )r'   r   rP   r   r   Nr   r   r   r   TNry   ry   ry   ry   r   r   r   r   r   )r"  F)Tr@   )rD   rE   rF   r/   r.   r   r   r   r   r+   r  r   r   ignorer6  r:  r>  r0   ModulerB  rD  r   rF  rI  rC   rI   rJ   s   @r=   r   r   t  s        #$(+,<+/&3)6"!+/!$&$&$&#+!,"-NO NONO NO 	NO
 CHoNO  S/NO sCxNO NO #s(ONO S#XNO NO NO "%NO NO "NO  "!NO" "#NO& 'NO( )NO* +NO, -NO NO NO NO NO NO`0 0 0 0  YU U U
 Y    Y* * * * Y	    t tC thsm t t t t%, 5<    
S S$ S S S S %,        r>   Fc                 x    |                     dd           rt          d          t          t          | |fi |}|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretrainedkwargsmodels       r=   _create_gcvitrV     sG    zz/4(( [YZZZ !17JQQ&QQELr>   r   c                 6    | dddddt           t          dddd	|S )
Nr   )r'   r   r   )r   r   g      ?bicubicz
stem.conv1zhead.fcT)urlr   
input_size	pool_sizecrop_pctinterpolationmeanr$  
first_conv
classifierfixed_input_sizer	   )rY  rT  s     r=   _cfgrb    s8    4}SYI%.B")    r>   z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)rY  z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1kr?  c                 <    t          dddd|}t          dd| i|S )N)rQ   rQ      rQ   r   r   rz   gcvit_xxtinyrS  r  )rf  r,   rV  rS  rT  model_kwargss      r=   rf  rf     sF        L OOJO,OOOr>   c                 <    t          dddd|}t          dd| i|S )N)r'   r   rd  r   r   re  gcvit_xtinyrS  r  )rk  rg  rh  s      r=   rk  rk  )  sF        L NN:NNNNr>   c                 <    t          dddd|}t          dd| i|S )Nr   r   re  
gcvit_tinyrS  r  )rm  rg  rh  s      r=   rm  rm  2  sF        L MM*MMMMr>   c           	      B    t          d	dddddd|}t          d
d| i|S )Nr   )r'   rd        rr   rQ   r   r   rz   r   r   r   gcvit_smallrS  r  )rr  rg  rh  s      r=   rr  rr  ;  sO         L NN:NNNNr>   c           	      B    t          d	dddddd|}t          d
d| i|S )Nr   )r   r   r   r      rQ   r   rq  
gcvit_baserS  r  )ru  rg  rh  s      r=   ru  ru  G  sO         L MM*MMMMr>   rK  )r   )>rG   r   	functoolsr   typingr   r   r   r   r   r   torch.nnr0   torch.utils.checkpointutilsr   	timm.datar
   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   _builderr   _features_fxr   _manipulater   	_registryr   r   __all__rM  r   rL   rd   rq   rx   r/   r   r   r   r   r   r   rV  rb  default_cfgsrf  rk  rm  rr  ru  r  r>   r=   <module>r     s   *        9 9 9 9 9 9 9 9 9 9 9 9 9 9        + + + + + + + + + A A A A A A A AA A A A A A A A A A A A A A A A A A A A A A A A A A * * * * * * 3 3 3 3 3 3 $ $ $ $ $ $ < < < < < < < <
! ! ! ! !") ! ! !H    29   B    29   4    29   $5 5 5 5 5BI 5 5 5pU38_     sCx E#s(O    F F F F F F F F5 5 5 5 5BI 5 5 5pM M M M MBI M M M`I I I I Iry I I IX       %$ LM M M KL L Lt JK K K KL L Lt JK K K& &   P P0@ P P P P O O/? O O O O N N.> N N N N O O/? O O O O N N.> N N N N N Nr>   