
    Ng
_                        d Z 	 ddlmZ ddlmZmZmZ ddlZddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ dgZ G d de
j                  Z G d de
j                  Z G d de
j                  Z  G d de
j                  Z!d Z"ed2dee#e#f         de$fd            Z% G d de
j                  Z&d2dZ'd3dZ( e e(d           e(dd !           e(dd"d d#$           e(d           e(dd !           e(dd"d d#$           e(d           e(dd !           e(d           e(d           e(d          d%          Z)ed2d&e&fd'            Z*ed2d&e&fd(            Z+ed2d&e&fd)            Z,ed2d&e&fd*            Z-ed2d&e&fd+            Z.ed2d&e&fd,            Z/ed2d&e&fd-            Z0ed2d&e&fd.            Z1ed2d&e&fd/            Z2ed2d&e&fd0            Z3ed2d&e&fd1            Z4dS )4a   CrossViT Model

@inproceedings{
    chen2021crossvit,
    title={{CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification}},
    author={Chun-Fu (Richard) Chen and Quanfu Fan and Rameswar Panda},
    booktitle={International Conference on Computer Vision (ICCV)},
    year={2021}
}

Paper link: https://arxiv.org/abs/2103.14899
Original code: https://github.com/IBM/CrossViT/blob/main/models/crossvit.py

NOTE: model names have been renamed from originals to represent actual input res all *_224 -> *_240 and *_384 -> *_408

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )partial)ListOptionalTupleNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tupletrunc_normal__assert   )build_model_with_cfg)register_notrace_function)register_modelgenerate_default_cfgs)BlockCrossVitc                   *     e Zd ZdZd	 fd	Zd Z xZS )

PatchEmbedz Image to Patch Embedding
                Fc                    t                                                       t          |          }t          |          }|d         |d         z  |d         |d         z  z  }|| _        || _        || _        |rF|d         dk    rt          j        t          j        ||dz  ddd          t          j	        d	          t          j        |dz  |d
z  ddd          t          j	        d	          t          j        |d
z  |ddd                    | _
        d S |d         dk    rt          j        t          j        ||dz  ddd          t          j	        d	          t          j        |dz  |d
z  dd
d          t          j	        d	          t          j        |d
z  |dd
d                    | _
        d S d S t          j        ||||          | _
        d S )Nr   r            r   )kernel_sizestridepaddingT)inplace   r   )r   r    )super__init__r   img_size
patch_sizenum_patchesnn
SequentialConv2dReLUproj)selfr&   r'   in_chans	embed_dim
multi_convr(   	__class__s          P/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/crossvit.pyr%   zPatchEmbed.__init__0   s   X&&z**
{jm3zRS}8TU $& 	b!}""MIh	QAaYZ[[[GD)))Ii1ni1n!TU_`aaaGD)))Ii1niQqZ[\\\ 			 A"$$MIh	QAaYZ[[[GD)))Ii1ni1n!TU_`aaaGD)))Ii1niQqZ[\\\ 			 %$ 	(I:V`aaaDIII    c                    |j         \  }}}}t          || j        d         k    d| d| d| j        d          d| j        d          d	           t          || j        d         k    d| d| d| j        d          d| j        d          d	           |                     |                              d                              dd          }|S )Nr   zInput image size (*z) doesn't match model (r   z).r#   )shaper   r&   r-   flatten	transpose)r.   xBCHWs         r3   forwardzPatchEmbed.forwardL   s    W
1aT]1%%jQjjjj4=QRCSjjVZVcdeVfjjj	l 	l 	lT]1%%jQjjjj4=QRCSjjVZVcdeVfjjj	l 	l 	lIIaLL  ##--a33r4   )r   r   r   r   F)__name__
__module____qualname____doc__r%   r?   __classcell__r2   s   @r3   r   r   ,   s\         b b b b b b8      r4   r   c                   .     e Zd Z	 	 	 	 d fd	Zd Z xZS )CrossAttention   F        c                    t                                                       || _        ||z  }|dz  | _        t	          j        |||          | _        t	          j        |||          | _        t	          j        |||          | _        t	          j	        |          | _
        t	          j        ||          | _        t	          j	        |          | _        d S )Ng      )bias)r$   r%   	num_headsscaler)   LinearwqwkwvDropout	attn_dropr-   	proj_drop)r.   dimrL   qkv_biasrS   rT   head_dimr2   s          r3   r%   zCrossAttention.__init__X   s     	")#%
)C8444)C8444)C8444I..Ic3''	I..r4   c                 R   |j         \  }}}|                     |d d dddf                                       |d| j        || j        z                                dddd          }|                     |                              ||| j        || j        z                                dddd          }|                     |                              ||| j        || j        z                                dddd          }||                    dd          z  | j        z  }|	                    d          }| 
                    |          }||z                      dd                              |d|          }|                     |          }|                     |          }|S )	Nr   r   .r#   r   rU   )r7   rO   reshaperL   permuterP   rQ   r9   rM   softmaxrS   r-   rT   )	r.   r:   r;   Nr<   qkvattns	            r3   r?   zCrossAttention.forwardm   s   '1aGGAaaa1ckN##++Aq$.!t~BUVV^^_`bcefhijjGGAJJq!T^Q$.5HIIQQRSUVXY[\]]GGAJJq!T^Q$.5HIIQQRSUVXY[\]]AKKB'''4:5|||##~~d##AX  A&&..q!Q77IIaLLNN1r4   )rH   FrI   rI   )r@   rA   rB   r%   r?   rD   rE   s   @r3   rG   rG   W   sZ         / / / / / /*      r4   rG   c                   H     e Zd Zdddddej        ej        f fd	Zd Z xZS )CrossAttentionBlock      @FrI   c
                     t                                                        |	|          | _        t          |||||          | _        |dk    rt          |          nt          j                    | _        d S )N)rL   rV   rS   rT   rI   )	r$   r%   norm1rG   rc   r
   r)   Identity	drop_path)r.   rU   rL   	mlp_ratiorV   rT   rS   rj   	act_layer
norm_layerr2   s             r3   r%   zCrossAttentionBlock.__init__   sw     	Z__
"
 
 
	 1:B),,,BKMMr4   c                     |d d dddf         |                      |                     |                     |                              z   }|S )Nr   r   .)rj   rc   rh   )r.   r:   s     r3   r?   zCrossAttentionBlock.forward   sE    aaa1ckNT^^DIIdjjmm,D,DEEEr4   )	r@   rA   rB   r)   GELU	LayerNormr%   r?   rD   rE   s   @r3   re   re      sm         g|R R R R R R0      r4   re   c                   |     e Zd Zddddej        ej        f fd	Zdeej	                 deej	                 fdZ
 xZS )MultiScaleBlockFrI   c                    t                                                       t          |          }|| _        t	          j                    | _        t          |          D ]}g }t          ||                   D ]D}|                    t          ||         ||         ||         ||||	|         |                     Et          |          dk    r&| j                            t	          j
        |            t          | j                  dk    rd | _        t	          j                    | _        t          |          D ]}||         ||dz   |z           k    r	  |||                    |
            t	          j        ||         ||dz   |z                     g}| j                            t	          j
        |            t	          j                    | _        t          |          D ]}|dz   |z  }||         }|d         dk    rB| j                            t          ||         |||         ||||	d         |                     `g }t          |d                   D ]>}|                    t          ||         |||         ||||	d         |                     ?| j                            t	          j
        |            t	          j                    | _        t          |          D ]}||dz   |z           ||         k    r	  |||dz   |z                      |
            t	          j        ||dz   |z           ||                   g}| j                            t	          j
        |            d S )N)rU   rL   rk   rV   rT   rS   rj   rm   r   r   FrZ   )r$   r%   lennum_branchesr)   
ModuleListblocksrangeappendr   r*   projsri   rN   fusionre   revert_projs)r.   rU   patchesdepthrL   rk   rV   rT   rS   rj   rl   rm   ru   dtmpid_nh_r2   s                      r3   r%   zMultiScaleBlock.__init__   s    	3xx(moo|$$ 	8 	8AC58__ 
 


5A'l'l%'''l)	 	 	 	 	 	 	 3xx1}}""2=##6777t{q  DK]__
|$$ 	3 	3A1va!e|34444 "z#a&))99;;	#a&#qSTuXdNdJe8f8fgJbmS12222moo|$$ 	8 	8Aa%<'B2BRyA~~""'G"$"+A,!)"+"+"+B-#-	 	 	
 
 
 
 uRy)) 
 
AJJ2G"$"+A,!)"+"+"+B-#-	  	  	  	 	 	 	 ""2=##67777MOO|$$ 	: 	:AAE\)*c!f444 "z#q1u&<"=>>		ya!e|%;!<c!fEEG$$R]C%89999	: 	:r4   r:   returnc           	      (   g }t          | j                  D ])\  }}|                     |||                              *t          j                            t          t          j                 g           }t          | j                  D ]6\  }}|                     |||         d d dddf                              7g }t          t          | j
        | j                            D ]\  }\  }}	t          j        ||         ||dz   | j        z           d d dd df         fd          }
 ||
          }
 |	|
d d dddf                   }t          j        |||         d d dd df         fd          }
|                    |
           |S )Nr   r   .r[   )	enumeraterw   ry   torchjitannotater   Tensorrz   zipr{   r|   catru   )r.   r:   outs_br   blockproj_cls_tokenr-   outsr{   revert_projr   reverted_proj_cls_tokens               r3   r?   zMultiScaleBlock.forward   s   !$+.. 	' 	'HAuMM%%!++&&&& ++D,>CC ,, 	@ 	@GAt!!$$vayAaC'=">">???? (1#dk4CT2U2U(V(V 	 	$A$)^A.AAR7R0STUTUTUWXWYWY[^T^0_`fghhhC&++C&1k#aaa1ck2B&C&C#)4fQi122s
6KLRSTTTCKKr4   )r@   rA   rB   r)   ro   rp   r%   r   r   r   r?   rD   rE   s   @r3   rr   rr      s         g|U: U: U: U: U: U:nel+ U\0B        r4   rr   c                 6    d t          | |          D             S )Nc                 B    g | ]\  }}|d          |z  |d         z  |z  S )r   r    ).0r   ps      r3   
<listcomp>z(_compute_num_patches.<locals>.<listcomp>  s2    EEEdaAaDAI!!EEEr4   )r   )r&   r}   s     r3   _compute_num_patchesr     s     EEc(G.D.DEEEEr4   Fss
crop_scalec                    | j         dd         \  }}||d         k    s||d         k    r|r|d         |k    r|d         |k    rwt          t          ||d         z
  dz                      t          t          ||d         z
  dz                      }}| dddd|||d         z   |||d         z   f         } n(t          j        j                            | |dd          } | S )	a~  
    Pulled out of CrossViT.forward_features to bury conditional logic in a leaf node for FX tracing.
    Args:
        x (Tensor): input image
        ss (tuple[int, int]): height and width to scale to
        crop_scale (bool): whether to crop instead of interpolate to achieve the desired scale. Defaults to False
    Returns:
        Tensor: the "scaled" image batch tensor
    rY   Nr   r          @bicubicF)sizemodealign_corners)r7   introundr   r)   
functionalinterpolate)r:   r   r   r=   r>   cucls          r3   scale_imager     s     7233<DAqBqEzzQ"Q%ZZ 	a"Q%1**A!BqE	R/00113ua"Q%i2=M7N7N3O3OB!!!QQQ21:r"r!u*}45AA#//Z_/``AHr4   c                       e Zd ZdZdddddddd	d
dddddddd eej        d          df fd	Zd Ze	j
        j        d             Ze	j
        j        d!d            Ze	j
        j        d"d            Ze	j
        j        dej        fd            Zd#dedee         fdZdee	j                 fdZd!dee	j                 dede	j        fdZd  Z xZS )$r   zI Vision Transformer with support for patch or hybrid CNN input stage
    r   )      ?r   )rH   r   r     )     )r   r   r   r   r   )   r   )r   r   rf   FTrI   gư>)epstokenc                 T    t                                                       |dv sJ  _        | _        t	          |           _        t	          |          } fd|D              _        | _        t           j        |          }t          |           _
         _        t                    x _         _        t          j                     _        t%           j
                  D ]}t'           d| t          j        t+          j        dd||         z   |                                        t'           d| t          j        t+          j        dd|                                        t/           j        |          D ]2\  }}} j                            t3          |||||
                     3t          j        |           _        t          d |D                       }d	 t+          j        d
||          D             }d
}t          j                     _        t=          |          D ]j\  }}t?          |d d                   |d         z   }||||z            }tA          ||||	||||
  
        } ||z  } j                            |            kt          j        fdt%           j
                  D                        _!        t          j        |           _"        t          j        fdt%           j
                  D                        _#        t%           j
                  D ]F}tI          tK           d|           d           tI          tK           d|           d           G &                     j'                   d S )Nr   avgc                 R    g | ]"t          fd j        D                       #S )c                 4    g | ]}t          |z            S r   )r   )r   sjsis     r3   r   z0CrossVit.__init__.<locals>.<listcomp>.<listcomp>H  s#    &L&L&Ls27||&L&L&Lr4   )tupler&   )r   r   r.   s    @r3   r   z%CrossVit.__init__.<locals>.<listcomp>H  s:    bbbRT&L&L&L&Ldm&L&L&L M Mbbbr4   
pos_embed_r   
cls_token_)r&   r'   r/   r0   r1   )r   c                 <    g | ]}t          |d d                   S )rY   N)sumr   r:   s     r3   r   z%CrossVit.__init__.<locals>.<listcomp>a  s&    66613qv;;666r4   c                 6    g | ]}|                                 S r   )itemr   s     r3   r   z%CrossVit.__init__.<locals>.<listcomp>b  s     PPPAqvvxxPPPr4   r   rZ   )rL   rk   rV   rT   rS   rj   rm   c                 2    g | ]} |                   S r   r   )r   r   r0   rm   s     r3   r   z%CrossVit.__init__.<locals>.<listcomp>w  s'    "^"^"^::il#;#;"^"^"^r4   c                 x    g | ]6}d k    rt          j        |                   nt          j                    7S r   )r)   rN   ri   )r   r   r0   num_classess     r3   r   z%CrossVit.__init__.<locals>.<listcomp>y  sM     #/ #/ #/ 5@!OOBIilK000#/ #/ #/r4   {Gz?std)(r$   r%   r   global_poolr   r&   img_size_scaledr   r   rt   ru   r0   r   num_featureshead_hidden_sizer)   rv   patch_embedrx   setattr	Parameterr   zerosr   ry   r   rR   pos_droplinspacerw   r   maxrr   norm	head_dropheadr   getattrapply_init_weights)"r.   r&   	img_scaler'   r/   r   r0   r~   rL   rk   r1   r   rV   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raterm   r   r(   r   im_sr   r   total_depthdprdpr_ptridx	block_cfg
curr_depthdpr_blkr2   s"   `    ``           `              r3   r%   zCrossVit.__init__+  s   , 	.....&&!(++i((	bbbbXabbb$*4+?LL
OO"47	NNBD1=?? t()) 	[ 	[AD*q**BLQKXYNHZ\efg\h9i9i,j,jkkkD*q**BLQ9UV<9X9X,Y,YZZZZd2J	JJ 	 	JD!Q##! %)      
]3336666677PP>;!O!OPPPmoo'.. 	$ 	$NCYss^,,y}<Jww334D!##!((%  C z!GKs####M"^"^"^"^"^U4K\E]E]"^"^"^__	I..M #/ #/ #/ #/ #/4,--#/ #/ #/ 0 0	 t()) 	D 	DA'$(8Q(8(899sCCCC'$(8Q(8(899sCCCCC

4%&&&&&r4   c                    t          |t          j                  rbt          |j        d           t          |t          j                  r.|j        )t          j                            |j        d           d S d S d S t          |t          j                  rLt          j                            |j        d           t          j                            |j        d           d S d S )Nr   r   r   r   )	
isinstancer)   rN   r   weightrK   init	constant_rp   )r.   ms     r3   r   zCrossVit._init_weights  s    a## 	-!(,,,,!RY'' -AF,>!!!&!,,,,,- -,>,>2<(( 	-Gafa(((Gah,,,,,	- 	-r4   c                     t                      }t          | j                  D ]O}|                    d|            t	          | d| d           }||j        r|                    d|            P|S )Nr   r   )setrx   ru   addr   requires_grad)r.   outr   pes       r3   no_weight_decayzCrossVit.no_weight_decay  s    eet()) 	* 	*AGG$$$%%%/A//66B~""2~(Q(()))
r4   c                 (    t          dddg          S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemrw   )dict)r.   coarses     r3   group_matcherzCrossVit.group_matcher  s%    4-/CD
 
 
 	
r4   c                     |r
J d            d S )Nz$gradient checkpointing not supportedr   )r.   enables     r3   set_grad_checkpointingzCrossVit.set_grad_checkpointing  s    AAAAAAAAr4   r   c                     | j         S N)r   )r.   s    r3   get_classifierzCrossVit.get_classifier  s
    yr4   Nr   r   c                       _         ||dv sJ | _        t          j         fdt	           j                  D                        _        d S )Nr   c                     g | ];}d k    r t          j        j        |                   nt          j                    <S r   )r)   rN   r0   ri   )r   r   r   r.   s     r3   r   z-CrossVit.reset_classifier.<locals>.<listcomp>  sP     #
 #
 #
 :EqBIdnQ'555bkmm#
 #
 #
r4   )r   r   r)   rv   rx   ru   r   )r.   r   r   s   `` r3   reset_classifierzCrossVit.reset_classifier  sy    &""22222*DM #
 #
 #
 #
 #
4,--#
 #
 #
  			r4   c                 P  
 |j         d         }g 
t          | j                  D ]\  }}|}| j        |         }t	          ||| j                  } ||          }|dk    r| j        n| j        }|                    |dd          }t          j
        ||fd          }|dk    r| j        n| j        }||z   }|                     |          }
                    |           t          | j                  D ]\  }}	 |	
          

fdt          | j                  D             

S )Nr   rZ   r   r[   c                 8    g | ]\  }} ||                   S r   r   )r   r   r   xss      r3   r   z-CrossVit.forward_features.<locals>.<listcomp>  s)    >>>gadd2a5kk>>>r4   )r7   r   r   r   r   r   cls_token_0cls_token_1expandr   r   pos_embed_0pos_embed_1r   ry   rw   r   )r.   r:   r;   r   r   x_r   
cls_tokens	pos_embedr   r  s             @r3   forward_featureszCrossVit.forward_features  sD   GAJ'(899 	 	NA{B%a(BRT_55BRB-.!VV))9IJ#**1b"55JJ+333B,-FF((8HIiBr""BIIbMMMM,, 	 	FAsRBB ?>>>49)=)=>>>	r4   r  
pre_logitsc                      j         dk    rd D             nd D              fdD             |s%t           j        d         t          j                  r t          j        d D             d          S t          j        t          j        fd	t           j                  D             d          d          S )
Nr   c                 R    g | ]$}|d d dd f                              d          %S )Nr   r[   )meanr   s     r3   r   z)CrossVit.forward_head.<locals>.<listcomp>  s6    ///qa122hmmm""///r4   c                 &    g | ]}|d d df         S )Nr   r   r   s     r3   r   z)CrossVit.forward_head.<locals>.<listcomp>  s,    RgRgRg_`STUVUVUVXYUYSZRgRgRgr4   c                 :    g | ]}                     |          S r   )r   )r   r:   r.   s     r3   r   z)CrossVit.forward_head.<locals>.<listcomp>  s%    ,,,AdnnQ,,,r4   r   c                     g | ]}|S r   r   r   s     r3   r   z)CrossVit.forward_head.<locals>.<listcomp>  s    ___Aa___r4   r   r[   c                 8    g | ]\  }} ||                   S r   r   )r   r   r   r  s      r3   r   z)CrossVit.forward_head.<locals>.<listcomp>  s)    &W&W&Wwq$ttBqE{{&W&W&Wr4   )
r   r   r   r)   ri   r   r   r  stackr   )r.   r  r  s   `` r3   forward_headzCrossVit.forward_head  s    373Cu3L3L//B////RgRgdfRgRgRg,,,,,,, 	5DIaL"+>> 	59_____!4444z%+&W&W&W&W)DIBVBV&W&W&W]^___efggggr4   c                 Z    |                      |          }|                     |          }|S r   )r  r  )r.   r:   r  s      r3   r?   zCrossVit.forward  s-    ""1%%b!!r4   F)Tr   )r@   rA   rB   rC   r   r)   rp   r%   r   r   r   ignorer   r   r   Moduler   r   r   strr  r   r   r  boolr  r?   rD   rE   s   @r3   r   r   '  s        
   3"wr|666)V' V' V' V' V' V'p- - - Y   Y
 
 
 
 YB B B B Y	     C hsm    T%,%7    .h htEL1 ht hPUP\ h h h h      r4   c                 ~    |                     dd           rt          d          d }t          t          | |fd|i|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.c                     i }|                                  D ].}d|v sd|v r|                    dd          }n|}| |         ||<   /|S )Nr  	cls_token.r   )keysreplace)
state_dictnew_state_dictkeynew_keys       r3   pretrained_filter_fnz._create_crossvit.<locals>.pretrained_filter_fn  sf    ??$$ 	6 	6Cc!![C%7%7++c3//&0oN7##r4   r+  )getRuntimeErrorr   r   )variant
pretrainedkwargsr+  s       r3   _create_crossvitr1    sm    zz/4(( [YZZZ      2	
   r4    c                 4    | ddd dt           t          dddd
|S )Nr   )r      r4  g      ?T)zpatch_embed.0.projzpatch_embed.1.proj)zhead.0zhead.1)
urlr   
input_size	pool_sizecrop_pctr  r   fixed_input_size
first_conv
classifierr   )r5  r0  s     r3   _cfgr<    s7    =tY^%.BX\B*   r4   ztimm/)	hf_hub_id)zpatch_embed.0.proj.0zpatch_embed.1.proj.0)r=  r:  )r     r>  r   )r=  r6  r:  r8  )zcrossvit_15_240.in1kzcrossvit_15_dagger_240.in1kzcrossvit_15_dagger_408.in1kzcrossvit_18_240.in1kzcrossvit_18_dagger_240.in1kzcrossvit_18_dagger_408.in1kzcrossvit_9_240.in1kzcrossvit_9_dagger_240.in1kzcrossvit_base_240.in1kzcrossvit_small_240.in1kzcrossvit_tiny_240.in1kr   c           	          t          dddgddgg dg dg dgddgg d	          }t          dd
| dt          |fi |}|S )Nr   g?r   r   `   r   r   r   r   r   r   r   r   r   r'   r0   r~   rL   rk   crossvit_tiny_240r.  r/  r   r   r1  r/  r0  
model_argsmodels       r3   rE  rE    s     b"X"cS\S\S\^g^g^giririrRsa&III/ / /J n%8ZnnSWXbSmSmflSmSmnnELr4   c           	          t          dddgddgg dg dg dgddgg d	          }t          dd
| dt          |fi |}|S )Nr@  r   r   r   r   rB  r   rC  rD  crossvit_small_240rF  r   rG  rH  s       r3   rL  rL    s     b"X#sT]T]T]_h_h_hjsjsjsSta&III/ / /J o%9jooTXYcTnTngmTnTnooELr4   c           	          t          dddgddgg dg dg dgddgg d          }t          dd	| d
t          |fi |}|S )Nr@  r   r   r   r   rB  rC  rD  crossvit_base_240rF  r   rG  rH  s       r3   rN  rN  %  s     b"X#sT]T]T]_h_h_hjsjsjsStr(iii1 1 1J n%8ZnnSWXbSmSmflSmSmnnELr4   c           	          t          dddgddgg dg dg dgddgg d	          }t          dd
| dt          |fi |}|S )Nr@  r   r         r   r   r   r   r   r   r   rD  crossvit_9_240rF  r   rG  rH  s       r3   rT  rT  .  s     b"X#sT]T]T]_h_h_hjsjsjsSta&III/ / /J k%5*kkPTU_PjPjciPjPjkkELr4   c           	          t          dddgddgg dg dg dgddgg d	          }t          dd
| dt          |fi |}|S )Nr@  r   r   r   r   r      r   r   rS  rD  crossvit_15_240rF  r   rG  rH  s       r3   rX  rX  7  s     b"X#sT]T]T]_h_h_hjsjsjsSta&III/ / /J l%6:llQUV`QkQkdjQkQkllELr4   c           
          t          ddddgddgg dg dg dgddgg dd	|}t          dd
| dt          |fi |}|S )Nr@  r   r   r     r   r   r   r   rS  rD  crossvit_18_240rF  r   rG  rH  s       r3   r\  r\  @  s     9"BxC:V_V_V_ajajajlululuUva&III9 9179 9J l%6:llQUV`QkQkdjQkQkllELr4   c           	          t          dddgddgg dg dg dgddgg dd	
          }t          dd| dt          |fi |}|S )Nr@  r   r   rP  rQ  rR  r   rS  Tr   r'   r0   r~   rL   rk   r1   crossvit_9_dagger_240rF  r   rG  rH  s       r3   r_  r_  I  s    "BxC:V_V_V_ajajajlululuUva&III$@ @ @J r%<rrW[\fWqWqjpWqWqrrELr4   c           	          t          dddgddgg dg dg dgddgg dd	
          }t          dd| dt          |fi |}|S )Nr@  r   r   r   r   rV  r   rS  Tr^  crossvit_15_dagger_240rF  r   rG  rH  s       r3   ra  ra  R       b"X#sT]T]T]_h_h_hjsjsjsSta&III$@ @ @J s%=*ssX\]gXrXrkqXrXrssELr4   c           	          t          dddgddgg dg dg dgddgg dd	
          }t          dd| dt          |fi |}|S )Nr   g?r   r   r   r   rV  r   rS  Tr^  crossvit_15_dagger_408rF  r   rG  rH  s       r3   re  re  [  rb  r4   c           	          t          dddgddgg dg dg dgddgg dd	
          }t          dd| dt          |fi |}|S )Nr@  r   r   r   rZ  r[  r   rS  Tr^  crossvit_18_dagger_240rF  r   rG  rH  s       r3   rg  rg  d  rb  r4   c           	          t          dddgddgg dg dg dgddgg dd	
          }t          dd| dt          |fi |}|S )Nrd  r   r   r   rZ  r[  r   rS  Tr^  crossvit_18_dagger_408rF  r   rG  rH  s       r3   ri  ri  m  rb  r4   r  )r2  )5rC   	functoolsr   typingr   r   r   r   	torch.hubtorch.nnr)   	timm.datar   r	   timm.layersr
   r   r   r   _builderr   _features_fxr   	_registryr   r   vision_transformerr   __all__r  r   rG   re   rr   r   r   r  r   r   r1  r<  default_cfgsrE  rL  rN  rT  rX  r\  r_  ra  re  rg  ri  r   r4   r3   <module>rv     sh   ,       ( ( ( ( ( ( ( ( ( (            A A A A A A A A C C C C C C C C C C C C * * * * * * 3 3 3 3 3 3 < < < < < < < < % % % % % %,( ( ( ( ( ( ( (V& & & & &RY & & &R    ")   >l l l l lbi l l l^F F F  uS#X D    (i i i i iry i i iX   .    %$ D7333#'4C$ $ $ $(4 -]hk$ $ $ !D7333#'4C$ $ $ $(4 -]hk$ $ $  4'222"&$C# # # #dW555#tg666"dW5555& &  <  X      h      X      (      8      8            (      (      (      (      r4   