
    NgUW                       d Z ddlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZmZmZmZmZ ddlZddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddlm$Z$m%Z%m&Z&m'Z'm(Z( ddlm)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8m9Z9 g dZ:e G d d                      Z;e G d d                      Z<e G d d                      Z= G d dej>                  Z? G d dej>                  Z@ G d d ej>                  ZA G d! d"ej>                  ZB G d# d$ej>                  ZCdsd&ZD G d' d(ej>                  ZEdsd)ZFd* ZG G d+ d,ej>                  ZH G d- d.ej>                  ZId/eeJ         fd0ZKe3d/eeJ         d1eeJ         fd2            ZLd3eeJ         fd4ZMe3d3eeJ         d1eeJ         fd5            ZNd6e;fd7ZO G d8 d9ej>                  ZP G d: d;ej>                  ZQd/eeJ         fd<ZRe3d/eeJ         d1eeJ         fd=            ZSd3eeJ         fd>ZTe3d3eeJ         d1eeJ         fd?            ZU G d@ dAej>                  ZV G dB dCej>                  ZW G dD dEej>                  ZX G dF dGej>                  ZY G dH dIej>                  ZZd6e;d1eeJeJf         fdJZ[d6e=fdKZ\ G dL dMej>                  Z]	 	 	 	 	 	 	 	 	 	 	 	 dtdWZ^	 	 	 	 	 	 	 	 	 	 	 	 dud[Z_	 	 	 	 	 	 	 	 	 	 	 dvd^Z`d_ Za ebdwi d` e=dwdadbdcdd e_dRdef          dg e=dwdadhdcdd e_dNdRdei          dj e=dwdkdldcdd e^dRdPm          dn e=dwdkdodcdd e^dXdRdPp          dq e=dwdrdodsdd e^dXdtu          dv e=dwdwdodxdd e^dXdtd\y          dz e=dwdkdldcdd e^dXdRdPd{|          d} e=dwdadhdcdd e_dRded]d~          d e=dwdkdldcdd e^dXd]          d e=dwdkdodcdd e^ddRdPd]d~          d e=dwdkdodcdd e^dXd]dV          d e=dwdrdodsdd e^dXdtd\d]          d e=dwdwdodxdd e^dXdtd\d]          d e=dwdadhdcdd e^            d e=dwdadhdcdd e`dUd          d e=dkdbdd          d e=dkdodd          d e=drdodd          d e=dwdodd          d e=dwddd          d e=dddd          d e=dwddddd e_            d e=dwdadddcd e_            d e=dwdadddcd e_            d e=dwdadddcd e_            d e=dwddddd e_d]          d e=dwdadddcd e_d]          d e=dwdadddcd e_d]          d e=dwdkdddcd e_d]d\          d e=dwdkdoddcdd e_d]          d e=dwdadddcdd e`            d e=dwdadddcd e`            d e=dwdkdddd e`            d e=dwdkddddd e`dRdU          d e=dwdrdddsd e`dR          d e=dwdddddd e`dR          d e=dwdaddddRdVdĜ ea            d e=dwdkddddRddĜ ea            d e=dwdkdodddRddĜ ea            d e=dwdrdodddRddĜ ea            d e=dwdwdodddRddĜ ea            Zcdej>        fdʄZddxd˄Zedsd̄Zf e8i d efd%Φ          d efdddҬӦ          d efddլ֦          d efddج֦          d efdЬڦ          d efdЬڦ          d efdЬڦ          d efddddd          d efddeed          d efdddҬӦ          d efd%Φ          d efdd֦          d efdd֦          d efd%Φ          d efd%Φ          d efdddҬӦ          d efdd          i d efdd          d efdd          d efdd          d efd%Φ          d efd%Φ          d efd%Φ          d efd%Φ          d efd%Φ          d efd%Φ          d efd%dd           d efdАddd           d efdАd֦          d efd%dd           d efd%dd           d	 efdАd
dd           d efdАddd           d efdАddd           i d efdАddҬӦ          d efd%dd           d efdЬڦ          d efddddd          d efdd          d efdАddd           d efd%dd           d efdАddd           d efddd           d efdЬڦ          d efddddd          d efd%Φ          d efdd          d  efdee!          d" efddddd          d# efdАd$d%dd          d& efdee!          i d' efddddd          d( efdАd$d%dd          d) efdee!          d* efddddd          d+ efdАd$d%dd          d, efdee!          d- efddddd          d. efdАd$d%dd          d/ efdАd0          d1 efddddd          d2 efdАd$d%dd          d3 efdАd0          d4 efddddd          d5 efdАd$dd6          d7 efdАd0          d8 efddddd          d9 efdАd$d%dd                    Zge9dyd:e]fd;            Zhe9dyd:e]fd<            Zie9dyd:e]fd=            Zje9dyd:e]fd>            Zke9dyd:e]fd?            Zle9dyd:e]fd@            Zme9dyd:e]fdA            Zne9dyd:e]fdB            Zoe9dyd:e]fdC            Zpe9dyd:e]fdD            Zqe9dyd:e]fdE            Zre9dyd:e]fdF            Zse9dyd:e]fdG            Zte9dyd:e]fdH            Zue9dyd:e]fdI            Zve9dyd:e]fdJ            Zwe9dyd:e]fdK            Zxe9dyd:e]fdL            Zye9dyd:e]fdM            Zze9dyd:e]fdN            Z{e9dyd:e]fdO            Z|e9dyd:e]fdP            Z}e9dyd:e]fdQ            Z~e9dyd:e]fdR            Ze9dyd:e]fdS            Ze9dyd:e]fdT            Ze9dyd:e]fdU            Ze9dyd:e]fdV            Ze9dyd:e]fdW            Ze9dyd:e]fdX            Ze9dyd:e]fdY            Ze9dyd:e]fdZ            Ze9dyd:e]fd[            Ze9dyd:e]fd\            Ze9dyd:e]fd]            Ze9dyd:e]fd^            Ze9dyd:e]fd_            Ze9dyd:e]fd`            Ze9dyd:e]fda            Ze9dyd:e]fdb            Ze9dyd:e]fdc            Ze9dyd:e]fdd            Ze9dyd:e]fde            Ze9dyd:e]fdf            Ze9dyd:e]fdg            Ze9dyd:e]fdh            Ze9dyd:e]fdi            Ze9dyd:e]fdj            Ze9dyd:e]fdk            Ze9dyd:e]fdl            Ze9dyd:e]fdm            Ze9dyd:e]fdn            Ze9dyd:e]fdo            Ze9dyd:e]fdp            Ze9dyd:e]fdq            Ze9dyd:e]fdr            ZdS (z  a   MaxVit and CoAtNet Vision Transformer - CNN Hybrids in PyTorch

This is a from-scratch implementation of both CoAtNet and MaxVit in PyTorch.

99% of the implementation was done from papers, however last minute some adjustments were made
based on the (as yet unfinished?) public code release https://github.com/google-research/maxvit

There are multiple sets of models defined for both architectures. Typically, names with a
 `_rw` suffix are my own original configs prior to referencing https://github.com/google-research/maxvit.
These configs work well and appear to be a bit faster / lower resource than the paper.

The models without extra prefix / suffix' (coatnet_0_224, maxvit_tiny_224, etc), are intended to
match paper, BUT, without any official pretrained weights it's difficult to confirm a 100% match.

Papers:

MaxViT: Multi-Axis Vision Transformer - https://arxiv.org/abs/2204.01697
@article{tu2022maxvit,
  title={MaxViT: Multi-Axis Vision Transformer},
  author={Tu, Zhengzhong and Talebi, Hossein and Zhang, Han and Yang, Feng and Milanfar, Peyman and Bovik, Alan and Li, Yinxiao},
  journal={ECCV},
  year={2022},
}

CoAtNet: Marrying Convolution and Attention for All Data Sizes - https://arxiv.org/abs/2106.04803
@article{DBLP:journals/corr/abs-2106-04803,
  author    = {Zihang Dai and Hanxiao Liu and Quoc V. Le and Mingxing Tan},
  title     = {CoAtNet: Marrying Convolution and Attention for All Data Sizes},
  journal   = {CoRR},
  volume    = {abs/2106.04803},
  year      = {2021}
}

Hacked together by / Copyright 2022, Ross Wightman
    N)OrderedDict)	dataclassreplacefield)partial)CallableOptionalUnionTupleList)nn)Final)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpConvMlpDropPath	LayerNormClassifierHeadNormMlpClassifierHead)create_attnget_act_layerget_norm_layerget_norm_act_layercreate_conv2dcreate_pool2d)trunc_normal_tf_	to_2tupleextend_tuplemake_divisible_assert)	RelPosMlp
RelPosBiasRelPosBiasTfuse_fused_attnresize_rel_pos_bias_table   )build_model_with_cfg)feature_take_indices)register_notrace_function)named_applycheckpoint_seq)generate_default_cfgsregister_model)
MaxxVitCfgMaxxVitConvCfgMaxxVitTransformerCfgMaxxVitc                      e Zd ZU dZeed<   dZeed<   dZe	ed<   dZ
eed<   dZeed<   dZeed	<   d
Ze	ed<   d
Ze	ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeef                  ed<   dZeeeef                  ed<   dZeed<   dZeed<   dZee	         ed<   dZeed<   dZeed<   dZeed <   d!Ze	ed"<   d# ZdS )$r1       dim_headT
head_first      @expand_ratioexpand_firstshortcut_bias	attn_bias        	attn_drop	proj_dropavg2	pool_typebiasrel_pos_type   rel_pos_dimpartition_ratioNwindow_size	grid_sizeFno_block_attnuse_nchw_attninit_valuesgelu	act_layerlayernorm2d
norm_layer	layernormnorm_layer_clư>norm_epsc                     | j         t          | j                   | _         | j        .t          | j                  | _        | j         | j        | _         d S d S d S N)rG   r   rF   selfs    O/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/maxxvit.py__post_init__z#MaxxVitTransformerCfg.__post_init__U   s[    >%&t~66DN'()9::D~%!%!1 ('%%    ) __name__
__module____qualname__r5   int__annotations__r6   boolr8   floatr9   r:   r;   r=   r>   r@   strrB   rD   rE   rF   r	   r   rG   rH   rI   rJ   rL   rN   rP   rR   rX    rY   rW   r1   r1   =   s        HcJL%L$M4ItIuIuIsL#KOS-1K%S/*111+/Ixc3h(///M4M4#'K%'''Is#J###$M3$$$He2 2 2 2 2rY   r1   c                   F   e Zd ZU dZeed<   dZeed<   dZe	ed<   dZ
eed<   d	Zeed
<   dZe	ed<   dZe	ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZe	ed<   dZeed<   dZeed<   dZeed<   dZee         ed<   dZeed<   dZeed <   dZeed!<   d"Zee         ed#<   d$ Zd"S )%r0   mbconv
block_typer7   r8   Texpand_output   kernel_sizer'   
group_sizeFpre_norm_actoutput_biasdwstride_moder?   r@   downsample_pool_type padding
attn_earlyse
attn_layersiluattn_act_layer      ?
attn_ratiorQ   rJ   rK   rL   rN   rP   NrR   c                     | j         dv sJ | j         dk    }| j        s|rdnd| _        | j        s	|sd| _        | j        |rdnd| _        | j        p| j        | _        d S )N)rd   convnextrd   batchnorm2drM   rO   h㈵>rQ   )re   rN   rP   rR   rn   r@   )rV   
use_mbconvs     rW   rX   zMaxxVitConvCfg.__post_init__u   s    "88888_0
 	M/9Lmm}DO! 	-* 	-!,D= $.8DDDDM$($=$O!!!rY   )rZ   r[   r\   re   ra   r^   r8   r`   rf   r_   rh   r]   ri   rj   rk   rm   r@   rn   rp   rq   rs   ru   rw   rJ   r	   rL   rN   rP   rR   rX   rb   rY   rW   r0   r0   ^   ss        JL%M4KJL$KKIs &#&&&GSJJ NC   J#'K%'''IsJM3 $Hhuo$$$
P 
P 
P 
P 
PrY   r0   c                   &   e Zd ZU dZeedf         ed<   dZeedf         ed<   dZee	e
ee
df         f         df         ed<   dZe	eeeef         f         ed	<   d
Zeed<    ee          Zeed<    ee          Zeed<   dZeed<   dZe
ed<   dS )r/   `           .	embed_dim   rg      r   depths)Cr   Tr   re   @   
stem_widthF	stem_bias)default_factoryconv_cfgtransformer_cfgNhead_hidden_sizevit_effweight_init)rZ   r[   r\   r   r   r]   r^   r   re   r
   ra   r   r   r_   r   r0   r   r1   r   r   r   rb   rY   rW   r/   r/      s         !4IuS#X444*FE#s(O***:NJeCsCx01367NNN.0Jc5c?*+000It$u^DDDHnDDD-2UCX-Y-Y-YO*YYY c    K     rY   r/   c                        e Zd ZU ee         ed<   	 	 	 	 	 	 	 	 	 ddedee         ded	ed
ededede	de	f fdZ
ddeej                 fdZ xZS )Attention2d
fused_attnNr4   Tr<   dimdim_outr5   rA   r9   r6   rel_pos_clsr=   r>   c
                    t                                                       |p|}|r|n|}
|
|z  | _        || _        || _        |dz  | _        t                      | _        t          j	        ||
dz  d|          | _
        |r || j                  nd | _        t          j        |          | _        t          j	        |
|d|          | _        t          j        |	          | _        d S )N      rg   r'   rA   	num_heads)super__init__r   r5   r6   scaler%   r   r   Conv2dqkvrel_posDropoutr=   projr>   rV   r   r   r5   rA   r9   r6   r   r=   r>   dim_attn	__class__s              rW   r   zAttention2d.__init__   s     	.S*377!X- $%
(**9S(Q,===@KU{{T^<<<<QUI..Ih>>>	I..rY   shared_rel_posc                    |j         \  }}}}| j        rR|                     |                              || j        | j        dz  d                              dd          \  }}}	nM|                     |                              |d| j        | j        d                              d          \  }}}	| j	        r d }
| j
        | j
                                        }
n||}
t          j        j                            |                    dd                                          |                    dd                                          |	                    dd                                          |
| j        r| j        j        nd                              dd                              |d||          }n|| j        z  }|                    dd          |z  }| j
        | 
                    |          }n|||z   }|                    d          }|                     |          }|	|                    dd          z                      |d||          }|                     |          }|                     |          }|S )	Nrg   r   r   r'   r<   	attn_mask	dropout_p)shaper6   r   viewr   r5   chunkreshapeunbindr   r   get_biastorchr   
functionalscaled_dot_product_attention	transpose
contiguoustrainingr=   pr   softmaxr   r>   )rV   xr   Br   HWqkvr;   attns               rW   forwardzAttention2d.forward   sU   W
1a? 	]hhqkk&&q$.$-!:KRPPVVWX^_V``GAq!!hhqkk))!QrRRYYZ[\\GAq!? 	?I|' L1133		+*	#@@B##..00B##..00B##..00#.2mC$.** A   iB2q! 4 4 A DJA;;r2&&*D|'||D))+n,<<B<''D>>$''DT^^B+++11!RA>>AIIaLLNN1rY   Nr4   TTTNr<   r<   rT   )rZ   r[   r\   r   r_   r^   r]   r	   r   r`   r   r   Tensorr   __classcell__r   s   @rW   r   r      s         d3 &*!%#$(!!/ // c]/ 	/
 / / / "/ / / / / / / /6# #%,)? # # # # # # # #rY   r   c                        e Zd ZU dZee         ed<   	 	 	 	 	 	 	 	 ddedee         d	ed
ededede	de
de
f fdZddeej                 fdZ xZS )AttentionClz0 Channels-last multi-head attention (B, ..., C) r   Nr4   Tr<   r   r   r5   rA   r9   r6   r   r=   r>   c
                    t                                                       |p|}|r||k    r|n|}
|
|z  dk    s
J d            |
|z  | _        || _        || _        |dz  | _        t                      | _        t          j	        ||
dz  |          | _
        |r || j                  nd | _        t          j        |          | _        t          j	        |
||          | _        t          j        |	          | _        d S )Nr   z(attn dim should be divisible by head_dimr   rg   r   r   )r   r   r   r5   r6   r   r%   r   r   Linearr   r   r   r=   r   r>   r   s              rW   r   zAttentionCl.__init__   s    	.S*Ew}}77#("a''')S'''!X- $%
(**9S(Q,T:::@KU{{T^<<<<QUI..Ihd;;;	I..rY   r   c                 N   |j         d         }|j         d d         }| j        rf|                     |                              |d| j        | j        dz                                dd                              dd          \  }}}na|                     |                              |dd| j        | j                                      dd          	                    d          \  }}}| j
        rdd }| j        | j                                        }n||}t          j        j                            ||||| j        r| j        j        nd          }ny|| j        z  }||                    d	d          z  }	| j        |                     |	|
          }	n||	|z   }	|	                    d          }	|                     |	          }	|	|z  }|                    dd                              |dz             }|                     |          }|                     |          }|S )Nr   r   rg   r'   r   r   r<   r   r   r   )r   )r   r6   r   r   r   r5   r   r   r   r   r   r   r   r   r   r   r   r   r=   r   r   r   r   r>   )
rV   r   r   r   restore_shaper   r   r   r;   r   s
             rW   r   zAttentionCl.forward   s   GAJ? 	mhhqkk&&q"dndma>OPPZZ[\^_``ffghnofppGAq!!hhqkk))!RDNDMRR\\]^`abbiijkllGAq!? 	I|' L1133		+*	#@@1a#.2mC$.** A  AA DJAq{{2r***D|'||D|HH+n,<<B<''D>>$''DqAKK1%%me&;<<IIaLLNN1rY   r   rT   )rZ   r[   r\   __doc__r   r_   r^   r]   r	   r   r`   r   r   r   r   r   r   s   @rW   r   r      s         ::d
 &*!%#$(!!/ // c]/ 	/
 / / / "/ / / / / / / /8# #%,)? # # # # # # # #rY   r   c                   &     e Zd Zd fd	Zd Z xZS )
LayerScaler{   Fc                     t                                                       || _        t          j        |t          j        |          z            | _        d S rT   r   r   inplacer   	Parameterr   onesgammarV   r   rJ   r   r   s       rW   r   zLayerScale.__init__  B    \+
3"?@@


rY   c                 R    | j         }| j        r|                    |          n||z  S rT   )r   r   mul_rV   r   r   s      rW   r   zLayerScale.forward   s(    
 $;qvve}}}!e);rY   r{   FrZ   r[   r\   r   r   r   r   s   @rW   r   r     R        A A A A A A
< < < < < < <rY   r   c                   &     e Zd Zd fd	Zd Z xZS )LayerScale2dr{   Fc                     t                                                       || _        t          j        |t          j        |          z            | _        d S rT   r   r   s       rW   r   zLayerScale2d.__init__&  r   rY   c                 ~    | j                             dddd          }| j        r|                    |          n||z  S )Nr'   r   )r   r   r   r   r   s      rW   r   zLayerScale2d.forward+  s:    
2q!,, $;qvve}}}!e);rY   r   r   r   s   @rW   r   r   %  r   rY   r   c                   F     e Zd ZdZ	 	 	 ddedededed	ef
 fd
Zd Z xZ	S )Downsample2da4   A downsample pooling module supporting several maxpool and avgpool modes
    * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1
    * 'max2' - MaxPool2d w/ kernel_size = stride = 2
    * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1
    * 'avg2' - AvgPool2d w/ kernel_size = stride = 2
    r?   ro   Tr   r   r@   rp   rA   c                    t                                                       |dv sJ |dk    rt          ddd|pd          | _        n[|dk    rt          dd|pd	          | _        n;|d
k    rt          d
ddd|pd          | _        nt          d
d|pd	          | _        ||k    rt	          j        ||d|          | _        d S t	          j                    | _        d S )N)maxmax2avgr?   r   rg   r   r'   )rh   striderp   r   r   )rp   r   F)rh   r   count_include_padrp   r   )r   r   r   poolr   r   expandIdentity)rV   r   r   r@   rp   rA   r   s         rW   r   zDownsample2d.__init__8  s
    	:::::%e1glYZ[[[DII&  %eQ1EEEDII%%1Q%QXQ]\]_ _ _DII &eQ1EEEDI'>>)C!$???DKKK+--DKKKrY   c                 Z    |                      |          }|                     |          }|S rT   )r   r   rV   r   s     rW   r   zDownsample2d.forwardQ  s%    IIaLLKKNNrY   )r?   ro   T)
rZ   r[   r\   r   r]   ra   r_   r   r   r   r   s   @rW   r   r   0  s          $( (( ( 	(
 ( ( ( ( ( ( (2      rY   r   ro   c                 P   t          | t          j        t          j        f          r||dk    rUt          j                            | j        d           | j        &t          j                            | j                   d S d S |dk    rEt          | j        d           | j        &t          j                            | j                   d S d S |dk    rSt          j        
                    | j                   | j        &t          j                            | j                   d S d S t          j                            | j                   | j        Td|v r(t          j                            | j        d           d S t          j                            | j                   d S d S d S )Nnormal{Gz?stdtrunc_normalxavier_normalmlprQ   )
isinstancer   r   r   initnormal_weightrA   zeros_r   xavier_normal_xavier_uniform_)modulenameschemes      rW   _init_transformerr  W  s   &29bi011 0XGOOFMsO333{&v{+++++ '&~%%V]4444{&v{+++++ '&&&G""6=111{&v{+++++ '& G##FM222{&D==GOOFKTO:::::GNN6;/////)0 0  '&rY   c                        e Zd ZdZdd e            dfdedededed	ed
ef fdZddZ	dde
ej                 fdZ xZS )TransformerBlock2daX   Transformer block with 2D downsampling
    '2D' NCHW tensor layout

    Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW
    for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs.

    This impl was faster on TPU w/ PT XLA than the 1D experiment.
    r'   Nr<   r   r   r   r   cfg	drop_pathc                 |   t                                                       t          t          |j                  |j                  }t          |j                  }|dk    rmt          |||j	        |j
                  | _        t          j        t          d ||          fdt          |||j	                  fg                    | _        n0||k    sJ t          j                    | _         ||          | _        t#          |||j        |j        |j        ||j        |j                  | _        |j        rt3          ||j                  nt          j                    | _        |d	k    rt7          |          nt          j                    | _         ||          | _        t=          |t?          ||j         z            ||j        
          | _!        |j        rt3          ||j                  nt          j                    | _"        |d	k    rt7          |          nt          j                    | _#        d S )Nepsr   )r@   rA   normdownr@   )r5   r9   rA   r   r=   r>   rJ   r<   in_featureshidden_featuresrL   drop)$r   r   r   r   rN   rR   r   rL   r   r@   r:   shortcutr   
Sequentialr   norm1r   r   r5   r9   r;   r=   r>   r   rJ   r   ls1r   
drop_path1norm2r   r]   r8   r   ls2
drop_path2)
rV   r   r   r   r   r
  r  rN   rL   r   s
            rW   r   zTransformerBlock2d.__init__y  s    	^CN;;NNN
!#-00	Q;;(gUXUfgggDM{C)c3#-HHHI4 ( (  DJJ
 '>>>>KMMDM#CDJ\)#mm	
 	
 	
	 JMk<S_EEEE^`^i^k^k1:R(9---R[]]Z((
#*: :;;	     
 JMk<S_EEEE^`^i^k^k1:R(9---R[]]rY   ro   c                 N    t          t          t          |          |            d S Nr  )r+   r   r  rV   r  s     rW   init_weightszTransformerBlock2d.init_weights  s&    G-f===tDDDDDrY   r   c           
      t   |                      |          |                     |                     |                     |                     |          |                              z   }||                     |                     |                     |                     |                                        z   }|S )Nr   )	r  r  r  r   r  r  r  r   r  )rV   r   r   s      rW   r   zTransformerBlock2d.forward  s    MM!ttxx		$**Q--`n	8o8o/p/pqqq$**Q--)@)@ A ABBBrY   ro   rT   )rZ   r[   r\   r   r1   r]   r   r`   r   r#  r	   r   r   r   r   r   s   @rW   r	  r	  o  s          $()>)>)@)@!,S ,S,S ,S 	,S
 ",S ',S ,S ,S ,S ,S ,S ,S\E E E E %,)?        rY   r	  c                 h   t          | t          j                  r|dk    rUt          j                            | j        d           | j        &t          j                            | j                   d S d S |dk    rEt          | j        d           | j        &t          j                            | j                   d S d S |dk    rSt          j        	                    | j                   | j        &t          j                            | j                   d S d S | j
        d         | j
        d         z  | j        z  }|| j        z  }t          j                            | j        dt          j        d|z                       | j        (t          j                            | j                   d S d S d S )	Nr   r   r   r   r   r   r'   g       @)r   r   r   r   r   r   rA   r  r   r  rh   out_channelsgroupsmathsqrt)r  r  r  fan_outs       rW   
_init_convr,    s   &")$$ ,XGOOFMsO333{&v{+++++ '&~%%V]4444{&v{+++++ '&&&G""6=111{&v{+++++ '& (+f.@.CCfFYYG%GGOOFM1dig.F.FGGG{&v{+++++', ,$ '&rY   c                 *    | sdS || z  dk    sJ || z  S )Nr'   r   rb   )ri   channelss     rW   
num_groupsr/    s2     &q *$)))):%%rY   c                   t     e Zd ZdZdd e            dfdedededeeef         d	ed
ef fdZddZ	d Z
 xZS )MbConvBlockzL Pre-Norm Conv Block - 1x1 - kxk - 1x1, w/ inverted bottleneck (expand)
    r'   r'   r'   r<   in_chsout_chsr   dilationr
  r  c           	         t          t          |                                            t          t	          |j        |j                  |j                  }t          |j	        r|n||j
        z            }t          |j        |          }	|dk    r)t          |||j        |j        |j                  | _        nt%          j                    | _        |j        dv sJ d\  }
}}|j        dk    r||d         }}
n |j        dk    r||d         }}n
||d	         }} |||j        
          | _        |
dk    r#t          |||j        |j                  | _        nt%          j                    | _        t3          ||d|          | _         ||          | _        t3          |||j        |||	|j                  | _        i }t=          |j        t@                    rC|j        dk    s|j        dk    r-|j!        |d<   tE          |j#        |j	        r|n|z            |d<   |j$        r0tK          |j        |fi || _&         ||          | _'        d | _(        n/d | _&         ||          | _'        tK          |j        |fi || _(        t3          ||d|j                  | _)        |dk    rtU          |          nt%          j                    | _+        d S )Nr  r   )r@   rA   rp   )r   1x1rl   )r'   r'   r'   r   r'   r7  r   )	apply_act)r@   rp   )r   )r   r5  r(  rp   rr   ecarL   rd_channelsr   r<   ),r   r1  r   r   r   rN   rL   rR   r    rf   r8   r/  ri   r   r@   rk   rp   r  r   r   rm   rj   pre_normrn   r  r   	conv1_1x1r  rh   	conv2_kxkr   rs   ra   ru   r]   rw   rq   r   se_earlyr  rr   	conv3_1x1r   r  )rV   r3  r4  r   r5  r
  r  norm_act_layermid_chsr(  stride_poolstride_1stride_2
dilation_2attn_kwargsr   s                  rW   r   zMbConvBlock.__init__  s    	k4  ))+++ !3CNCM!R!RX[Xdeee S->"J''FcN^!^__CNG44Q;;(3=sX[Xce e eDMM KMMDM"77777*1'Xx?f$$&,hqkKK_%%#)8A;jHH#)8A;jH&v9IJJJ??$VVs?WadalmmmDIIDI&vw(KKK#^G,,
&WcojV V V cnc** 	o~%%5)@)@+.+=K(-0cN_Cl77el1m-n-nM* > 	J'OO;OODM'00DJDGG DM'00DJ!#.'II[IIDG&wQQQ09B),,,BKMMrY   ro   c                 N    t          t          t          |          |            d S r   r+   r   r,  r"  s     rW   r#  zMbConvBlock.init_weights  %    GJv666=====rY   c                    |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        |                     |          }|                     |          }| j        |                     |          }| 	                    |          }| 
                    |          |z   }|S rT   )r  r;  r  r<  r  r=  r>  r  rr   r?  r  rV   r   r  s      rW   r   zMbConvBlock.forward  s    ==##MM!IIaLL NN1JJqMM NN1=$a  AJJqMM7

A NN1NN1(rY   r%  )rZ   r[   r\   r   r0   r]   r   r`   r   r#  r   r   r   s   @rW   r1  r1    s          (."0."2"2!=R =R=R =R 	=R
 CHo=R  =R =R =R =R =R =R =R~> > > >      rY   r1  c                        e Zd ZdZdddd e            ddfded	ee         d
ededeeef         dedede	f fdZ
d Z xZS )ConvNeXtBlockz ConvNeXt Block
    N   r'   r2  Tr<   r3  r4  rh   r   r5  r
  conv_mlpr  c	           	         t                                                       |p|}t          |j                  }	|r0t	          t          |j                  |j                  }
t          }nd|j        v sJ t          }
t          }|| _        |dk    rt          ||          | _        nA||k    r#t          j        ||d|j                  | _        nt          j                    | _        |j        dv sJ d\  }}|j        dk    r|}n|}|dk    rt          |||j        	          | _        nt          j                    | _        t+          |||||d         d
|j                  | _         |
|          | _         ||t1          |j        |z            |j        |	          | _        |r5|j        rt9          ||j                  nt          j                    | _        n4|j        rt=          ||j                  nt          j                    | _        |dk    rt?          |          nt          j                    | _         d S )Nr  rO   r   r'   )rh   rA   )r   rl   r2  r   r  T)rh   r   r5  	depthwiserA   )rA   rL   r<   )!r   r   r   rL   r   r   rN   rR   r   r   r   use_conv_mlpr   r  r   r   rk   r   rm   rn   r  r   conv_dwr  r]   r8   r   rJ   r   lsr   r   r  )rV   r3  r4  rh   r   r5  r
  rO  r  rL   rN   	mlp_layerrB  	stride_dwr   s                 rW   r   zConvNeXtBlock.__init__0  s3    	#V!#-00	 	 !?!?S\RRRJII#.0000"JI$Q;;(99DMMwIfg13?[[[DMMKMMDM.0000!%Y?f$$ KKI!$VVs?WXXXDIIDI$GYQYZ[Q\2 2 2 Jw''	9Wc#*:W*D&E&ECOgpqqq 	a@Cbl7CO<<<UWU`UbUbDGG>Ao`j#/:::SUS^S`S`DG09B),,,BKMMrY   c                     |                      |          }|                     |          }|                     |          }| j        r@|                     |          }|                     |          }|                     |          }no|                    dddd          }|                     |          }|                     |          }|                     |          }|                    dddd          }|                     |          |z   }|S Nr   r   rg   r'   )	r  r  rS  rR  r  r   rT  permuter  rK  s      rW   r   zConvNeXtBlock.forwardf  s    ==##IIaLLLLOO 		&		!AA

AA		!Q1%%A		!AA

A		!Q1%%ANN1(rY   )rZ   r[   r\   r   r0   r]   r	   r   r_   r`   r   r   r   r   s   @rW   rM  rM  ,  s          &* (."0."2"2!!4R 4R4R c]4R 	4R
 4R CHo4R  4R 4R 4R 4R 4R 4R 4R 4Rl      rY   rM  rF   c                    | j         \  }}}}t          ||d         z  dk    d| d|d          d           t          ||d         z  dk    d| d|d          d           |                     |||d         z  |d         ||d         z  |d         |          } |                     ddddd	d
                                                              d|d         |d         |          }|S )Nr   height () must be divisible by window ()r'   width (rg   r      r   r   r   r!   r   rY  r   )r   rF   r   r   r   r   windowss          rW   window_partitionrb  y  s   JAq!QAA!#%c%c%cR]^_R`%c%c%cdddAA!#%bq%b%bQ\]^Q_%b%b%bccc	q!{1~%{1~qKN7JKXYN\]^^Aii1aAq))4466;;BAP[\]P^`abbGNrY   img_sizec                    |\  }}| j         d         }|                     d||d         z  ||d         z  |d         |d         |          }|                    dddddd                                                              d|||          }|S )Nr   r   r'   rg   r   r_  r   r   r   rY  r   ra  rF   rc  r   r   r   r   s          rW   window_reverserg    s    DAqbARk!n,a;q>.A;q>S^_`SacdeeA			!Q1a##..0055b!QBBAHrY   rG   c           	         | j         \  }}}}t          ||d         z  dk    d| d|d                     t          ||d         z  dk    d| d|d                     |                     ||d         ||d         z  |d         ||d         z  |          } |                     dddddd	                                                              d
|d         |d         |          }|S )Nr   height  must be divisible by grid r'   width r   r_  rg   r   r   r`  )r   rG   r   r   r   r   ra  s          rW   grid_partitionrl    s    JAq!QA	!!#YQ#Y#Y9UV<#Y#YZZZA	!!#XA#X#X)TU,#X#XYYY	q)A,Yq\ 19Q<iPQlARTUVVAii1aAq))4466;;B	!iXYl\]^^GNrY   c                    |\  }}| j         d         }|                     d||d         z  ||d         z  |d         |d         |          }|                    dddddd                                                              d|||          }|S )Nr   r   r'   rg   r_  r   r   re  ra  rG   rc  r   r   r   r   s          rW   grid_reversero    s    DAqbARil*A1,=y|YWX\[\]]A			!Q1a##..0055b!QBBAHrY   r
  c                     d }| j         dk    rt          t          || j                  }nC| j         dk    rt          t          |          }n!| j         dk    rt          t
          |          }|S )Nr   )rF   
hidden_dimrA   )rF   bias_tf)rB   r   r"   rD   r#   r$   )r
  rF   r   s      rW   get_rel_pos_clsrs    su    K
5  i[S_]]]		V	#	#jkBBB		Y	&	&lDDDrY   c            	       X     e Zd ZdZd e            dfdedededef fdZd	 Z	d
 Z
 xZS )PartitionAttentionClzR Grid or Block partition + Attn + FFN.
    NxC 'channels last' tensor layout.
    blockr<   r   partition_typer
  r  c           
         t                                                       t          t          |j                  |j                  }t          |j                  }|dk    | _        t          | j        r|j
        n|j                  | _        t          || j                  } ||          | _        t          |||j        |j        |j        ||j        |j                  | _        |j        rt/          ||j                  nt1          j                    | _        |dk    rt7          |          nt1          j                    | _         ||          | _        t=          |t?          ||j         z            ||j                  | _!        |j        rt/          ||j                  nt1          j                    | _"        |dk    rt7          |          nt1          j                    | _#        d S Nr  rv  r5   rA   r6   r   r=   r>   r  r<   r  )$r   r   r   r   rP   rR   r   rL   partition_blockr   rF   rG   partition_sizers  r  r   r5   r;   r6   r=   r>   r   rJ   r   r   r   r  r   r  r  r   r]   r8   r   r  r  	rV   r   rw  r
  r  rN   rL   r   r   s	           rW   r   zPartitionAttentionCl.__init__  s    	^C,=>>CLQQQ
!#-00	-8'4;O(bUXUbcc%c4+>??Z__
\~#mm	
 	
 	
	 DG?e:cs????XZXcXeXe1:R(9---R[]]Z__
c&6 677	     
 DG?e:cs????XZXcXeXe1:R(9---R[]]rY   c                    |j         dd         }| j        rt          || j                  }nt	          || j                  }|                     |          }| j        rt          || j        |          }nt          || j        |          }|S )Nr'   rg   )r   r{  rb  r|  rl  r   rg  ro  rV   r   rc  partitioneds       rW   _partition_attnz$PartitionAttentionCl._partition_attn  s    71Q3< 	A*1d.ABBKK(D,?@@Kii,, 	I{D,?JJAA[$*=xHHArY   c           
      J   ||                      |                     |                     |                     |                                        z   }||                     |                     |                     |                     |                                        z   }|S rT   r  r  r  r  r  r  r   r  r   s     rW   r   zPartitionAttentionCl.forward  y    )=)=djjmm)L)L M MNNN$**Q--)@)@ A ABBBrY   rZ   r[   r\   r   r1   r]   ra   r`   r   r  r   r   r   s   @rW   ru  ru    s          #*)>)>)@)@!$S $S$S  $S '	$S
 $S $S $S $S $S $SL        rY   ru  c                   R     e Zd ZdZ e            dfdededef fdZd Zd Z	 xZ
S )	ParallelPartitionAttentionzP Experimental. Grid and Block partition + single FFN
    NxC tensor layout.
    r<   r   r
  r  c           
      d   t                                                       |dz  dk    sJ t          t          |j                  |j                  }t          |j                  }|j        |j	        k    sJ t          |j                  | _        t          || j                  } ||          | _        t          ||dz  |j        |j        |j        ||j        |j                  | _        t          ||dz  |j        |j        |j        ||j        |j                  | _        |j        rt/          ||j                  nt1          j                    | _        |dk    rt7          |          nt1          j                    | _         ||          | _        t=          |t?          ||j         z            |||j                  | _!        |j        rt/          ||j                  nt1          j                    | _"        |dk    rt7          |          nt1          j                    | _#        d S )Nr   r   r  rz  r  r<   )r  r  out_featuresrL   r  )$r   r   r   r   rP   rR   r   rL   rF   rG   r   r|  rs  r  r   r5   r;   r6   r=   r>   
attn_block	attn_gridrJ   r   r   r   r  r   r  r  r   r]   r8   r   r  r  )rV   r   r
  r  rN   rL   r   r   s          rW   r   z#ParallelPartitionAttention.__init__  s
    	Qw!||||^C,=>>CLQQQ
!#-00	#-////'88%c4+>??Z__
%1H\~#mm	
 	
 	
 %1H\~#mm	
 	
 	
 DG?e:cs????XZXcXeXe1:R(9---R[]]Z__
c&6 677      DG?e:cs????XZXcXeXe1:R(9---R[]]rY   c                 P   |j         dd         }t          || j                  }|                     |          }t	          || j        |          }t          || j                  }|                     |          }t          || j        |          }t          j	        ||gd          S )Nr'   rg   r   r   )
r   rb  r|  r  rg  rl  r  ro  r   cat)rV   r   rc  partitioned_blockx_windowpartitioned_gridx_grids          rW   r  z*ParallelPartitionAttention._partition_attn  s    71Q3<,Q0CDD OO,=>>!"3T5H(SS)!T-@AA>>*:;;.0CXNNy(F+4444rY   c           
      J   ||                      |                     |                     |                     |                                        z   }||                     |                     |                     |                     |                                        z   }|S rT   r  r   s     rW   r   z"ParallelPartitionAttention.forward+  r  rY   )rZ   r[   r\   r   r1   r]   r`   r   r  r   r   r   s   @rW   r  r    s          *?)>)@)@!	/S /S/S '/S 	/S /S /S /S /S /Sb5 5 5      rY   r  c           	         | j         \  }}}}t          ||d         z  dk    d| d|d          d           t          ||d         z  dk    d| d|d          d           |                     ||||d         z  |d         ||d         z  |d                   } |                     ddddd	d
                                                              d||d         |d                   }|S )Nr   r[  r\  r]  r'   r^  r   r_  rg   r   r   r`  )r   rF   r   r   r   r   ra  s          rW   window_partition_nchwr  1  s   JAq!QAA!#%c%c%cR]^_R`%c%c%cdddAA!#%bq%b%bQ\]^Q_%b%b%bccc	q!Q+a.(+a.!{1~:M{[\~^^Aii1aAq))4466;;B;q>S^_`SabbGNrY   c           	         |\  }}| j         d         }|                     d||d         z  ||d         z  ||d         |d                   }|                    dddddd                                                              d|||          }|S )Nr'   r   r   rg   r_  r   r   re  rf  s          rW   window_reverse_nchwr  :  s    DAqaARk!n,a;q>.A1kRSnVabcVdeeA			!Q1a##..0055b!QBBAHrY   c           
         | j         \  }}}}t          ||d         z  dk    d| d|d                     t          ||d         z  dk    d| d|d                     |                     |||d         ||d         z  |d         ||d         z            } |                     dddddd	                                                              d
||d         |d                   }|S )Nr   ri  rj  r'   rk  rg   r   r   r_  r   r`  )r   rG   r   r   r   r   ra  s          rW   grid_partition_nchwr  C  s    JAq!QA	!!#YQ#Y#Y9UV<#Y#YZZZA	!!#XA#X#X)TU,#X#XYYY	q!Yq\1	!#4ilASTDUVVAii1aAq))4466;;B9Q<QZ[\Q]^^GNrY   c           	         |\  }}| j         d         }|                     d||d         z  ||d         z  ||d         |d                   }|                    dddddd                                                              d|||          }|S )Nr'   r   r   rg   r_  r   r   re  rn  s          rW   grid_reverse_nchwr  L  s    DAqaARil*A1,=q)A,PYZ[P\]]A			!Q1a##..0055b!QBBAHrY   c            	       X     e Zd ZdZd e            dfdedededef fdZd	 Z	d
 Z
 xZS )PartitionAttention2dzH Grid or Block partition + Attn + FFN

    '2D' NCHW tensor layout.
    rv  r<   r   rw  r
  r  c           
         t                                                       t          t          |j                  |j                  }t          |j                  }|dk    | _        t          | j        r|j
        n|j                  | _        t          || j                  } ||          | _        t          |||j        |j        |j        ||j        |j                  | _        |j        rt/          ||j                  nt1          j                    | _        |dk    rt7          |          nt1          j                    | _         ||          | _        t=          |t?          ||j         z            ||j                  | _!        |j        rt/          ||j                  nt1          j                    | _"        |dk    rt7          |          nt1          j                    | _#        d S ry  )$r   r   r   r   rN   rR   r   rL   r{  r   rF   rG   r|  rs  r  r   r5   r;   r6   r=   r>   r   rJ   r   r   r   r  r   r  r  r   r]   r8   r   r  r  r}  s	           rW   r   zPartitionAttention2d.__init__[  s    	^CN;;NNN
!#-00	-8'4;O(bUXUbcc%c4+>??Z__
\~#mm	
 	
 	
	 FI_g<AAAAZ\ZeZgZg1:R(9---R[]]Z__
c&6 677	     
 FI_g<AAAAZ\ZeZgZg1:R(9---R[]]rY   c                    |j         dd          }| j        rt          || j                  }nt	          || j                  }|                     |          }| j        rt          || j        |          }nt          || j        |          }|S )Nr   )r   r{  r  r|  r  r   r  r  r  s       rW   r  z$PartitionAttention2d._partition_attn  s    7233< 	F/43FGGKK-a1DEEKii,, 	N#K1DhOOAA!+t/BHMMArY   c           
      J   ||                      |                     |                     |                     |                                        z   }||                     |                     |                     |                     |                                        z   }|S rT   r  r   s     rW   r   zPartitionAttention2d.forward  r  rY   r  r   s   @rW   r  r  U  s          #*)>)>)@)@!$S $S$S  $S '	$S
 $S $S $S $S $S $SL        rY   r  c                   t     e Zd ZdZd e             e            dfdededededed	ef fd
ZddZ	d Z
 xZS )MaxxVitBlockz@ MaxVit conv, window partition + FFN , grid partition + FFN
    r'   r<   r   r   r   r   r   r  c                 T   t                                                       |j        | _        |j        dk    rt
          nt          } ||||||          | _        t          |||          }| j        rt          nt          }	|j        rd n |	di || _         |	dddi|| _        d S )Nry   r   r
  r  r   r
  r  rw  gridrb   )r   r   rI   	nchw_attnre   rM  r1  convdictr  ru  rH   r  r  )rV   r   r   r   r   r   r  conv_clsrF  partition_layerr   s             rW   r   zMaxxVitBlock.__init__  s     	(6$,$7:$E$E==;HS'&hR[\\\	wOyQQQ26.Z..FZ"1"?c$$__EcEcWbEcEc(NNN+NNrY   ro   c                 
   | j         )t          t          t          |          | j                    t          t          t          |          | j                   t          t          t
          |          | j                   d S r   )r  r+   r   r  r  r,  r  r"  s     rW   r#  zMaxxVitBlock.init_weights  so    ?& 1&AAA4?SSSG-f===t~NNNGJv666	BBBBBrY   c                    |                      |          }| j        s|                    dddd          }| j        |                     |          }|                     |          }| j        s|                    dddd          }|S rX  )r  r  rY  r  r  r   s     rW   r   zMaxxVitBlock.forward  s    IIaLL~ 	&		!Q1%%A?&""ANN1~ 	&		!Q1%%ArY   r%  )rZ   r[   r\   r   r0   r1   r]   r`   r   r#  r   r   r   s   @rW   r  r    s          '5~'7'75J5J5L5L!O OO O 	O
 %O 3O O O O O O O(C C C C      rY   r  c                   f     e Zd ZdZdd e             e            dfdedef fdZdd	Zd
 Z xZ	S )ParallelMaxxVitBlockzX MaxVit block with parallel cat(window + grid), one FF
    Experimental timm block.
    r'   r   r<   r   r   c                 Z   t                                                       |j        dk    rt          nt          }|dk    r> ||||||          g}	|	 |||||          g|dz
  z  z  }	t          j        |	 | _        n ||||||          | _        t          |||          | _	        d S )Nry   r'   r  )r
  r  r  )
r   r   re   rM  r1  r   r  r  r  r   )rV   r   r   r   num_convr   r   r  r  convsr   s             rW   r   zParallelMaxxVitBlock.__init__  s     	$,$7:$E$E==;a<<Xc76xS\]]]^EhhwXSSSTX`cdXdeeEu-DII gf(V_```DI.7[deee			rY   ro   c                     t          t          t          |          | j                   t          t          t          |          | j                   d S r   )r+   r   r  r   r,  r  r"  s     rW   r#  z!ParallelMaxxVitBlock.init_weights  sG    G-f===tyIIIGJv666	BBBBBrY   c                     |                      |          }|                    dddd          }|                     |          }|                    dddd          }|S rX  )r  rY  r   r   s     rW   r   zParallelMaxxVitBlock.forward  sS    IIaLLIIaAq!!IIaLLIIaAq!!rY   r%  )
rZ   r[   r\   r   r0   r1   r   r#  r   r   r   s   @rW   r  r    s          '5~'7'75J5J5L5Lf f %f 3f f f f f f*C C C C      rY   r  c                        e Zd Zdddd e             e            dfdededed	ed
eeef         deeee         f         dededee	e
e	         f         f fdZd Z xZS )MaxxVitStager   r_  )   r  r   r<   r3  r4  r   depth	feat_sizeblock_typesr   r   r  c
                 d   t                                                       d| _        t          ||          }g }
t	          |          D ]\  }}|dk    r|nd}|dv sJ |dk    r4|j        dk    rt          nt          }|
 ||||||	|                   gz  }
n|dk    r0t          ||          }|
t          ||||||	|         	          gz  }
nK|d
k    r |
t          ||||||	|                   gz  }
n%|dk    r|
t          ||||||	|                   gz  }
|}t          j        |
 | _        d S )NFr   r'   )r   r   MPMr   ry   r  r   )r   r   r
  r  r  )r   r   r   r  r  )r   r   grad_checkpointingr   	enumeratere   rM  r1  rs  r	  r  r  r   r  blocks)rV   r3  r4  r   r  r  r  r   r   r  r  itblock_strider  r   r   s                   rW   r   zMaxxVitStage.__init__  s    	"'";66k** (	 (	DAq%&!VV66L-----Cxx,4,?:,M,M==S^88' 'l     c-oyII-' +''l     c<'%$3'l     d/'%$3'l     FFmV,rY   c                     | j         r4t          j                                        st	          | j        |          }n|                     |          }|S rT   )r  r   jitis_scriptingr,   r  r   s     rW   r   zMaxxVitStage.forward%  sI    " 	59+A+A+C+C 	t{A..AAAArY   )rZ   r[   r\   r1   r0   r]   r   r
   ra   r`   r   r   r   r   r   s   @rW   r  r    s        
 )1255J5J5L5L'5~'7'735:- :-:- :- 	:-
 :- S#X:- sE#J/:- 3:- %:- UDK/0:- :- :- :- :- :-x      rY   r  c                   \     e Zd Z	 	 	 	 	 	 ddeded	ed
ededededef fdZddZd Z	 xZ
S )Stemrg   ro   FrK   rz   r{   r3  r4  rh   rp   rA   rL   rN   rR   c	                    t                                                       t          |t          t          f          st          |          }t          t          ||          |          }	|d         | _        d| _	        t          ||d         |d||          | _         |	|d                   | _        t          |d         |d         |d||          | _        d S )Nr  r   r   r   )r   rp   rA   r'   )r   r   r   listtupler   r   r   r4  r   r   conv1r  conv2)rV   r3  r4  rh   rp   rA   rL   rN   rR   r@  r   s             rW   r   zStem.__init__/  s     	'D%=11 	)((G !3J	!J!JPXYYYr{"671:{1V]dhiii
#^GAJ//
"71:wqz;qZahlmmm


rY   c                 N    t          t          t          |          |            d S r   rH  r"  s     rW   r#  zStem.init_weightsF  rI  rY   c                     |                      |          }|                     |          }|                     |          }|S rT   )r  r  r  r   s     rW   r   zStem.forwardI  s4    JJqMMJJqMMJJqMMrY   )rg   ro   FrK   rz   r{   r%  )rZ   r[   r\   r]   ra   r_   r`   r   r#  r   r   r   s   @rW   r  r  -  s          !#+"n nn n 	n
 n n n n n n n n n n.> > > >      rY   r  c                     | j         | j        sJ | S |d         | j        z  |d         | j        z  f}t          | ||          } | S )Nr   r'   )rF   rG   )rF   rG   rE   r   )r
  rc  r|  s      rW   cfg_window_sizer  P  sX    
"}
a[C$77!H[9[[N
#>^
L
L
LCJrY   c           	      l   i }i }i }|                                 D ]h\  }}|                    d          r|||                    dd          <   4|                    d          r|||                    dd          <   c|||<   it          | ft          | j        fi |t          | j        fi |d|} | S )Ntransformer_ro   conv_)r   r   )items
startswithr   r   r   )r
  kwargstransformer_kwargsconv_kwargsbase_kwargsr   r   s          rW   _overlay_kwargsr  Y  s    KK  1<<'' 	@Aqyy<<==\\'"" 	23K		'2..//KNN
 3JJ7IJJ5555  	 C JrY   c                   Z    e Zd ZdZ	 	 	 	 	 	 d*dedeeeeef         f         d	ed
edede	de	f fdZ
d+dZej        j        d             Zej        j        d,d            Zej        j        d-d            Zej        j        dej        fd            Zd.d
edee         fdZ	 	 	 	 	 d/dej        deeeee         f                  dededed edeeej                 eej        eej                 f         f         fd!Z	 	 	 d0deeee         f         d#ed$efd%Zd& Zd,d'efd(Zd) Z xZS )1r2   z| CoaTNet + MaxVit base model.

    Highly configurable for different block compositions, tensor layouts, pooling types.
       rg     r   r<   r
  rc  in_chansnum_classesglobal_pool	drop_ratedrop_path_ratec                 
   t                                                       t          |          }|rt          |fi |}t	          |j        |          }	|| _        || _        |j        d         x| _	        | _        || _
        d| _        g | _        t          ||j        |j        j        |j        |j        j        |j        j        |j        j                  | _        | j        j        }
| xj        t/          | j        j        dd          gz  c_        t3          d t5          |t          |
                    D                       }t7          |j                  }t7          |j                  |k    sJ d t;          j        d	|t?          |j                                                 |j                  D             }| j        j        }g }tC          |          D ]}d|j        |         }t3          fd
|D                       }|tE          |||j        |         |j#        |         |j        |	|||                   gz  }|
z  }
|}| xj        t/          ||
d|           gz  c_        tI          j%        | | _&        tO          tQ          |j        j                  |j        j                  }|j)        rItI          j*                    | _+        |j)        | _)        tY          | j	        || j)        |||          | _-        n>| j	        | _)         || j	                  | _+        t]          | j	        |||          | _-        |j/        dv sJ |j/        r+ta          tO          | j1        |j/                  |            d S d S )Nr   F)r3  r4  rp   rA   rL   rN   rR   r   stem)num_chs	reductionr  c                     g | ]
\  }}||z  S rb   rb   ).0r  ss      rW   
<listcomp>z$MaxxVit.__init__.<locals>.<listcomp>  s     OOOda16OOOrY   c                 6    g | ]}|                                 S rb   )tolist)r  r   s     rW   r  z$MaxxVit.__init__.<locals>.<listcomp>  s     hhhaqxxzzhhhrY   r   c                 &    g | ]}|d z
  z  d z   S )r'   rb   )r  rstage_strides     rW   r  z$MaxxVit.__init__.<locals>.<listcomp>  s'    NNNqA,6:NNNrY   )r  r  r   r   r  r  zstages.r  )hidden_sizer@   r  rN   )r@   r  )ro   r   r   r   r   r!  )2r   r   r   r  r  r   r  r  r   num_featuresr  r  feature_infor  r   r   rp   r   rL   rN   rR   r  r   r  r4  r  ziplenr   r   linspacesumsplitranger  re   r   r  stagesr   r   r   r   r  r   headr   r   r+   _init_weights)rV   r
  rc  r  r  r  r  r  r  r   r   r  
num_stagesdprr3  r  r  r4  final_norm_layerr  r   s                      @rW   r   zMaxxVit.__init__s  s    	X&& 	1!#0000C)#*=xHH&&-0]2->>DN""'NL(l,|.\*
 
 
	 !d49+<RXYYYZZOOc(If<M<M.N.NOOOPP	''
3:*,,,,hh5>!^S__#U#U#[#[\_\f#g#ghhh"z"" 	a 	aALmA&GNNNNINNNOOI|jmN1- /#a&	 	 	 	 	F l"FF$w&Q^[\Q^Q^"_"_"_!``mV,">#2E2P#Q#QWZWjWsttt 	sDI$'$8D!-! 1%#+  DII %)$5D!(():;;DI&t'8+Q\hqrrrDI "\\\\\? 	S 23?KKKTRRRRR	S 	SrY   ro   c                     t          |d          r>	 |                    |           d S # t          $ r |                                 Y d S w xY wd S )Nr#  r!  )hasattrr#  	TypeError)rV   r  r  r  s       rW   r  zMaxxVit._init_weights  sv    6>** 	&&##6#22222 & & &##%%%%%%&	& 	&s   * AAc                 >    d |                                  D             S )Nc                 P    h | ]"\  }t          fd dD                        #S )c              3       K   | ]}|v V  	d S rT   rb   )r  nr   s     rW   	<genexpr>z4MaxxVit.no_weight_decay.<locals>.<setcomp>.<genexpr>  s'      SSa16SSSSSSrY   )relative_position_bias_tablezrel_pos.mlp)any)r  _r   s     @rW   	<setcomp>z*MaxxVit.no_weight_decay.<locals>.<setcomp>  sX     U U U!QSSSS#RSSSSSUU U UrY   )named_parametersrU   s    rW   no_weight_decayzMaxxVit.no_weight_decay  s5    U U//11U U U 	UrY   Fc                 ,    t          dddg          }|S )Nz^stem)z^stages\.(\d+)N)z^norm)i )r  r  )r  )rV   coarsematchers      rW   group_matcherzMaxxVit.group_matcher  s)    -/CD
 
 
 rY   Tc                 (    | j         D ]	}||_        
d S rT   )r  r  )rV   enabler  s      rW   set_grad_checkpointingzMaxxVit.set_grad_checkpointing  s(     	* 	*A#)A  	* 	*rY   returnc                     | j         j        S rT   )r  fcrU   s    rW   get_classifierzMaxxVit.get_classifier  s    y|rY   Nc                 J    || _         | j                            ||           d S rT   )r  r  reset)rV   r  r  s      rW   reset_classifierzMaxxVit.reset_classifier  s&    &	[11111rY   NCHWr   indicesr  
stop_early
output_fmtintermediates_onlyc                 >   |dv s
J d            g }t          t          | j                  dz   |          \  }}	d}
|                     |          }|
|v r|                    |           t          | j                  }t
          j                                        s|s| j        }n| j        d|	         }|D ]K}|
dz  }
 ||          }|
|v r5|r|
|k    r|                     |          }n|}|                    |           L|r|S |                     |          }||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r  zOutput shape must be NCHW.r'   r   N)	r)   r  r  r  appendr   r  r  r  )rV   r   r  r  r  r  r  intermediatestake_indices	max_indexfeat_idxlast_idxr  stagex_inters                  rW   forward_intermediateszMaxxVit.forward_intermediates  sU   * Y&&&(D&&&"6s4;7G7G!7KW"U"Ui IIaLL|##  ###t{##9!!## 	-: 	-[FF[),F 	. 	.EMHaA<''  H00"iillGGG$$W--- 	!  IIaLL-rY   r'   
prune_norm
prune_headc                     t          t          | j                  dz   |          \  }}| j        d|         | _        |rt          j                    | _        |r|                     dd          | _        |S )z@ Prune layers not required for specified intermediates.
        r'   Nr   ro   )r)   r  r  r   r   r  r  r  )rV   r  r*  r+  r#  r$  s         rW   prune_intermediate_layersz!MaxxVit.prune_intermediate_layers  ss     #7s4;7G7G!7KW"U"Uik*9*- 	&DI 	5--a44DIrY   c                     |                      |          }|                     |          }|                     |          }|S rT   )r  r  r  r   s     rW   forward_featureszMaxxVit.forward_features-  s4    IIaLLKKNNIIaLLrY   
pre_logitsc                 ^    |r|                      ||          n|                      |          S )N)r0  )r  )rV   r   r0  s      rW   forward_headzMaxxVit.forward_head3  s-    6@Rtyyzy222diiPQllRrY   c                 Z    |                      |          }|                     |          }|S rT   )r/  r2  r   s     rW   r   zMaxxVit.forward6  s-    !!!$$a  rY   )r  rg   r  r   r<   r<   r%  F)TrT   )NFFr  F)r'   FT) rZ   r[   r\   r   r/   r
   r]   r   ra   r`   r   r  r   r  ignorer  r  r  r   Moduler  r	   r  r   r   r_   r)  r-  r/  r2  r   r   r   s   @rW   r2   r2   m  s         58#$!$&QS QSQS CsCx01QS 	QS
 QS QS QS "QS QS QS QS QS QSf& & & & YU U U
 Y    Y* * * * Y	    2 2C 2hsm 2 2 2 2 8<$$',3  3 |3  eCcN343  	3 
 3  3  !%3  
tEL!5tEL7I)I#JJ	K3  3  3  3 n ./$#	 3S	>*  	      S S$ S S S S      rY   r2   r   r?   FreluTrM   rO   rA   rC   c                 z    t          t          | |dd|||d|	  	        t          d|||	|||
|                    S )NTFrt   )	rm   r@   rj   rf   rk   rq   ru   rL   rN   )r9   r:   r@   rJ   rN   rP   rB   rD   r   r   r  r0   r1   )rm   r@   conv_output_biasconv_attn_earlyconv_attn_act_layerconv_norm_layertransformer_shortcut_biastransformer_norm_layertransformer_norm_layer_clrJ   rB   rD   s               rW   _rw_coat_cfgrB  <  st    6 #(&.&

 

 

 .3#-3%#	
 	
 	
   rY   rl         ?r4   c                 x    t          t          | |d||d|          t          d||||	|||
|	  	                  S )NFrt   )rm   r@   rf   rk   rw   rL   rN   )	r9   r@   r5   rF   rJ   rN   rP   rB   rD   r9  r:  )rm   r@   r;  conv_attn_ratior>  r@  rA  rF   r5   rJ   rB   rD   s               rW   _rw_max_cfgrF  p  sq    ( #(&&
 
 
 .##-3%#

 

 

   rY   rQ   r   c                     t          |          }t          t          d| |d|d         ||          t          d||||d         |||	|
	  	                  S )Nry   Fr   )re   rm   r@   rf   rJ   rN   rP   r'   )	r9   r@   rF   rH   rJ   rN   rP   rB   rD   r9  )r   r  r0   r1   )rm   r@   r>  conv_norm_layer_clr@  rA  rF   rH   rJ   rB   rD   s              rW   	_next_cfgrI    s     K((K!##A&,
 
 
 .#'#A-3%#

 

 

   rY   c            	      f    t          t          ddd          t          dddd          	          S )
NgMbP?	gelu_tanhsame)rR   rL   rp   r{   Frr  )rR   rL   r6   rB   r9  r:  rb   rY   rW   _tf_cfgrM    sT    !
 
 

 .!"	
 
 
   rY   coatnet_pico_rw)r         rC   r   )r4   r   )r   r   r   rv   )r;  rE  coatnet_nano_rw)rg   r_     rg   )rm   r;  rE  coatnet_0_rwr~   )r   rg   rN  r   )r<  r?  coatnet_1_rw)r   rR  r  r   )rm   r<  r?  coatnet_2_rw)rO  rP  rC      )r   rO  rt   )rm   r=  coatnet_3_rw)r   r   r      )r   r   )rm   r=  rJ   coatnet_bn_0_rwrz   )rm   r<  r?  r@  coatnet_rmlp_nano_rwr   )r;  rE  rB   rD   coatnet_rmlp_0_rw)rm   rB   coatnet_rmlp_1_rwr   )r@   r<  r?  rB   rD   coatnet_rmlp_1_rw2)rm   rB   rD   coatnet_rmlp_2_rw)rm   r=  rJ   rB   coatnet_rmlp_3_rwcoatnet_nano_cc)r   r   r   r   ra  )r   r   r   re   coatnext_nano_rwr   )r   r   r   r   )r{   N)rB   rJ   	coatnet_0r   r   )r   r   r   r   	coatnet_1	coatnet_2rO  rV  	coatnet_3r   rX  	coatnet_4)r         r   	coatnet_5)rP  rC         rl  maxvit_pico_rw)r4   r   rO  rP  )r   r   r   r   )r  r  r  r  )   r4   )r   r   re   r   maxvit_nano_rw)r'   r   rg   r'   maxvit_tiny_rwmaxvit_tiny_pm)r  r  r  r  maxvit_rmlp_pico_rw)rB   maxvit_rmlp_nano_rwmaxvit_rmlp_tiny_rwmaxvit_rmlp_small_rwmaxvit_rmlp_base_rw)r   r   re   r   r   maxxvit_rmlp_nano_rw)r   r   re   r   r   maxxvit_rmlp_tiny_rwmaxxvit_rmlp_small_rw)0   r   maxxvitv2_nano_rw)rH   rB   maxxvitv2_rmlp_base_rw)r   rR  rh  r   )rH   maxxvitv2_rmlp_large_rw)   i@  i  rk  )r   rR     r   )P   r~  rk  maxvit_tiny_tf)r   r   re   r   r   r   maxvit_small_tfmaxvit_base_tfmaxvit_large_tfmaxvit_xlarge_tfmodelc                 f   |                                 }i }|                                 D ]\  }}|                    d          ro|                    |d d                   }|j        |j        j        k    s|j        d         |j        d         k    r!t          ||j        |j        j                  }||v rq|j        ||         j        k    r[|	                                ||         	                                k    r+|j        dv sJ |
                    ||         j                  }|||<   |S )Nr  ir   r'   )new_window_sizenew_bias_shape)r   r_  )
state_dictr  endswithget_submoduler   r  rF   r&   ndimnumelr   )r  r  model_state_dictout_dictr   r   ms          rW   checkpoint_filter_fnr  D  sA   ''))H  ""  1::455 	##AdsdG,,Aw!8>>>!-PQBRVWVcdeVfBfBf-$%M#$#A#G      QV/?/B/G%G%GAGGIIYijkYlYrYrYtYtLtLt6V####		*1-344AOrY   c                     |<| t           v r| }n0d                    |                     d          d d                   }t          t          | |ft           |         t          d          t          d|S )Nr	  r   T)flatten_sequential)	model_cfgfeature_cfgpretrained_filter_fn)
model_cfgsjoinr  r(   r2   r  r  )variantcfg_variant
pretrainedr  s       rW   _create_maxxvitr  Y  s    j  !KK((7==#5#5crc#:;;K*[)D1111	 
   rY   c                 "    | dddddddddd	d
|S )Nr  )rg   r  r  )rN  rN  ffffff?bicubic)      ?r  r  z
stem.conv1zhead.fcT)urlr  
input_size	pool_sizecrop_pctinterpolationmeanr   
first_conv
classifierfixed_input_sizerb   )r  r  s     rW   _cfgr  g  s7    4}SY9")    rY   zcoatnet_pico_rw_224.untrained)r  zcoatnet_nano_rw_224.sw_in1kztimm/zyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_nano_rw_224_sw-f53093b4.pthg?)	hf_hub_idr  r  zcoatnet_0_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_0_rw_224_sw-a6439706.pth)r  r  zcoatnet_1_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_1_rw_224_sw-5cae1ea8.pthz!coatnet_2_rw_224.sw_in12k_ft_in1k)r  z'coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_384.sw_in12k_ft_in1k)rg   r   r   )rh  rh  g      ?squash)r  r  r  r  	crop_modezcoatnet_bn_0_rw_224.sw_in1kzyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_bn_0_rw_224_sw-c228e218.pthr  )r  r  r  r   r  z coatnet_rmlp_nano_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_nano_rw_224_sw-bd1d51b3.pthzcoatnet_rmlp_0_rw_224.untrainedzcoatnet_rmlp_1_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_1_rw_224_sw-9051e6c3.pthzcoatnet_rmlp_2_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_2_rw_224_sw-5ccfac55.pthzcoatnet_rmlp_3_rw_224.untrainedzcoatnet_nano_cc_224.untrainedzcoatnext_nano_rw_224.sw_in1kzzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnext_nano_rw_224_ad-22cb71c2.pthzcoatnet_2_rw_224.sw_in12ki-.  )r  r  zcoatnet_3_rw_224.sw_in12kzcoatnet_rmlp_1_rw2_224.sw_in12kzcoatnet_rmlp_2_rw_224.sw_in12kzcoatnet_0_224.untrainedzcoatnet_1_224.untrainedzcoatnet_2_224.untrainedzcoatnet_3_224.untrainedzcoatnet_4_224.untrainedzcoatnet_5_224.untrainedzmaxvit_pico_rw_256.untrained)rg   rP  rP  )   r  )r  r  r  zmaxvit_nano_rw_256.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth)r  r  r  r  zmaxvit_tiny_rw_224.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_tiny_rw_224_sw-7d0dffeb.pthzmaxvit_tiny_rw_256.untrainedzmaxvit_tiny_pm_256.untrainedzmaxvit_rmlp_pico_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_pico_rw_256_sw-8d82f2c6.pthzmaxvit_rmlp_nano_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pthzmaxvit_rmlp_tiny_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_tiny_rw_256_sw-bbef0ff5.pthz maxvit_rmlp_small_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_small_rw_224_sw-6ef0ae4f.pthz"maxvit_rmlp_small_rw_256.untrainedz(maxvit_rmlp_base_rw_224.sw_in12k_ft_in1kz(maxvit_rmlp_base_rw_384.sw_in12k_ft_in1kz maxvit_rmlp_base_rw_224.sw_in12kz maxxvit_rmlp_nano_rw_256.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_nano_rw_256_sw-0325d459.pthz"maxxvit_rmlp_tiny_rw_256.untrainedz!maxxvit_rmlp_small_rw_256.sw_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_small_rw_256_sw-37e217ff.pthzmaxxvitv2_nano_rw_256.sw_in1k)r  r  r  z+maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1kz+maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1kz%maxxvitv2_rmlp_large_rw_224.untrainedz#maxxvitv2_rmlp_base_rw_224.sw_in12kzmaxvit_tiny_tf_224.in1k)r  r  r   zmaxvit_tiny_tf_384.in1kzmaxvit_tiny_tf_512.in1k)rg   rC   rC   )r  r  zmaxvit_small_tf_224.in1kzmaxvit_small_tf_384.in1kzmaxvit_small_tf_512.in1kzmaxvit_base_tf_224.in1kzmaxvit_base_tf_384.in1kzmaxvit_base_tf_512.in1kzmaxvit_large_tf_224.in1kzmaxvit_large_tf_384.in1kzmaxvit_large_tf_512.in1kzmaxvit_base_tf_224.in21kiSU  z maxvit_base_tf_384.in21k_ft_in1kz maxvit_base_tf_512.in21k_ft_in1kzmaxvit_large_tf_224.in21kz!maxvit_large_tf_384.in21k_ft_in1kz!maxvit_large_tf_512.in21k_ft_in1k)r  r  r  r  zmaxvit_xlarge_tf_224.in21kz"maxvit_xlarge_tf_384.in21k_ft_in1kz"maxvit_xlarge_tf_512.in21k_ft_in1kr  c                     t          dd| i|S )Ncoatnet_pico_rw_224r  )r  r  r  r  s     rW   r  r  M      RRZR6RRRrY   c                     t          dd| i|S )Ncoatnet_nano_rw_224r  )r  r  r  s     rW   r  r  R  r  rY   c                     t          dd| i|S )Ncoatnet_0_rw_224r  )r  r  r  s     rW   r  r  W      OO*OOOOrY   c                     t          dd| i|S )Ncoatnet_1_rw_224r  )r  r  r  s     rW   r  r  \  r  rY   c                     t          dd| i|S )Ncoatnet_2_rw_224r  )r  r  r  s     rW   r  r  a  r  rY   c                     t          dd| i|S )Ncoatnet_3_rw_224r  )r  r  r  s     rW   r  r  f  r  rY   c                     t          dd| i|S )Ncoatnet_bn_0_rw_224r  )r  r  r  s     rW   r  r  k  r  rY   c                     t          dd| i|S )Ncoatnet_rmlp_nano_rw_224r  )r  r  r  s     rW   r  r  p      WW*WPVWWWrY   c                     t          dd| i|S )Ncoatnet_rmlp_0_rw_224r  )r  r  r  s     rW   r  r  u      TTzTVTTTrY   c                     t          dd| i|S )Ncoatnet_rmlp_1_rw_224r  )r  r  r  s     rW   r  r  z  r  rY   c                     t          dd| i|S )Ncoatnet_rmlp_1_rw2_224r  )r  r  r  s     rW   r  r    s    UU
UfUUUrY   c                     t          dd| i|S )Ncoatnet_rmlp_2_rw_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_rmlp_2_rw_384r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_rmlp_3_rw_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_nano_cc_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnext_nano_rw_224r  )r  r  r  s     rW   r  r    s    SSjSFSSSrY   c                     t          dd| i|S )Ncoatnet_0_224r  )r  r  r  s     rW   r  r        LLzLVLLLrY   c                     t          dd| i|S )Ncoatnet_1_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_2_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_3_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_4_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Ncoatnet_5_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_pico_rw_256r  )r  r  r  s     rW   r  r        QQJQ&QQQrY   c                     t          dd| i|S )Nmaxvit_nano_rw_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_tiny_rw_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_tiny_rw_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_rmlp_pico_rw_256r  )r  r  r  s     rW   r  r        VVVvVVVrY   c                     t          dd| i|S )Nmaxvit_rmlp_nano_rw_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_rmlp_tiny_rw_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_rmlp_small_rw_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_rmlp_small_rw_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_rmlp_base_rw_224r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_rmlp_base_rw_384r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxvit_tiny_pm_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxxvit_rmlp_nano_rw_256r  )r  r  r  s     rW   r  r    r  rY   c                     t          dd| i|S )Nmaxxvit_rmlp_tiny_rw_256r  )r   r  r  s     rW   r   r     r  rY   c                     t          dd| i|S )Nmaxxvit_rmlp_small_rw_256r  )r  r  r  s     rW   r  r  	  s    XX:XQWXXXrY   c                     t          dd| i|S )Nmaxxvitv2_nano_rw_256r  )r  r  r  s     rW   r  r  	  r  rY   c                     t          dd| i|S )Nmaxxvitv2_rmlp_base_rw_224r  )r  r  r  s     rW   r  r  	      YYJYRXYYYrY   c                     t          dd| i|S )Nmaxxvitv2_rmlp_base_rw_384r  )r	  r  r  s     rW   r	  r	  	  r  rY   c                     t          dd| i|S )Nmaxxvitv2_rmlp_large_rw_224r  )r  r  r  s     rW   r  r  	  s    ZZZZSYZZZrY   c                     t          dd| i|S )Nmaxvit_tiny_tf_224r  r  )r  r  r  r  s     rW   r  r  	      ccjc\bcccrY   c                     t          dd| i|S )Nmaxvit_tiny_tf_384r  r  )r  r  r  r  s     rW   r  r  	  r  rY   c                     t          dd| i|S )Nmaxvit_tiny_tf_512r  r  )r  r  r  r  s     rW   r  r  $	  r  rY   c                     t          dd| i|S )Nmaxvit_small_tf_224r  r  )r  r  r  r  s     rW   r  r  )	      eePZe^deeerY   c                     t          dd| i|S )Nmaxvit_small_tf_384r  r  )r  r  r  r  s     rW   r  r  .	  r  rY   c                     t          dd| i|S )Nmaxvit_small_tf_512r  r  )r  r  r  r  s     rW   r  r  3	  r  rY   c                     t          dd| i|S )Nmaxvit_base_tf_224r  r  )r  r  r  r  s     rW   r  r  8	  r  rY   c                     t          dd| i|S )Nmaxvit_base_tf_384r  r  )r  r  r  r  s     rW   r  r  =	  r  rY   c                     t          dd| i|S )Nmaxvit_base_tf_512r  r  )r  r  r  r  s     rW   r  r  B	  r  rY   c                     t          dd| i|S )Nmaxvit_large_tf_224r  r  )r!  r  r  r  s     rW   r!  r!  G	  r  rY   c                     t          dd| i|S )Nmaxvit_large_tf_384r  r  )r#  r  r  r  s     rW   r#  r#  L	  r  rY   c                     t          dd| i|S )Nmaxvit_large_tf_512r  r  )r%  r  r  r  s     rW   r%  r%  Q	  r  rY   c                     t          dd| i|S )Nmaxvit_xlarge_tf_224r  r  )r'  r  r  r  s     rW   r'  r'  V	      ggR\g`fgggrY   c                     t          dd| i|S )Nmaxvit_xlarge_tf_384r  r  )r*  r  r  r  s     rW   r*  r*  [	  r(  rY   c                     t          dd| i|S )Nmaxvit_xlarge_tf_512r  r  )r,  r  r  r  s     rW   r,  r,  `	  r(  rY   r%  )r   r?   FFr7  ro   TrM   rO   NrA   rC   )rl   r?   FrC  ro   rM   rO   Nr4   NrA   rC   )rl   r?   rM   rO   rM   rO   NFrQ   r   rC   rb   )NFr4  )r   r)  collectionsr   dataclassesr   r   r   	functoolsr   typingr   r	   r
   r   r   r   r   	torch.jitr   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   _builderr(   	_featuresr)   _features_fxr*   _manipulater+   r,   	_registryr-   r.   __all__r1   r0   r/   r6  r   r   r   r   r   r  r	  r,  r/  r1  rM  r]   rb  rg  rl  ro  rs  ru  r  r  r  r  r  r  r  r  r  r  r  r  r2   rB  rF  rI  rM  r  r  r  r  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r	  r  r  r  r  r  r  r  r  r  r  r!  r#  r%  r'  r*  r,  rb   rY   rW   <module>r;     s3*  " "H  # # # # # # 1 1 1 1 1 1 1 1 1 1       9 9 9 9 9 9 9 9 9 9 9 9 9 9              A A A A A A A A ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` t t t t t t t t t t t t t t t t Z Z Z Z Z Z Z Z Z Z Z Z Z Z f f f f f f f f f f f f f f * * * * * * + + + + + + 3 3 3 3 3 3 4 4 4 4 4 4 4 4 < < < < < < < <
N
N
N 2 2 2 2 2 2 2 2@  P  P  P  P  P  P  P  PF 	! 	! 	! 	! 	! 	! 	! 	!B B B B B") B B BJC C C C C") C C CL< < < < < < < << < < < <29 < < <$ $ $ $ $29 $ $ $N0 0 0 00> > > > > > > >B, , , ,.& & &Y Y Y Y Y") Y Y YxJ J J J JBI J J JZT#Y     c d3i    c     T#Y $s)    .    = = = = =29 = = =@F F F F F F F FR$s)     d3i 49    d3i     $s) tCy    > > > > >29 > > >B) ) ) ) )29 ) ) )X# # # # #29 # # #LB B B B B29 B B BJ         29      F. %S/        (L L L L Lbi L L L` ""&,"-1 1 1 1j ,"-) ) ) )Z %&,"-$ $ $ $N    T n n nJ %  +! 
 
 
	  n J 	%	 	 +! 
 
 
		 	 	n*  %  , &+
 
 
	  +n<  	%	 	 , &+
 
 
		 	 	=nP  	'	 	 , &
 
 
		 	 	Qnd  	'	 	 , &
 
 
		 	 	en| J 
%
 
 , &+#0	
 
 
	
 
 
}nR $ 
%
 
 +! 	
 
 
	
 
 
Snh !j %  ,
 
 
	  inz !j %  , &+
 
 
	  {nR "z 	%	 	 ,
 
 
		 	 	Snf !j 
'
 
 , &	
 
 
	
 
 
gn| !j 
'
 
 , &	
 
 
	
 
 
}nT J %5	 
 ,..  Unb  Z 	%		 	
 )$
 
 
	 	 	cnz j%	   {nF j%	   GnR j'	   Sn^ j'	   _nj j'	   knv j(	   wnF : $	 
 +--  GnT : %	 
 +--  Unb : %	 
 +--  cnp : %	 
 +--  qn@ #
 $	 
 +5
)
)
)  AnN #
 %	 
 +5
)
)
)  On\ #
 %	 
 +5
)
)
)  ]nj $ 	%		 	
 +
 
 
	 	 	kn~ #
 	%	 	 +
 
 
	 	 	nT $ %  )++  Und $ %	 
 )++  enr %* %	 
 )++  snB	 !j 
%
 
 )
 
 

 
 
C	nX	 &: '	 
 )
 
 
  Y	nj	 'J 	'	 	 )
 
 
	 	 	k	nB
 : %  '))  C
nT
 J %  '))  U
nf
 : %  '))  g
nx
 J '  '))  y
nJ  Z '  '))  Kn
bBI    *       %$ X&#TTb\\\X& "44 H$ $ $X&  E!F !F !FX&  E! ! !X&  (* * *!X&* .tt0 0 0+X&. -dd/ / //X&2 -dd Hsh/X /X /X3X&< "44 H"(<	$ $ $=X&F ' M) ) )GX&N &tt|||OX&P $TT J&K &K &KQX&V $TT J&K &K &KWX&\ &tt|||]X&^ $TTb\\\_X&` #DD I% % %aX&l  " " "mX& X&r  " " "sX&x &tt( ( (yX&~ %dd' ' 'X&H tt|||IX&J tt|||KX&L tt|||MX&N tt|||OX&P tt|||QX&R tt|||SX&X #DDRMU[$\$\$\YX&Z !$$ G F#4 #4 #4[X&b !$$ G#H #H #HcX&h #DD F%4 %4 %4iX&n #DDRMU[$\$\$\oX&t &tt L F(4 (4 (4uX&| &tt L F(4 (4 (4}X&D &tt L F(4 (4 (4EX& X& X&L ' M) ) )MX&V )$$ F+4 +4 +4WX&` /1 1 1aX&f / Hsh1X 1X 1XgX&p ') ) )qX&| ' M F)4 )4 )4}X&D )$$2-[a*b*b*bEX&F ( N F*4 *4 *4GX&R $TT F&4 &4 &4SX&X 2444 4 4YX&\ 244 Hsh4X 4X 4X]X&b ,TTb\\\cX&f *44, , ,gX&p tt"(< >  >  >qX&v tt Hsh X  X  XwX&| tt Hsh X  X  X}X&B "(<!> !> !>CX& X& X&H  Hsh!X !X !XIX&N  Hsh!X !X !XOX&T tt"(< >  >  >UX&Z tt Hsh X  X  X[X&` tt Hsh X  X  XaX&f "(<!> !> !>gX&l  Hsh!X !X !XmX&r  Hsh!X !X !XsX&z ! ! !{X&@ ' Hsh)X )X )XAX&F ' Hsh)X )X )XGX&L  " " "MX&R ( Hsh*X *X *XSX&X ( 3(*D *D *DYX&^ !$$# # #_X&d )$$ Hsh+X +X +XeX&j )$$ Hsh+X +X +XkX& X& X Xv S Sw S S S S S Sw S S S S P PG P P P P P PG P P P P P PG P P P P P PG P P P P S Sw S S S S X XG X X X X U U U U U U U U U U U U V V' V V V V U U U U U U U U U U U U U U U U U U S Sw S S S S T T T T T T M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M M R Rg R R R R R Rg R R R R R Rg R R R R R Rg R R R R W W7 W W W W W W7 W W W W W W7 W W W W X XG X X X X X XG X X X X W W7 W W W W W W7 W W W W R Rg R R R R X XG X X X X X XG X X X X Y YW Y Y Y Y U U U U U U Z Zg Z Z Z Z Z Zg Z Z Z Z [ [w [ [ [ [ d dg d d d d d dg d d d d d dg d d d d f fw f f f f f fw f f f f f fw f f f f d dg d d d d d dg d d d d d dg d d d d f fw f f f f f fw f f f f f fw f f f f h h h h h h h h h h h h h h h h h h h hrY   