
    Ng                        d Z ddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
m	c mZ ddlmZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZmZ d
dlmZ d
dlmZ d
dl m!Z!m"Z" dgZ# G d de	j$                  Z% G d de	j$                  Z& G d de	j$                  Z' G d de	j$                  Z( G d de	j$                  Z) G d de	j$                  Z* G d de          Z+ G d de          Z, G d d e	j                  Z- G d! d"e	j$                  Z. G d# d$e	j$                  Z/ G d% d&e	j$                  Z0 G d' d(e	j$                  Z1 G d) d*e	j$                  Z2 G d+ de	j$                  Z3d, Z4dd.Z5dd0Z6 e!i d1 e6d2d34          d5 e6d2d34          d6 e6d2d34          d7 e6d2d84          d9 e6d2d84          d: e6d2;          d< e6d2;          d= e6d2;          d> e6d2;          d? e6d2;          d@ e6d2dAB          dC e6d2dAdDdEF          dG e6d2dAB          dH e6d2dAdDdEF          dI e6d2dAdJK          dL e6d2dAB          dM e6d2dAdDdEF          i dN e6d2dAB          dO e6d2dAdDdEF          dP e6d2dAdJK          dQ e6d2dAB          dR e6d2dAdDdEF          dS e6d2dAB          dT e6d2dAdDdEF          dU e6d2dAdJK          dV e6d2dAB          dW e6d2dAdDdEF          dX e6d2dAB          dY e6d2dAdDdEF          dZ e6d2dAdJK          d[ e6d2dAB          d\ e6d2dAdDdEF          d] e6d2dAB          d^ e6d2dAdDdEF          i d_ e6d2dAdJK          d` e6d2dAB          da e6d2dAdDdEF          db e6d2dAB          dc e6d2dAdDdEF          dd e6d2dAdJK          de e6d2dAB          df e6d2dAdDdEF          dg e6d2dAB          dh e6d2dAdDdEF          di e6d2dAdJK          dj e6d2dAB          dk e6d2dAdDdEF          dl e6d2dAB          dm e6d2dAdDdEF          dn e6d2dAdJK                    Z7e"ddoe3fdp            Z8e"ddoe3fdq            Z9e"ddoe3fdr            Z:e"ddoe3fds            Z;e"ddoe3fdt            Z<e"ddoe3fdu            Z=e"ddoe3fdv            Z>e"ddoe3fdw            Z?e"ddoe3fdx            Z@e"ddoe3fdy            ZAe"ddoe3fdz            ZBe"ddoe3fd{            ZCe"ddoe3fd|            ZDe"ddoe3fd}            ZEe"ddoe3fd~            ZFe"ddoe3fd            ZGe"ddoe3fd            ZHe"ddoe3fd            ZIdS )a  
Poolformer from MetaFormer is Actually What You Need for Vision https://arxiv.org/abs/2111.11418

IdentityFormer, RandFormer, PoolFormerV2, ConvFormer, and CAFormer
from MetaFormer Baselines for Vision https://arxiv.org/abs/2210.13452

All implemented models support feature extraction and variable input resolution.

Original implementation by Weihao Yu et al.,
adapted for timm by Fredo Guan and Ross Wightman.

Adapted from https://github.com/sail-sg/metaformer, original copyright below
    )OrderedDict)partial)OptionalN)Tensor)FinalIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_DropPathSelectAdaptivePool2d
GroupNorm1	LayerNormLayerNorm2dMlpuse_fused_attn   )build_model_with_cfg)checkpoint_seq)generate_default_cfgsregister_model
MetaFormerc                   ,     e Zd ZdZ	 d fd	Zd Z xZS )Stemzc
    Stem implemented by a layer of convolution.
    Conv2d params constant across all models.
    Nc                     t                                                       t          j        ||ddd          | _        |r ||          nt          j                    | _        d S )N         kernel_sizestridepadding)super__init__nnConv2dconvIdentitynorm)selfin_channelsout_channels
norm_layer	__class__s       R/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/metaformer.pyr$   zStem.__init__7   si     	I
 
 
	 1;MJJ|,,,			    c                 Z    |                      |          }|                     |          }|S N)r'   r)   r*   xs     r/   forwardzStem.forwardG   %    IIaLLIIaLLr0   r2   __name__
__module____qualname____doc__r$   r5   __classcell__r.   s   @r/   r   r   1   sa          	N N N N N N       r0   r   c                   0     e Zd ZdZ	 	 	 d fd	Zd Z xZS )Downsamplingz=
    Downsampling implemented by a layer of convolution.
    r   r   Nc                     t                                                       |r ||          nt          j                    | _        t          j        |||||          | _        d S )Nr   )r#   r$   r%   r(   r)   r&   r'   )r*   r+   r,   r    r!   r"   r-   r.   s          r/   r$   zDownsampling.__init__R   sg     	/9LJJ{+++r{}}	I#
 
 
			r0   c                 Z    |                      |          }|                     |          }|S r2   )r)   r'   r3   s     r/   r5   zDownsampling.forwarde   r6   r0   )r   r   Nr7   r=   s   @r/   r?   r?   M   sa          
 
 
 
 
 
&      r0   r?   c                   *     e Zd ZdZd fd	Zd Z xZS )Scalez2
    Scale vector by element multiplications.
          ?Tc                     t                                                       |r|ddfn|f| _        t          j        |t          j        |          z  |          | _        d S )Nr   requires_grad)r#   r$   shaper%   	Parametertorchonesscale)r*   dim
init_value	trainableuse_nchwr.   s        r/   r$   zScale.__init__p   sX    $,8c1a[[3&
\*uz#">iXXX


r0   c                 F    || j                             | j                  z  S r2   )rL   viewrH   r3   s     r/   r5   zScale.forwardu   s    4:??4:....r0   )rD   TTr7   r=   s   @r/   rC   rC   k   s\         Y Y Y Y Y Y
/ / / / / / /r0   rC   c                   *     e Zd ZdZd fd	Zd Z xZS )SquaredReLUz<
        Squared ReLU: https://arxiv.org/abs/2109.08668
    Fc                 |    t                                                       t          j        |          | _        d S )Ninplace)r#   r$   r%   ReLUrelu)r*   rW   r.   s     r/   r$   zSquaredReLU.__init__~   s1    GG,,,			r0   c                 P    t          j        |                     |                    S r2   )rJ   squarerY   r3   s     r/   r5   zSquaredReLU.forward   s    |DIIaLL)))r0   Fr7   r=   s   @r/   rT   rT   y   sV         - - - - - -* * * * * * *r0   rT   c                   6     e Zd ZdZ	 	 	 	 	 	 d	 fd	Zd Z xZS )
StarReLUz(
    StarReLU: s * relu(x) ** 2 + b
    rD           TNFc                 J   t                                                       || _        t          j        |          | _        t          j        |t          j        d          z  |          | _	        t          j        |t          j        d          z  |          | _
        d S )NrV   r   rF   )r#   r$   rW   r%   rX   rY   rI   rJ   rK   rL   bias)r*   scale_value
bias_valuescale_learnablebias_learnablemoderW   r.   s          r/   r$   zStarReLU.__init__   s     	GG,,,	\+
1"=_]]]
Lejmm!;>ZZZ			r0   c                 R    | j         |                     |          dz  z  | j        z   S )Nr   )rL   rY   ra   r3   s     r/   r5   zStarReLU.forward   s%    zDIIaLLA--	99r0   )rD   r_   TTNFr7   r=   s   @r/   r^   r^      sp           [ [ [ [ [ [: : : : : : :r0   r^   c                   N     e Zd ZU dZee         ed<   	 	 	 	 	 	 d	 fd	Zd Z xZ	S )
	Attentionzl
    Vanilla self-attention from Transformer: https://arxiv.org/abs/1706.03762.
    Modified from timm.
    
fused_attn    NFr_   c                    t                                                       || _        |dz  | _        t	                      | _        |r|n||z  | _        | j        dk    rd| _        | j        | j        z  | _        t          j	        || j        dz  |          | _
        t          j        |          | _        t          j	        | j        ||          | _        t          j        |          | _        d S )Ng      r   r      ra   )r#   r$   head_dimrL   r   rj   	num_headsattention_dimr%   LinearqkvDropout	attn_dropproj	proj_drop)
r*   rM   ro   rp   qkv_biasru   rw   	proj_biaskwargsr.   s
            r/   r$   zAttention.__init__   s     	 %
(**&/DSH_>QDN!^dm;9S$"4q"8xHHHI..Id0#IFFF	I..r0   c                    |j         \  }}}|                     |                              ||d| j        | j                                      ddddd          }|                    d          \  }}}| j        r,t          j	        |||| j
        r| j        j        nd          }nQ||                    dd	          z  | j        z  }	|	                    d	
          }	|                     |	          }	|	|z  }|                    dd                              |||          }|                     |          }|                     |          }|S )Nrm   r   r   r   r   r_   )	dropout_p)rM   )rH   rs   reshaperp   ro   permuteunbindrj   Fscaled_dot_product_attentiontrainingru   p	transposerL   softmaxrv   rw   )
r*   r4   BNCrs   qkvattns
             r/   r5   zAttention.forward   s>   '1ahhqkk!!!Q4>4=IIQQRSUVXY[\^_``**Q--1a? 		.1a.2mC$.**  AA
 B+++tz9D<<B<''D>>$''DqAKK1%%aA..IIaLLNN1r0   )rk   NFr_   r_   F)
r8   r9   r:   r;   r   bool__annotations__r$   r5   r<   r=   s   @r/   ri   ri      s~           d
 / / / / / /8      r0   ri   c                        e Zd Z fdZ xZS )GroupNorm1NoBiasc                      t                      j        |fi | |                    dd          | _        d | _        d S Nepsư>r#   r$   getr   ra   r*   num_channelsrz   r.   s      r/   r$   zGroupNorm1NoBias.__init__   B    00000::eT**			r0   r8   r9   r:   r$   r<   r=   s   @r/   r   r      8                r0   r   c                        e Zd Z fdZ xZS )LayerNorm2dNoBiasc                      t                      j        |fi | |                    dd          | _        d | _        d S r   r   r   s      r/   r$   zLayerNorm2dNoBias.__init__   r   r0   r   r=   s   @r/   r   r      r   r0   r   c                        e Zd Z fdZ xZS )LayerNormNoBiasc                      t                      j        |fi | |                    dd          | _        d | _        d S r   r   r   s      r/   r$   zLayerNormNoBias.__init__   r   r0   r   r=   s   @r/   r   r      r   r0   r   c                   @     e Zd ZdZdeej        dddf fd	Zd Z xZ	S )SepConvz\
    Inverted separable convolution from MobileNetV2: https://arxiv.org/abs/1801.04381.
    r   Fr   rm   c                 Z   t                                                       t          ||z            }	t          j        ||	d|          | _         |            | _        t          j        |	|	|||	|          | _         |            | _        t          j        |	|d|          | _	        d S )Nr   )r    ra   )r    r"   groupsra   )
r#   r$   intr%   r&   pwconv1act1dwconvact2pwconv2)r*   rM   expansion_ratio
act1_layer
act2_layerra   r    r"   rz   mid_channelsr.   s             r/   r$   zSepConv.__init__   s     	?S011ylMMMJLL	i,KLt= = = JLL	ysMMMr0   c                     |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|S r2   )r   r   r   r   r   r3   s     r/   r5   zSepConv.forward	  sR    LLOOIIaLLKKNNIIaLLLLOOr0   )
r8   r9   r:   r;   r^   r%   r(   r$   r5   r<   r=   s   @r/   r   r      sr          {N N N N N N*      r0   r   c                   *     e Zd ZdZd fd	Zd Z xZS )PoolingzT
    Implementation of pooling for PoolFormer: https://arxiv.org/abs/2111.11418
    rm   c                     t                                                       t          j        |d|dz  d          | _        d S )Nr   r   F)r!   r"   count_include_pad)r#   r$   r%   	AvgPool2dpool)r*   	pool_sizerz   r.   s      r/   r$   zPooling.__init__  sE    Laa5R R R			r0   c                 6    |                      |          }||z
  S r2   )r   )r*   r4   ys      r/   r5   zPooling.forward  s    IIaLL1ur0   )rm   r7   r=   s   @r/   r   r     s\         R R R R R R
      r0   r   c                   6     e Zd ZdZddeeddf fd	Zd Z xZS )MlpHeadz MLP classification head
      r   r_   Tc                 L   t                                                       t          ||z            }t          j        |||          | _         |            | _         ||          | _        t          j        |||          | _        t          j	        |          | _
        d S )Nrn   )r#   r$   r   r%   rr   fc1actr)   fc2rt   	head_drop)
r*   rM   num_classes	mlp_ratio	act_layerr-   	drop_ratera   hidden_featuresr.   s
            r/   r$   zMlpHead.__init__%  s     	i#o..9S/===9;;J//	9_kEEEI..r0   c                     |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|S r2   )r   r   r)   r   r   r3   s     r/   r5   zMlpHead.forward7  sT    HHQKKHHQKKIIaLLNN1HHQKKr0   )	r8   r9   r:   r;   rT   r   r$   r5   r<   r=   s   @r/   r   r   !  sj          ! / / / / / /$      r0   r   c            	       <     e Zd ZdZeededddddf	 fd	Zd Z xZ	S )MetaFormerBlockz1
    Implementation of one MetaFormer block.
    Fr_   TNc                 0   t                                                       t          t          ||	|          }t          t          ||
|          } ||          | _         |d||d|| _        |dk    rt          |          nt          j                    | _	        |	
 |            nt          j                    | _
        |

 |            nt          j                    | _         ||          | _        t          |t          d|z            ||||          | _        |dk    rt          |          nt          j                    | _        |	
 |            nt          j                    | _        |

 |            nt          j                    | _        d S )N)rM   rN   rP   )rM   rw   r_   r   )r   ra   dropuse_conv )r#   r$   r   rC   norm1token_mixerr   r%   r(   
drop_path1layer_scale1
res_scale1norm2r   r   mlp
drop_path2layer_scale2
res_scale2)r*   rM   r   mlp_actmlp_biasr-   rw   	drop_pathrP   layer_scale_init_valueres_scale_init_valuerz   ls_layerrs_layerr.   s                 r/   r$   zMetaFormerBlock.__init__E  s~    	5c6LW_```5c6JU]^^^Z__
&;N3)NNvNN1:R(9---R[]]*@*LHHJJJRTR]R_R_(<(H((***bkmmZ__
CLL
 
 
 2;R(9---R[]]*@*LHHJJJRTR]R_R_(<(H((***bkmmr0   c           
         |                      |          |                     |                     |                     |                     |                                        z   }|                     |          |                     |                     |                     | 	                    |                                        z   }|S r2   )
r   r   r   r   r   r   r   r   r   r   r3   s     r/   r5   zMetaFormerBlock.forwardj  s    OOA$$TZZ]]33   OOAHHTZZ]]++   r0   )
r8   r9   r:   r;   r   r^   r   r$   r5   r<   r=   s   @r/   r   r   @  sz           "#'!%#\ #\ #\ #\ #\ #\J      r0   r   c            
       ~     e Zd Zdej        edeeddgdz  ddf
 fd	Zej	        j
        d
d            Zdefd	Z xZS )MetaFormerStager   Fr_   Nc                 t   	
 t                                                       d _        t          t                      _        |k    rt          j                    nt          |ddd|           _	        t          j
        
	 fdt          |          D               _        d S )NFrm   r   r   )r    r!   r"   r-   c                 T    g | ]$}t          d	|         
j        d 
%S ))
rM   r   r   r   r-   rw   r   r   r   rP   r   )r   rP   ).0idp_ratesrz   r   r   r   r-   out_chsrw   r   r*   r   s     r/   
<listcomp>z,MetaFormerStage.__init__.<locals>.<listcomp>  sm     &! &! &!  '6 '
#!qk#9!5]'
 '
 '
 '
 &! &! &!r0   )r#   r$   grad_checkpointing
issubclassri   rP   r%   r(   r?   
downsample
Sequentialrangeblocks)r*   in_chsr   depthr   r   r   downsample_normr-   rw   r   r   r   rz   r.   s   ` ` ``` ``````r/   r$   zMetaFormerStage.__init__|  s      	"'&{I>>> ,2W+<+<"+---,&C
 C
 C
 m &! &! &! &! &! &! &! &! &! &! &! &! &! &! <<&! &! &! "r0   Tc                     || _         d S r2   )r   )r*   enables     r/   set_grad_checkpointingz&MetaFormerStage.set_grad_checkpointing  s    "(r0   r4   c                    |                      |          }|j        \  }}}}| j        s+|                    ||d                              dd          }| j        r4t          j                                        st          | j
        |          }n| 
                    |          }| j        s,|                    dd                              ||||          }|S )Nr~   r   r   )r   rH   rP   r   r   r   rJ   jitis_scriptingr   r   )r*   r4   r   r   HWs         r/   r5   zMetaFormerStage.forward  s    OOAW
1a} 	4		!Q##--a33A" 	59+A+A+C+C 	t{A..AAAA} 	6Aq!!))!Q155Ar0   T)r8   r9   r:   r%   r(   r^   r   r$   rJ   r   ignorer   r   r5   r<   r=   s   @r/   r   r   z  s         '"TAX#'!%+" +" +" +" +" +"Z Y) ) ) )        r0   r   c                        e Zd ZdZdddddeeddddd	d
eeedf fd	Zd Z	e
j        j        dd            Ze
j        j        dej        fd            Zddedee         fdZddedefdZdefdZdefdZ xZS )r   aM   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels.
        num_classes (int): Number of classes for classification head.
        global_pool: Pooling for classifier head.
        depths (list or tuple): Number of blocks at each stage.
        dims (list or tuple): Feature dimension at each stage.
        token_mixers (list, tuple or token_fcn): Token mixer for each stage.
        mlp_act: Activation layer for MLP.
        mlp_bias (boolean): Enable or disable mlp bias term.
        drop_path_rate (float): Stochastic depth rate.
        drop_rate (float): Dropout rate.
        layer_scale_init_values (list, tuple, float or None): Init value for Layer Scale.
            None means not use the layer scale. Form: https://arxiv.org/abs/2103.17239.
        res_scale_init_values (list, tuple, float or None): Init value for res Scale on residual connections.
            None means not use the res scale. From: https://arxiv.org/abs/2110.09456.
        downsample_norm (nn.Module): Norm layer used in stem and downsampling layers.
        norm_layers (list, tuple or norm_fcn): Norm layers for each stage.
        output_norm: Norm layer before classifier head.
        use_mlp_head: Use MLP classification head.
    rm   r   avgr   r      r   @      i@     Fr_   N)NNrD   rD   Tc                    t                                                       || _        |d         | _        || _        || _        t          |          | _        t          |t          t          f          s|g}t          |t          t          f          s|g}t          |t          t          f          s|g| j        z  }t          |t          t          f          s|g| j        z  }t          |t          t          f          s|g| j        z  }t          |t          t          f          s|g| j        z  }d| _        g | _        t          ||d         |          | _        g }|d         }d t          j        d|	t#          |                                        |          D             }t'          | j                  D ]{}|t)          |||         f||         ||         |||
||         ||         ||         |||         d
|gz  }||         }| xj        t+          ||         dd| 	          gz  c_        |t-          j        | | _        |dk    rW| j        r)t3          | j        || j        
          }| j        | _        n:t-          j        | j        |          }| j        | _        nt-          j                    }t-          j        t;          dt=          |          fd || j                  fd|rt-          j        d          nt-          j                    fd| j        rt-          j         |          nt-          j                    fd|fg                    | _!        | "                    | j#                   d S )Nr~   Fr   )r-   c                 6    g | ]}|                                 S r   )tolist)r   r4   s     r/   r   z'MetaFormer.__init__.<locals>.<listcomp>  s     eee1AHHJJeeer0   )
r   r   r   r   rw   r   r   r   r   r-   r   zstages.)num_chs	reductionmoduler   global_pool	pool_typer)   flattenr   r   fc)$r#   r$   r   num_featuresr   use_mlp_headlen
num_stages
isinstancelisttupler   feature_infor   stemrJ   linspacesumsplitr   r   dictr%   r   stagesr   head_hidden_sizerr   r(   r   r   Flattenrt   headapply_init_weights)r*   in_chansr   r  depthsdimstoken_mixersr   r   drop_path_rateproj_drop_rater   layer_scale_init_valuesres_scale_init_valuesr   norm_layersoutput_normr  rz   r  prev_dimr   r   finalr.   s                           r/   r$   zMetaFormer.__init__  s   * 	& H"(f++ &4-00 	XF$u.. 	6D,u66 	<(>DO;L+e}55 	:&-$/9K1D%=AA 	R'>&?$/&Q#/$?? 	N%:$;do$M!"'G&
 
 
	 7eeq.#f++(V(V(\(\]c(d(deeet'' 	\ 	\AQ Qi(O!(!!'>q'A%:1%= /&q>     F AwH$tAw!MVWMM"Z"Z"Z![[mV, ??  : 1;$.YYY(,(9%%	$"3[AA(,(9%%KMMEM+0;GGGH[[!2334G
1"+--Hd.?RRZ	***R[]]S5M/
 # #  	 	

4%&&&&&r0   c                     t          |t          j        t          j        f          rDt	          |j        d           |j        )t          j                            |j        d           d S d S d S )Ng{Gz?)stdr   )	r  r%   r&   rr   r   weightra   init	constant_)r*   ms     r/   r$  zMetaFormer._init_weights:  sj    a")RY/00 	-!(,,,,v!!!!&!,,,,,	- 	-!!r0   c                 T    || _         | j        D ]}|                    |           d S )N)r   )r   r  r   )r*   r   stages      r/   r   z!MetaFormer.set_grad_checkpointing@  s?    "([ 	8 	8E(((7777	8 	8r0   returnc                     | j         j        S r2   )r"  r  )r*   s    r/   get_classifierzMetaFormer.get_classifierF  s    y|r0   r   r  c                 l   |Mt          |          | j        _        |rt          j        d          nt          j                    | j        _        |dk    r?| j        rt          | j	        || j
                  }n.t          j        | j	        |          }nt          j                    }|| j        _        d S )Nr  r   r   r  )r   r"  r  r%   r!  r(   r  r  r   r  r   rr   r  )r*   r   r  r0  s       r/   reset_classifierzMetaFormer.reset_classifierJ  s    "$8;$O$O$ODI!1< O
1"+--DI??  B 1;$.YYY	$"3[AAKMME	r0   r4   
pre_logitsc                    | j                             |          }| j                             |          }| j                             |          }| j                             |          }|r|n| j                             |          S r2   )r"  r  r)   r  r   r  )r*   r4   r>  s      r/   forward_headzMetaFormer.forward_headW  sm    I!!!$$INN1Ia  INN13qqDILLOO3r0   c                     |                      |          }| j        r4t          j                                        st          | j        |          }n|                     |          }|S r2   )r  r   rJ   r   r   r   r  r3   s     r/   forward_featureszMetaFormer.forward_features_  sX    IIaLL" 	59+A+A+C+C 	t{A..AAAAr0   c                 Z    |                      |          }|                     |          }|S r2   )rB  r@  r3   s     r/   r5   zMetaFormer.forwardg  s-    !!!$$a  r0   r   r2   r\   )r8   r9   r:   r;   r   r^   r   r   r$   r$  rJ   r   r   r   r%   Moduler;  r   r   strr=  r   r   r@  rB  r5   r<   r=   s   @r/   r   r     so        6 $ $("8-)#%_' _' _' _' _' _'B- - - Y8 8 8 8
 Y	     C hsm    4 4f 4$ 4 4 4 4&            r0   c                 x   d| v r| S dd l }i }d| v }|                                }|                                 D ]\  }}|r|                    dd|          }|                    dd          }|                    dd	          }|                    d
d          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd|          }|                    dd          }|                    dd          }|                    dd|          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd |          }|                    d!d"|          }|j        ||         k    rP|                                ||                                         k    r |                    ||         j                  }|||<   |S )#Nzstem.conv.weightr   znetwork.0.0.mlp.fc1.weightzlayer_scale_([0-9]+)zlayer_scale\1.scalez	network.1zdownsample_layers.1z	network.3zdownsample_layers.2z	network.5zdownsample_layers.3z	network.2z	network.4z	network.6networkr  zdownsample_layers.([0-9]+)zstages.\1.downsamplezdownsample.projzdownsample.convzpatch_embed.projzpatch_embed.convz([0-9]+).([0-9]+)z\1.blocks.\2zstages.0.downsamplepatch_embedr  	post_normr)   pre_normz^headhead.fcz^normz	head.norm)re
state_dictitemssubreplacerH   numelr   )rM  modelrL  out_dictis_poolformerv1model_state_dictr   r   s           r/   checkpoint_filter_fnrV  n  s2   Z''IIIH2j@O''))  ""  1 	/.0FJJA		+'<==A		+'<==A		+'<==A		+{33A		+{33A		+{33A		)X..AFF02I1MMII'):;;II(*<==FF'!<<II+];;IImV,,IIk6**IIj&))FF8Y**FF8[!,,7&q)))aggii;KA;N;T;T;V;V.V.V		*1-344AOr0   Fc           	          t          d t          |                    dd                    D                       }|                    d|          }t	          t
          | |ft          t          d|          d|}|S )Nc              3       K   | ]	\  }}|V  
d S r2   r   )r   r   _s      r/   	<genexpr>z%_create_metaformer.<locals>.<genexpr>  s&      \\da\\\\\\r0   r&  r   out_indicesT)flatten_sequentialr[  )pretrained_filter_fnfeature_cfg)r  	enumerater   popr   r   rV  r  )variant
pretrainedrz   default_out_indicesr[  rR  s         r/   _create_metaformerrd    s    \\i

8\8Z8Z.[.[\\\\\**],?@@K  2DkJJJ   E Lr0    c                 4    | dddddt           t          ddd
|S )	Nr   )rm      rg  )r   r   rD   bicubicrK  z	stem.conv)
urlr   
input_sizer   crop_pctinterpolationmeanr2  
classifier
first_convr   )ri  rz   s     r/   _cfgrp    s5    =v)%.B{   r0   zpoolformer_s12.sail_in1kztimm/g?)	hf_hub_idrk  zpoolformer_s24.sail_in1kzpoolformer_s36.sail_in1kzpoolformer_m36.sail_in1kgffffff?zpoolformer_m48.sail_in1kzpoolformerv2_s12.sail_in1k)rq  zpoolformerv2_s24.sail_in1kzpoolformerv2_s36.sail_in1kzpoolformerv2_m36.sail_in1kzpoolformerv2_m48.sail_in1kzconvformer_s18.sail_in1kzhead.fc.fc2)rq  rn  zconvformer_s18.sail_in1k_384)rm     rr  )   rs  )rq  rn  rj  r   z!convformer_s18.sail_in22k_ft_in1kz%convformer_s18.sail_in22k_ft_in1k_384zconvformer_s18.sail_in22kiQU  )rq  rn  r   zconvformer_s36.sail_in1kzconvformer_s36.sail_in1k_384z!convformer_s36.sail_in22k_ft_in1kz%convformer_s36.sail_in22k_ft_in1k_384zconvformer_s36.sail_in22kzconvformer_m36.sail_in1kzconvformer_m36.sail_in1k_384z!convformer_m36.sail_in22k_ft_in1kz%convformer_m36.sail_in22k_ft_in1k_384zconvformer_m36.sail_in22kzconvformer_b36.sail_in1kzconvformer_b36.sail_in1k_384z!convformer_b36.sail_in22k_ft_in1kz%convformer_b36.sail_in22k_ft_in1k_384zconvformer_b36.sail_in22kzcaformer_s18.sail_in1kzcaformer_s18.sail_in1k_384zcaformer_s18.sail_in22k_ft_in1kz#caformer_s18.sail_in22k_ft_in1k_384zcaformer_s18.sail_in22kzcaformer_s36.sail_in1kzcaformer_s36.sail_in1k_384zcaformer_s36.sail_in22k_ft_in1kz#caformer_s36.sail_in22k_ft_in1k_384zcaformer_s36.sail_in22kzcaformer_m36.sail_in1kzcaformer_m36.sail_in1k_384zcaformer_m36.sail_in22k_ft_in1kz#caformer_m36.sail_in22k_ft_in1k_384zcaformer_m36.sail_in22kzcaformer_b36.sail_in1kzcaformer_b36.sail_in1k_384zcaformer_b36.sail_in22k_ft_in1kz#caformer_b36.sail_in22k_ft_in1k_384zcaformer_b36.sail_in22kr9  c                 p    t          d	g dg dd t          j        dt          dd dd	|}t	          d
d| i|S )Nr   r  Th㈵>F	r&  r'  r   r   r   r-  r+  r,  r  poolformer_s12rb  r   )rw  r  r%   GELUr   rd  rb  rz   model_kwargss      r/   rw  rw  G  sg     
||    $"
 
 
 
L VV:VVVVr0   c                 p    t          d	g dg dd t          j        dt          dd dd	|}t	          d
d| i|S )Nr   r   rs  r   r  Tru  Frv  poolformer_s24rb  r   )r~  rx  rz  s      r/   r~  r~  W  g     
}}    $"
 
 
 
L VV:VVVVr0   c                 p    t          d	g dg dd t          j        dt          dd dd	|}t	          d
d| i|S )Nr  r     r  r  Tr   Frv  poolformer_s36rb  r   )r  rx  rz  s      r/   r  r  g  r  r0   c                 p    t          d	g dg dd t          j        dt          dd dd	|}t	          d
d| i|S )Nr  `      rr     Tr   Frv  poolformer_m36rb  r   )r  rx  rz  s      r/   r  r  w  r  r0   c                 p    t          d	g dg dd t          j        dt          dd dd	|}t	          d
d| i|S )N   r     r  r  Tr   Frv  poolformer_m48rb  r   )r  rx  rz  s      r/   r  r    r  r0   c                 R    t          dg dg dt          dd|}t          dd| i|S )	Nr   r  Fr&  r'  r-  r  poolformerv2_s12rb  r   )r  r  r   rd  rz  s      r/   r  r    sV     ||   $	 
  L XXZX<XXXr0   c                 R    t          dg dg dt          dd|}t          dd| i|S )	Nr}  r  Fr  poolformerv2_s24rb  r   )r  r  rz  s      r/   r  r    V     }}   $	 
  L XXZX<XXXr0   c                 R    t          dg dg dt          dd|}t          dd| i|S )	Nr  r  Fr  poolformerv2_s36rb  r   )r  r  rz  s      r/   r  r    r  r0   c                 R    t          dg dg dt          dd|}t          dd| i|S )	Nr  r  Fr  poolformerv2_m36rb  r   )r  r  rz  s      r/   r  r    r  r0   c                 R    t          dg dg dt          dd|}t          dd| i|S )	Nr  r  Fr  poolformerv2_m48rb  r   )r  r  rz  s      r/   r  r    r  r0   c                 \    t          dg dg dt          t          d|}t          dd| i|S )Nrm   rm   	   rm   r  r&  r'  r(  r-  convformer_s18rb  r   )r  r  r   r   rd  rz  s      r/   r  r    sV     ||   %	 
  L VV:VVVVr0   c                 \    t          dg dg dt          t          d|}t          dd| i|S )Nrm   rs  r  rm   r  r  convformer_s36rb  r   )r  r  rz  s      r/   r  r    V     ~~   %	 
  L VV:VVVVr0   c                 \    t          dg dg dt          t          d|}t          dd| i|S )Nr  r  r  rr  i@  r  convformer_m36rb  r   )r  r  rz  s      r/   r  r    r  r0   c                 \    t          dg dg dt          t          d|}t          dd| i|S )Nr  r     r  r  r  convformer_b36rb  r   )r  r  rz  s      r/   r  r    sV     ~~!!!%	 
  L VV:VVVVr0   c           	          t          dg dg dt          t          t          t          gt          gdz  t          gdz  z   d|}t          dd| i|S )	Nr  r  r   r  caformer_s18rb  r   )r  r  r   ri   r   r   rd  rz  s      r/   r  r    st     ||   w	9=&'!+.?!.CC	 
  L TTT|TTTr0   c           	          t          dg dg dt          t          t          t          gt          gdz  t          gdz  z   d|}t          dd| i|S )	Nr  r  r   r  caformer_s36rb  r   )r  r  rz  s      r/   r  r    t     ~~   w	9=&'!+.?!.CC	 
  L TTT|TTTr0   c           	          t          dg dg dt          t          t          t          gt          gdz  t          gdz  z   d|}t          dd| i|S )	Nr  r  r   r  caformer_m36rb  r   )r  r  rz  s      r/   r  r    r  r0   c           	          t          dg dg dt          t          t          t          gt          gdz  t          gdz  z   d|}t          dd| i|S )	Nr  r  r   r  caformer_b36rb  r   )r  r  rz  s      r/   r  r    st     ~~!!!w	9=&'!+.?!.CC	 
  L TTT|TTTr0   r\   )re  )Jr;   collectionsr   	functoolsr   typingr   rJ   torch.nnr%   torch.nn.functional
functionalr   r   	torch.jitr   	timm.datar	   r
   timm.layersr   r   r   r   r   r   r   r   _builderr   _manipulater   	_registryr   r   __all__rD  r   r?   rC   rT   r^   ri   r   r   r   r   r   r   r   r   r   rV  rd  rp  default_cfgsrw  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r0   r/   <module>r     s   8 $ # # # # #                                         A A A A A A A A                    * * * * * * ' ' ' ' ' ' < < < < < < < <.    29   8    29   </ / / / /BI / / /
* 
* 
* 
* 
*") 
* 
* 
*: : : : :ry : : :06 6 6 6 6	 6 6 6x    z              bl            bi      F    bi       bi   >7 7 7 7 7bi 7 7 7tB B B B Bbi B B BJk k k k k k k k^" " "J        %$ V&! ! !V& ! ! !	V& ! ! !V& ! ! !V& ! ! !V&" !$$"9"9"9#V&$ !$$"9"9"9%V&& !$$"9"9"9'V&( !$$"9"9"9)V&* !$$"9"9"9+V&.  !" !" !"/V&4 #DD ]h%P %P %P5V&: ( *" *" *";V&@ ,TT ]h.P .P .PAV&F   e"5 "5 "5GV&N  !" !" !"OV&T #DD ]h%P %P %PUV& V&Z ( *" *" *"[V&` ,TT ]h.P .P .PaV&f   e"5 "5 "5gV&n  !" !" !"oV&t #DD ]h%P %P %PuV&z ( *" *" *"{V&@ ,TT ]h.P .P .PAV&F   e"5 "5 "5GV&N  !" !" !"OV&T #DD ]h%P %P %PUV&Z ( *" *" *"[V&` ,TT ]h.P .P .PaV&f   e"5 "5 "5gV&n dd " " "oV&t !$$ ]h#P #P #PuV&z &tt (" (" ("{V&@ *44 ]h,P ,P ,PAV& V& V&F tt e 5  5  5GV&N dd " " "OV&T !$$ ]h#P #P #PUV&Z &tt (" (" ("[V&` *44 ]h,P ,P ,PaV&f tt e 5  5  5gV&n dd " " "oV&t !$$ ]h#P #P #PuV&z &tt (" (" ("{V&@ *44 ]h,P ,P ,PAV&F tt e 5  5  5GV&N dd " " "OV&T !$$ ]h#P #P #PUV&Z &tt (" (" ("[V&` *44 ]h,P ,P ,PaV&f tt e 5  5  5gV& V& V Vr W W* W W W W W W* W W W W W W* W W W W W W* W W W W W W* W W W W Y YJ Y Y Y Y Y YJ Y Y Y Y Y YJ Y Y Y Y Y YJ Y Y Y Y Y YJ Y Y Y Y W W* W W W W W W* W W W W W W* W W W W W W* W W W W U U
 U U U U U U
 U U U U U U
 U U U U U U
 U U U U U Ur0   