
    Ngoa                        d Z ddlZddlmZ ddlmZmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ dgZdddddZ dddddZ!dddddZ" G d de	j#                  Z$ G d dej	        j#                  Z% G d dej	        j#                  Z& G d  d!ej	        j#                  Z' G d" d#e	j#                  Z( G d$ d%e	j#                  Z) G d& d'e	j#                  Z* G d( d)e	j#                  Z+ G d* d+e	j,                  Z- G d, d-e	j#                  Z. G d. de	j#                  Z/d;d0Z0 e e0d12           e0d12           e0d12           e0d12          d3          Z1d<d5Z2ed<d6e/fd7            Z3ed<d6e/fd8            Z4ed<d6e/fd9            Z5ed<d6e/fd:            Z6dS )=aJ   EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
    N)partial)DictOptionalIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)create_conv2dcreate_norm_layerget_act_layerget_norm_layerConvNormAct)DropPathtrunc_normal_	to_2tuple	to_ntuplendgrid   )build_model_with_cfg)checkpoint_seq)generate_default_cfgsregister_modelEfficientFormerV2)(   P        )    @      i   )r   0   x      )r   r    `      )LS2S1S0)   r)      
   )   r,         )   r/   	      )   r2   r1   r,   )r,   r,   )r,   r,   r,   r,   r/   r/   r/   r/   r/   r/   r/   r,   r,   r,   r,   )
r,   r,   r,   r/   r/   r/   r/   r,   r,   r,   )r,   r,   )r,   r,   r/   r/   r/   r/   r/   r/   r,   r,   r,   r,   )r,   r,   r/   r/   r/   r/   r,   r,   )r,   r,   )	r,   r,   r/   r/   r/   r/   r,   r,   r,   )r,   r,   r/   r/   r,   r,   )r,   r,   )r,   r/   r/   r/   r,   r,   )r,   r/   r/   r,   c                   6     e Zd Z	 	 	 	 	 	 	 	 d fd	Zd Z xZS )	ConvNormr    Tbatchnorm2dNc           
          |
pi }
t          t          |                                            t          ||||||||          | _        t          |	|fi |
| _        d S )N)stridepaddingdilationgroupsbias)superr4   __init__r	   convr
   bn)selfin_channelsout_channelskernel_sizer8   r9   r:   r;   r<   
norm_layernorm_kwargs	__class__s              Z/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/efficientformer_v2.pyr>   zConvNorm.__init__9   sx     "'Rh&&(((!	
 	
 	
	 $JLLLL    c                 Z    |                      |          }|                     |          }|S N)r?   r@   rA   xs     rH   forwardzConvNorm.forwardT   s%    IIaLLGGAJJrI   )r   r   r5   r   r   Tr6   N__name__
__module____qualname__r>   rN   __classcell__rG   s   @rH   r4   r4   8   sl        
 $M M M M M M6      rI   r4   c                        e Zd ZU eeej        f         ed<   dddddej	        df fd	Z
 ej                    d fd
	            Zdej        dej        fdZd Z xZS )Attention2dattention_bias_cacher   r   r.   r,      Nc           	      F   t                                                       || _        |dz  | _        || _        t          |          }Pt          fd|D                       }t          ||d|          | _        t          j
        d          | _        nd | _        d | _        || _        | j        d         | j        d         z  | _        t          ||z            | _        t          ||z            |z  | _        || _        | j        | j        z  }t          ||          | _        t          ||          | _        t          || j                  | _        t          | j        | j        d| j        	          | _        t          j        | j        | j        d
          | _        t          j        | j        | j        d
          | _         |            | _        t          | j        |d          | _        t7          j        t;          t7          j        | j        d                   t7          j        | j        d                                                           d          }	|	dd d d f         |	dd d d f         z
                                   }
|
d         | j        d         z  |
d         z   }
t6          j	        !                    t7          j"        || j                            | _#        | $                    dt7          j%        |
          d           i | _&        d S )N      c                 >    g | ]}t          j        |z            S  mathceil).0rr8   s     rH   
<listcomp>z(Attention2d.__init__.<locals>.<listcomp>n   s'    JJJ!	!f* 5 5JJJrI   r/   rD   r8   r;   bilinear)scale_factormoder   r   )rD   r;   )rD   .attention_bias_idxsF
persistent)'r=   r>   	num_headsscalekey_dimr   tupler4   stride_convnnUpsampleupsample
resolutionNintddh
attn_ratioqkvv_localConv2dtalking_head1talking_head2actprojtorchstackr   arangeflattenabs	Parameterzerosattention_biasesregister_buffer
LongTensorrW   )rA   dimrl   rj   rw   rr   	act_layerr8   khposrel_posrG   s          `   rH   r>   zAttention2d.__init__]   s    	"_
z**
JJJJzJJJKKJ'SaWZ[[[DKV*MMMDMM#D DM$#doa&88Z')**j7*++i7$\DN*#r""#r""#tw''aPPPYt~t~STUUUYt~t~STUUU9;;TWc1--	k&doa.@!A!A5<PTP_`aPbCcCcddeemmnoppsAAAt|$s3aaa<'88==??1: 22gaj@ % 2 25;y$&3Q3Q R R2E4DW4M4MZ_```$&!!!rI   Tc                 r    t                                          |           |r| j        ri | _        d S d S d S rK   r=   trainrW   rA   rf   rG   s     rH   r   zAttention2d.train   M    d 	+D- 	+(*D%%%	+ 	+ 	+ 	+rI   devicereturnc                     t           j                                        s| j        r| j        d d | j        f         S t          |          }|| j        vr| j        d d | j        f         | j        |<   | j        |         S rK   r   jit
is_tracingtrainingr   rg   strrW   rA   r   
device_keys      rH   get_attention_biasesz Attention2d.get_attention_biases       9!! 	9T] 	9(D,D)DEEVJ!:::8<8MaaaQUQiNi8j)*5,Z88rI   c                 D   |j         \  }}}}| j        |                     |          }|                     |                              || j        d| j                                      dddd          }|                     |                              || j        d| j                                      dddd          }|                     |          }| 	                    |          }	|                    || j        d| j                                      dddd          }||z  | j
        z  }
|
|                     |j                  z   }
|                     |
          }
|
                    d          }
|                     |
          }
|
|z                      dd          }|                    || j        | j        d         | j        d                   |	z   }| j        |                     |          }|                     |          }|                     |          }|S Nr   r   r/   r2   r   )shapern   rx   reshaperj   rs   permutery   rz   r{   rk   r   r   r}   softmaxr~   	transposerv   rr   rq   r   r   rA   rM   BCHWrx   ry   rz   r{   attns              rH   rN   zAttention2d.forward   s   W
1a'  ##AFF1IIaTV<<DDQ1aPPFF1IIaTV<<DDQ1aPPFF1II,,q//IIaTV44<<Q1aHHA#d//999!!$''|||##!!$''AX  A&&IIa$/!"4doa6HIIGS=$a  AHHQKKIIaLLrI   TrP   rQ   rR   r   r   r   Tensor__annotations__ro   GELUr>   no_gradr   r   r   rN   rS   rT   s   @rH   rV   rV   Z   s         sEL01111 g.' .' .' .' .' .'` U]__+ + + + + _+
95< 9EL 9 9 9 9      rI   rV   c                   $     e Zd Z fdZd Z xZS )LocalGlobalQueryc                     t                                                       t          j        ddd          | _        t          j        ||ddd|          | _        t          ||d          | _        d S )Nr   r2   r   r/   )rD   r8   r9   r;   )	r=   r>   ro   	AvgPool2dpoolr|   localr4   r   )rA   in_dimout_dimrG   s      rH   r>   zLocalGlobalQuery.__init__   se    LAq))	Yvv1QPQZ`aaa
VWa00			rI   c                     |                      |          }|                     |          }||z   }|                     |          }|S rK   )r   r   r   )rA   rM   local_qpool_qrx   s        rH   rN   zLocalGlobalQuery.forward   s>    **Q--1fIIaLLrI   rO   rT   s   @rH   r   r      sG        1 1 1 1 1      rI   r   c                        e Zd ZU eeej        f         ed<   ddddddej	        f fd	Z
 ej                    d fd
	            Zdej        dej        fdZd Z xZS )Attention2dDownsamplerW   r      r.   r,   rX   Nc           
      @   t                                                       || _        |dz  | _        || _        t          |          | _        t          d | j        D                       | _        | j        d         | j        d         z  | _	        | j        d         | j        d         z  | _
        t          ||z            | _        t          ||z            |z  | _        || _        |p|| _        | j        | j        z  }t!          ||          | _        t%          ||d          | _        t%          || j        d          | _        t%          | j        | j        dd| j                  | _         |            | _        t%          | j        | j        d          | _        t1          j        t5          j        || j	                            | _        t5          j        t=          t5          j        | j        d                   t5          j        | j        d                                                            d          }	t5          j        t=          t5          j        d| j        d         d          t5          j        d| j        d         d                                                   d          }
|
d	d d d f         |	d	d d d f         z
  !                                }|d         | j        d         z  |d         z   }| "                    d
|d           i | _#        d S )NrZ   c                 <    g | ]}t          j        |d z            S r2   r]   r`   ra   s     rH   rb   z2Attention2dDownsample.__init__.<locals>.<listcomp>   s&    !L!L!Lq$)AE"2"2!L!L!LrI   r   r   r/   r2   rc   )step.rg   Frh   )$r=   r>   rj   rk   rl   r   rr   rm   resolution2rs   N2rt   ru   rv   rw   r   r   rx   r4   ry   rz   r{   r   r   ro   r   r   r   r   r   r   r   r   r   r   rW   )rA   r   rl   rj   rw   rr   r   r   r   k_posq_posr   rG   s               rH   r>   zAttention2dDownsample.__init__   s    	"_
#J// !L!LDO!L!L!LMM#doa&88"1%(8(;;Z')**j7*++i7$~#\DN*!#r**#r1%%#tw**aRVRYZZZ9;;TWdlA66	 "U[DF-K-K L LF5<0B#C#CU\RVRabcRdEeEeffggoopqrrFLDOA.Q777LDOA.Q777
 
   71:: 	 aaa&sD!!!|)<<AACC1: 22gaj@2GNNN$&!!!rI   Tc                 r    t                                          |           |r| j        ri | _        d S d S d S rK   r   r   s     rH   r   zAttention2dDownsample.train   r   rI   r   r   c                     t           j                                        s| j        r| j        d d | j        f         S t          |          }|| j        vr| j        d d | j        f         | j        |<   | j        |         S rK   r   r   s      rH   r   z*Attention2dDownsample.get_attention_biases   r   rI   c                    |j         \  }}}}|                     |                              || j        d| j                                      dddd          }|                     |                              || j        d| j                                      dddd          }|                     |          }| 	                    |          }	|                    || j        d| j                                      dddd          }||z  | j
        z  }
|
|                     |j                  z   }
|
                    d          }
|
|z                      dd          }|                    || j        | j        d         | j        d                   |	z   }|                     |          }|                     |          }|S r   )r   rx   r   rj   r   r   ry   rs   rz   r{   rk   r   r   r   r   rv   r   r   r   r   s              rH   rN   zAttention2dDownsample.forward  s{   W
1aFF1IIaTW==EEaAqQQFF1IIaTV<<DDQ1aPPFF1II,,q//IIaTV44<<Q1aHHA#d//999|||##AX  A&&IIa$"21"5t7G7JKKgUHHQKKIIaLLrI   r   r   rT   s   @rH   r   r      s         sEL01111 g+' +' +' +' +' +'Z U]__+ + + + + _+
95< 9EL 9 9 9 9      rI   r   c                   H     e Zd Zdddddej        ej        f fd	Zd Z xZS )
Downsampler/   r2   r   rX   Fc
                 F   t                                                       t          |          }t          |          }t          |          }|	pt          j                    }	t          ||||||	          | _        |rt          ||||          | _        d S d | _        d S )N)rD   r8   r9   rE   )r   r   rr   r   )	r=   r>   r   ro   Identityr4   r?   r   r   )rA   in_chsout_chsrD   r8   r9   rr   use_attnr   rE   rG   s             rH   r>   zDownsample.__init__  s     	,,6""G$$02;==
#!
 
 
	  	-%#	  DIII DIIIrI   c                 n    |                      |          }| j        |                     |          |z   S |S rK   )r?   r   )rA   rM   outs      rH   rN   zDownsample.forward?  s3    iill9 99Q<<#%%
rI   	rP   rQ   rR   ro   r   BatchNorm2dr>   rN   rS   rT   s   @rH   r   r     sh        
 g~# # # # # #J      rI   r   c                   J     e Zd ZdZddej        ej        ddf fd	Zd Z xZ	S )ConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    N        Fc           	         t                                                       |p|}|p|}t          ||dd||          | _        |rt          ||d|d||          | _        nt          j                    | _        t          j        |          | _        t          ||d|          | _
        t          j        |          | _        d S )Nr   T)r<   rE   r   r/   )r;   r<   rE   r   )rE   )r=   r>   r   fc1midro   r   Dropoutdrop1r4   fc2drop2)	rA   in_featureshidden_featuresout_featuresr   rE   dropmid_convrG   s	           rH   r>   zConvMlpWithNorm.__init__L  s     	#2{)8[!*	C C C  	%"!&TjT]_ _ _DHH {}}DHZ%%
O\1TTTZ%%


rI   c                     |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|S rK   )r   r   r   r   r   rL   s     rH   rN   zConvMlpWithNorm.forwardf  sR    HHQKKHHQKKJJqMMHHQKKJJqMMrI   )
rP   rQ   rR   __doc__ro   r   r   r>   rN   rS   rT   s   @rH   r   r   F  sn          !g~& & & & & &4      rI   r   c                   &     e Zd Zd fd	Zd Z xZS )LayerScale2dh㈵>Fc                     t                                                       || _        t          j        |t          j        |          z            | _        d S rK   )r=   r>   inplacero   r   r   onesgamma)rA   r   init_valuesr   rG   s       rH   r>   zLayerScale2d.__init__p  sB    \+
3"?@@


rI   c                 ~    | j                             dddd          }| j        r|                    |          n||z  S )Nr   r   )r   viewr   mul_)rA   rM   r   s      rH   rN   zLayerScale2d.forwardu  s:    
2q!,, $;qvve}}}!e);rI   )r   FrO   rT   s   @rH   r   r   o  sR        A A A A A A
< < < < < < <rI   r   c            	       L     e Zd Zdej        ej        ddddddf	 fd	Zd Z xZS )	EfficientFormerV2Block      @r   r   rX   NTc                 X   t                                                       |
rpt          ||||	          | _        |t	          ||          nt          j                    | _        |dk    rt          |          nt          j                    | _	        nd | _        d | _        d | _	        t          |t          ||z            |||d          | _        |t	          ||          nt          j                    | _        |dk    rt          |          nt          j                    | _        d S )N)rr   r   r8   r   T)r   r   r   rE   r   r   )r=   r>   rV   token_mixerr   ro   r   ls1r   
drop_path1r   rt   mlpls2
drop_path2)rA   r   	mlp_ratior   rE   	proj_drop	drop_pathlayer_scale_init_valuerr   r8   r   rG   s              rH   r>   zEfficientFormerV2Block.__init__{  sF    	 	#*%#	     D 1G0R $+- - -XZXcXeXe H5>^^hy111DOO#DDH"DO"i00!
 
 
 -C,N  ') ) )TVT_TaTa 	1:R(9---R[]]rI   c                    | j         >||                     |                     |                      |                              z   }||                     |                     |                     |                              z   }|S rK   )r   r   r   r   r   r   rL   s     rH   rN   zEfficientFormerV2Block.forward  sl    'DOODHHT-=-=a-@-@$A$ABBBA! 5 5666rI   r   rT   s   @rH   r   r   z  st         g~#'(S (S (S (S (S (ST      rI   r   c                   8     e Zd Zej        ej        f fd	Z xZS )Stem4c           
          t                                                       d| _        t          ||dz  dddd||          | _        t          |dz  |dddd||          | _        d S )Nr,   r2   r/   r   T)rD   r8   r9   r<   rE   r   )r=   r>   r8   r   conv1conv2)rA   r   r   r   rE   rG   s        rH   r>   zStem4.__init__  s~     GqLa14!Y
 
 

 !qL'qAD!Y
 
 



rI   )rP   rQ   rR   ro   r   r   r>   rS   rT   s   @rH   r  r    sF        24'bn 

 

 

 

 

 

 

 

 

 

rI   r  c                   R     e Zd Zddddddddddej        ej        f fd		Zd
 Z xZS )EfficientFormerV2StagerX   TNFr   r   r   r   c                    t                                                       d| _         t          |          |
          }
t	          |          }|r6t          ||||||          | _        |}t          d |D                       }n ||k    sJ t          j	                    | _        g }t          |          D ];}||	z
  dz
  }t          ||||
|         |o||k    |||         |||
  
        }||gz  }<t          j        | | _        d S )NF)r   rr   rE   r   c                 <    g | ]}t          j        |d z            S r   r]   r   s     rH   rb   z3EfficientFormerV2Stage.__init__.<locals>.<listcomp>  s&    EEEQ	!a% 0 0EEErI   r   )	rr   r8   r  r   r  r  r  r   rE   )r=   r>   grad_checkpointingr   r   r   
downsamplerm   ro   r   ranger   
Sequentialblocks)rA   r   dim_outdepthrr   r  block_stridedownsample_use_attnblock_use_attnnum_vitr  r  r  r  r   rE   r  	block_idx
remain_idxbrG   s                       rH   r>   zEfficientFormerV2Stage.__init__  sE   & 	"'$Ie$$Y//	z**
 	,(,%%#  DO CEE*EEEFFJJ'>>>> kmmDOu 	 	I1,J&%##I.'BI
,B##I.'=#%  A qcMFFmV,rI   c                     |                      |          }| j        r4t          j                                        st          | j        |          }n|                     |          }|S rK   )r  r  r   r   is_scriptingr   r  rL   s     rH   rN   zEfficientFormerV2Stage.forward  sZ    OOA" 	59+A+A+C+C 	t{A..AAAArI   r   rT   s   @rH   r  r    sw          % #'g~!7- 7- 7- 7- 7- 7-r      rI   r  c                   f    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Zd Zej        j        d             Zej        j        dd            Z	ej        j        dd            Z
ej        j        dej        fd            Zd dedee         fdZej        j        dd            Zd ZddefdZd Z xZS )!r   r/   r"   avgNr,   r6   r   gelu  r   r   Tc                    t                                                       |dv sJ || _        || _        g | _        t          |          }t          t          |          |	          }t          |
          }
t          ||d         |
|          | _
        |d         }dt          |          }d t          j        d|t          |                                        |          D             }|pddt          |          d	z
  z  z   } t!          |          |          }g }t#          |          D ]}t%          fd
|D                       }t'          |||         ||         |||         |dk    rdnd |dk    |dk    |||         |||         ||
|          }||         rdz  ||         }| xj        t)          |d|           gz  c_        |                    |           t-          j        | | _        |d         x| _        | _         ||d                   | _        t-          j        |          | _        |dk    rt-          j        |d         |          nt-          j                    | _         || _!        | j!        r:|dk    rt-          j        |d         |          nt-          j                    | _"        nd | _"        | #                    | j$                   d| _%        d S )N)r   r5   )epsr   )r   rE   r,   c                 6    g | ]}|                                 S r\   )tolist)r`   rM   s     rH   rb   z.EfficientFormerV2.__init__.<locals>.<listcomp>!  s     ```aqxxzz```rI   Fr   r   c                 >    g | ]}t          j        |z            S r\   r]   )r`   sr8   s     rH   rb   z.EfficientFormerV2.__init__.<locals>.<listcomp>&  s'    $M$M$MqTYq6z%:%:$M$M$MrI   r2   r/   )r  rr   r  r  r  r  r  r  r  r  r  r   rE   zstages.)num_chs	reductionmoduler   F)&r=   r>   num_classesglobal_poolfeature_infor   r   r   r   r  stemlenr   linspacesumsplitr   r  rm   r  dictappendro   r  stagesnum_featureshead_hidden_sizenormr   	head_dropLinearr   headdist	head_distapplyinit_weightsdistilled_training)rA   depthsin_chansimg_sizer.  
embed_dimsdownsamples
mlp_ratiosrE   norm_epsr   r-  	drop_rateproj_drop_ratedrop_path_rater  r  distillationprev_dim
num_stagesdprr7  icurr_resolutionstager8   rG   s                            @rH   r>   zEfficientFormerV2.__init__  s
   ( 	k))))&&X&&^J77XFFF
!),,	(JqMYS]^^^	a=[[
``5>!^S[[#Q#Q#W#WX^#_#_```!KX3v;;?0K%K*Yz**:66
z"" 	! 	!A#$M$M$M$MH$M$M$MNNO*1Qi*&q>"#q&&QQd$%F Av$Q-(a&'=#%  E" 1~ !!!}H$x6R_\]R_R_"`"`"`!aaMM%    mV, 5?rNBD1Jz"~..	I..>IAooBIjnk:::SUS^S`S`	 	9 	"GRUVRYz"~{CCC\^\g\i\iDNN!DN

4$%%%"'rI   c                     t          |t          j                  rDt          |j        d           |j        )t          j                            |j        d           d S d S d S )N{Gz?)stdr   )
isinstancero   r<  r   weightr<   init	constant_)rA   ms     rH   rA  zEfficientFormerV2.init_weightsN  sc    a## 	-!(,,,,v!!!!&!,,,,,	- 	-!!rI   c                 >    d |                                  D             S )Nc                      h | ]\  }}d |v 	|S )r   r\   )r`   ry   _s      rH   	<setcomp>z4EfficientFormerV2.no_weight_decay.<locals>.<setcomp>V  s'    QQQda9Kq9P9P9P9P9PrI   )named_parametersrA   s    rH   no_weight_decayz!EfficientFormerV2.no_weight_decayT  s"    QQd3355QQQQrI   Fc                 ,    t          dddg          }|S )Nz^stem)z^stages\.(\d+)N)z^norm)i )r0  r  )r5  )rA   coarsematchers      rH   group_matcherzEfficientFormerV2.group_matcherX  s)    -/CD
 
 
 rI   c                 (    | j         D ]	}||_        
d S rK   )r7  r  )rA   enabler)  s      rH   set_grad_checkpointingz(EfficientFormerV2.set_grad_checkpointing`  s(     	* 	*A#)A  	* 	*rI   r   c                     | j         | j        fS rK   r=  r?  ra  s    rH   get_classifierz EfficientFormerV2.get_classifiere  s    y$.((rI   r-  r.  c                    || _         ||| _        |dk    rt          j        | j        |          nt          j                    | _        |dk    rt          j        | j        |          nt          j                    | _        d S )Nr   )r-  r.  ro   r<  r8  r   r=  r?  )rA   r-  r.  s      rH   reset_classifierz"EfficientFormerV2.reset_classifieri  sw    &"*DALqBId/===VXVaVcVc	FQTUoo4#4kBBB[][f[h[hrI   c                     || _         d S rK   )rB  )rA   rh  s     rH   set_distilled_trainingz(EfficientFormerV2.set_distilled_trainingp  s    "(rI   c                     |                      |          }|                     |          }|                     |          }|S rK   )r0  r7  r:  rL   s     rH   forward_featuresz"EfficientFormerV2.forward_featurest  s4    IIaLLKKNNIIaLLrI   
pre_logitsc                 :   | j         dk    r|                    d          }|                     |          }|r|S |                     |          |                     |          }}| j        r)| j        r"t          j        	                                s||fS ||z   dz  S )Nr   )r2   r/   r   r2   )
r.  meanr;  r=  r?  rB  r   r   r   r  )rA   rM   rs  x_dists       rH   forward_headzEfficientFormerV2.forward_headz  s    u$$6""ANN1 	HIIaLL$.."3"36" 	$t} 	$UY=S=S=U=U 	$f9 J!##rI   c                 Z    |                      |          }|                     |          }|S rK   )rr  rw  rL   s     rH   rN   zEfficientFormerV2.forward  s-    !!!$$a  rI   )r/   r"   r   NNr,   r6   r   r!  r"  r   r   r   r   r   Tr'  r   rK   )rP   rQ   rR   r>   rA  r   r   ignorerb  rf  ri  ro   Modulerl  rt   r   r   rn  rp  rr  boolrw  rN   rS   rT   s   @rH   r   r     s        $#'%L( L( L( L( L( L(^- - - YR R R Y    Y* * * * Y)	 ) ) ) )i iC ihsm i i i i Y) ) ) )  $ $$ $ $ $ $      rI   r5   c                 6    | ddd dddt           t          ddd|S )	Nr"  )r/   r"   r"   Tgffffff?bicubicrk  zstem.conv1.conv)urlr-  
input_size	pool_sizefixed_input_sizecrop_pctinterpolationru  rV  
classifier
first_convr   )r~  kwargss     rH   _cfgr    s9    =tae)%.B+;L   rI   ztimm/)	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kFc                 |    |                     dd          }t          t          | |fdt          d|          i|}|S )Nout_indices)r   r   r2   r/   feature_cfgT)flatten_sequentialr  )popr   r   r5  )variant
pretrainedr  r  models        rH   _create_efficientformerv2r    sV    **]L99K 7J DkJJJ  E LrI   r   c           	          t          t          d         t          d         ddt          d                   }t	          dd| it          |fi |S )Nr(   r2   r   rC  rF  r  rL  rH  efficientformerv2_s0r  )r  r5  EfficientFormer_depthEfficientFormer_width EfficientFormer_expansion_ratiosr  r  r  
model_argss      rH   r  r    b    $T*(.3D9  J %qq
qVZ[eVpVpioVpVpqqqrI   c           	          t          t          d         t          d         ddt          d                   }t	          dd| it          |fi |S )Nr'   r2   r   r  efficientformerv2_s1r  )r  r  r  s      rH   r  r    r  rI   c           	          t          t          d         t          d         ddt          d                   }t	          dd| it          |fi |S )Nr&   r,   rU  r  efficientformerv2_s2r  )r  r  r  s      rH   r  r    sb    $T*(.3D9  J %qq
qVZ[eVpVpioVpVpqqqrI   c           	          t          t          d         t          d         ddt          d                   }t	          dd| it          |fi |S )Nr%   r1   g?r  efficientformerv2_lr  )r  r  r  s      rH   r  r    sb    $S)(-3C8  J %ppzpUYZdUoUohnUoUoppprI   )r5   r'  )7r   r^   	functoolsr   typingr   r   r   torch.nnro   	timm.datar   r   timm.layersr	   r
   r   r   r   r   r   r   r   r   _builderr   _manipulater   	_registryr   r   __all__r  r  r  rz  r4   rV   r   r   r   r   r   r   r  r  r  r   r  default_cfgsr  r  r  r  r  r\   rI   rH   <module>r     s            ! ! ! ! ! ! ! !        A A A A A A A A d d d d d d d d d d d d d d M M M M M M M M M M M M M M * * * * * * ' ' ' ' ' ' < < < < < < < < 
 



	   



	   
_
P
A
2	$ $      ry   DZ Z Z Z Z%(/ Z Z Zz    ux   P P P P PEHO P P Pf* * * * * * * *Z& & & & &bi & & &R< < < < <29 < < </ / / / /RY / / /d
 
 
 
 
BM 
 
 
A A A A ARY A A AHM M M M M	 M M M`    %$+/4, , , ,04, , , ,04, , , +/$+ + +& &       r r8I r r r r r r8I r r r r r r8I r r r r q q7H q q q q q qrI   