
    NgO@                        d Z dgZddlmZ ddlZddlmZ ddlmZm	Z	 ddl
mZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ  G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z  G d dej                  Z!d-dZ" e e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !           e"d !          d"          Z#d.d$Z$ed.d%            Z%ed.d&            Z&ed.d'            Z'ed.d(            Z(ed.d)            Z)ed.d*            Z*ed.d+            Z+ed.d,            Z,dS )/a   RepViT

Paper: `RepViT: Revisiting Mobile CNN From ViT Perspective`
    - https://arxiv.org/abs/2307.09283

@misc{wang2023repvit,
      title={RepViT: Revisiting Mobile CNN From ViT Perspective}, 
      author={Ao Wang and Hui Chen and Zijia Lin and Hengjun Pu and Guiguang Ding},
      year={2023},
      eprint={2307.09283},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Adapted from official impl at https://github.com/jameslahm/RepViT
RepVit    )OptionalNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)SqueezeExcitetrunc_normal_	to_ntuple	to_2tuple   )build_model_with_cfg)checkpoint_seq)register_modelgenerate_default_cfgsc                   P     e Zd Zd fd	Z ej                    d             Z xZS )ConvNormr   r   c	                    t                                                       |                     dt          j        |||||||d                     |                     dt          j        |                     t          j                            | j        j	        |           t          j                            | j        j
        d           d S )NcFbiasbnr   )super__init__
add_modulennConv2dBatchNorm2dinit	constant_r   weightr   )
selfin_dimout_dimksstridepaddilationgroupsbn_weight_init	__class__s
            N/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/repvit.pyr   zConvNorm.__init__    s    RYvwFCSY`efffgggbnW55666
$'..999
$',*****    c           
         | j                                         \  }}|j        |j        |j        z   dz  z  }|j        |d d d d d f         z  }|j        |j        |j        z  |j        |j        z   dz  z  z
  }t          j        |	                    d          | j
        j        z  |	                    d          |j        dd          | j
        j        | j
        j        | j
        j        | j
        j        |j        j                  }|j        j                            |           |j        j                            |           |S )N      ?r   r      )r%   paddingr'   r(   device)_modulesvaluesr    running_varepsr   running_meanr   r   sizer   r(   shaper%   r0   r'   r1   datacopy_)r!   r   r   wbms         r+   fusezConvNorm.fuse'   s   $$&&2I"&0S88HqD$,--Gbo	1R^bf5LQT4TTTIFF1II%FF1IIGABBK6=FNV_6=8?	
 	
 	
 	
A	!r,   )r   r   r   r   r   r   __name__
__module____qualname__r   torchno_gradr>   __classcell__r*   s   @r+   r   r      s`        + + + + + + U]__  _    r,   r   c                   P     e Zd Zd fd	Z ej                    d             Z xZS )
NormLinearT{Gz?c                    t                                                       |                     dt          j        |                     |                     dt          j        |||                     t          | j        j        |           |r,t          j	        
                    | j        j        d           d S d S )Nr   lr   )stdr   )r   r   r   r   BatchNorm1dLinearr	   rK   r    r   r   r   )r!   r"   r#   r   rL   r*   s        r+   r   zNormLinear.__init__=   s    bnV44555RYvwTBBBCCCdfm---- 	.Gdfk1-----	. 	.r,   c                    | j                                         \  }}|j        |j        |j        z   dz  z  }|j        | j        j        | j        j        z  |j        |j        z   dz  z  z
  }|j        |d d d f         z  }|j        || j        j        j	        z  }n4|j        |d d d f         z  
                    d          | j        j        z   }t          j        |                    d          |                    d          |j        j                  }|j        j                            |           |j        j                            |           |S )Nr.   r   r   r1   )r2   r3   r    r4   r5   r   r   r6   rK   Tviewr   rN   r7   r1   r9   r:   )r!   r   rK   r;   r<   r=   s         r+   r>   zNormLinear.fuseE   s&   $$&&AI"&0S88Gdg*TW^;r~PRPV?V[^>^^^Hqqqqz!6>DFMO#AAAaaagJ&,,R0046;>AIaffQii18?CCC	A	!r,   )TrI   r?   rF   s   @r+   rH   rH   <   s`        . . . . . . U]__  _    r,   rH   c                   V     e Zd Zd fd	Zd Z ej                    d             Z xZS )RepVggDwFc                 ~   t                                                       t          |||d|dz
  dz  |          | _        |r3t          ||ddd|          | _        t          j                    | _        n8t          j        ||ddd|          | _        t          j	        |          | _        || _
        || _        d S )Nr   r/   r(   r   )r   r   r   convconv1r   Identityr   r   r   dimlegacy)r!   edkernel_sizer\   r*   s       r+   r   zRepVggDw.__init__V   s    R[!kAo!5KTVWWW	 	)!"b!Q"===DJkmmDGG2r1a2>>>DJnR((DGr,   c                     |                      |                     |          |                     |          z   |z             S N)r   rX   rY   r!   xs     r+   forwardzRepVggDw.forwardc   s2    wwtyy||djjmm3a7888r,   c           	         | j                                         }| j        r| j                                        }n| j        }|j        }|j        }|j        }|j        }t          j                            |g d          }t          j                            t          j
        |j        d         |j        d         dd|j                  g d          }||z   |z   }||z   }	|j        j                            |           |j        j                            |	           | j        s| j        }
|
j        |
j        |
j        z   dz  z  }|j        |d d d d d f         z  }|
j        |j        |
j        z
  |
j        z  |
j        |
j        z   dz  z  z   }|j        j                            |           |j        j                            |           |S )N)r   r   r   r   r   r   rQ   r.   )rX   r>   r\   rY   r    r   r   
functionalr&   rC   onesr8   r1   r9   r:   r   r4   r5   r6   )r!   rX   rY   conv_wconv_bconv1_wconv1_bidentityfinal_conv_wfinal_conv_br   r;   r<   s                r+   r>   zRepVggDw.fusef   s   y~~; 	JOO%%EEJE,*-##G\\\::=$$Jw}Q'q)91aWWWYeYeYe
 
 '(2'|,,,	\***{ 	$B	R^bf4<<Aa4t 344A49r6")Cr~XZX^G^cfFfffAK""1%%%IN  ###r,   F	r@   rA   rB   r   rc   rC   rD   r>   rE   rF   s   @r+   rU   rU   U   so             9 9 9 U]__    _         r,   rU   c                   $     e Zd Z fdZd Z xZS )	RepVitMlpc                     t                                                       t          ||ddd          | _         |            | _        t          ||dddd          | _        d S )Nr   r   )r)   )r   r   r   rY   actconv2)r!   r"   
hidden_dim	act_layerr*   s       r+   r   zRepVitMlp.__init__   s]    fj!Q::
9;;j&!Q!LLL


r,   c                 x    |                      |                     |                     |                              S r`   )rt   rs   rY   ra   s     r+   rc   zRepVitMlp.forward   s*    zz$((4::a==11222r,   r@   rA   rB   r   rc   rE   rF   s   @r+   rq   rq      sL        M M M M M3 3 3 3 3 3 3r,   rq   c                   &     e Zd Zd fd	Zd Z xZS )RepViTBlockFc                    t          t          |                                            t          |||          | _        |rt          |d          nt          j                    | _        t          |||z  |          | _
        d S )Ng      ?)r   rz   r   rU   token_mixerr   r   rZ   serq   channel_mixer)r!   r"   	mlp_ratior^   use_serv   r\   r*   s          r+   r   zRepViTBlock.__init__   ss    k4  ))+++#FK@@17J----R[]]&vv	/A9MMr,   c                     |                      |          }|                     |          }|}|                     |          }||z   S r`   )r|   r}   r~   r!   rb   rk   s      r+   rc   zRepViTBlock.forward   sF    QGGAJJq!!!|r,   rn   rx   rF   s   @r+   rz   rz      sR        N N N N N N      r,   rz   c                   $     e Zd Z fdZd Z xZS )
RepVitStemc                     t                                                       t          ||dz  ddd          | _         |            | _        t          |dz  |ddd          | _        d| _        d S )Nr/      r      )r   r   r   rY   act1rt   r%   )r!   in_chsout_chsrv   r*   s       r+   r   zRepVitStem.__init__   sg    fglAq!<<
IKK	glGQ1==
r,   c                 x    |                      |                     |                     |                              S r`   )rt   r   rY   ra   s     r+   rc   zRepVitStem.forward   s*    zz$))DJJqMM22333r,   rx   rF   s   @r+   r   r      sG            4 4 4 4 4 4 4r,   r   c                   &     e Zd Zd fd	Zd Z xZS )RepVitDownsampleFc                    t                                                       t          |||d||          | _        t	          |||d|dz
  dz  |          | _        t	          ||dd          | _        t          |||z  |          | _        d S )NF)r   rv   r\   r/   r   rW   )	r   r   rz   	pre_blockr   spatial_downsamplechannel_downsamplerq   ffn)r!   r"   r   r#   r^   rv   r\   r*   s          r+   r   zRepVitDownsample.__init__   s    $VYE]fouvvv"*66;KZ[O`aKajp"q"q"q"*67Aq"A"AWg	&99EEr,   c                     |                      |          }|                     |          }|                     |          }|}|                     |          }||z   S r`   )r   r   r   r   r   s      r+   rc   zRepVitDownsample.forward   sW    NN1##A&&##A&&HHQKK8|r,   rn   rx   rF   s   @r+   r   r      sR        F F F F F F      r,   r   c                   V     e Zd Zd fd	Zd Z ej                    d             Z xZS )RepVitClassifierF        c                 d   t                                                       t          j        |          | _        |dk    rt          ||          nt          j                    | _        || _        d| _	        || _
        |r0|dk    rt          ||          nt          j                    | _        d S d S )Nr   F)r   r   r   Dropout	head_droprH   rZ   headdistillationdistilled_trainingnum_classes	head_dist)r!   r[   r   r   dropr*   s        r+   r   zRepVitClassifier.__init__   s    D))4?!OOJsK000	("'& 	`=H1__Z[999RTR]R_R_DNNN	` 	`r,   c                 ,   |                      |          }| j        rb|                     |          |                     |          }}| j        r)| j        r"t          j                                        s||fS ||z   dz  S |                     |          }|S )Nr/   )	r   r   r   r   trainingr   rC   jitis_scripting)r!   rb   x1x2s       r+   rc   zRepVitClassifier.forward   s    NN1 	YYq\\4>>!#4#4B} %!8 %AWAWAYAY %2vR1}$		!AHr,   c                 L   | j         dk    st          j                    S | j                                        }| j        re| j                                        }|xj        |j        z  c_        |xj        |j        z  c_        |xj        dz  c_        |xj        dz  c_        |S |S )Nr   r/   )	r   r   rZ   r   r>   r   r   r    r   )r!   r   r   s      r+   r>   zRepVitClassifier.fuse   s    !##;== y~~ 	++--IKK9++KKII'IIKK1KKIINIIKKr,   )Fr   ro   rF   s   @r+   r   r      su        ` ` ` ` ` `
 
 
 U]__  _    r,   r   c                   &     e Zd Zd fd	Zd Z xZS )RepVitStager   TFc	                 f   t                                                       |rt          ||||||          | _        n ||k    sJ t	          j                    | _        g }	d}
t          |          D ],}|	                    t          ||||
||                     |
 }
-t	          j	        |	 | _
        d S )NT)r   r   r   
downsampler   rZ   rangeappendrz   
Sequentialblocks)r!   r"   r#   depthr   rv   r^   r   r\   r   r   _r*   s               r+   r   zRepVitStage.__init__   s     	,.vy';XacijjDOOW$$$$ kmmDOu 	  	 AMM+gy+vyZ`aabbbZFFmV,r,   c                 Z    |                      |          }|                     |          }|S r`   )r   r   ra   s     r+   rc   zRepVitStage.forward   s'    OOAKKNNr,   )r   TFrx   rF   s   @r+   r   r      sL        - - - - - -       r,   r   c                   l    e Zd Zddddddddej        dd	d
f fd	Zej        j        dd            Z	ej        j        dd            Z
ej        j        dej        fd            Zddedee         defdZej        j        dd            Zd ZddefdZd Z ej                    d             Z xZS )r   r      )0   )r/   r/   avg  Tr   Fc                    t          t          |                                            d| _        || _        || _        || _        |d         }t          |||	          | _        | j        j	        }t          d t          t          |          t          |                    D                       }t          |          } t          |          |          }g | _        g }t!          |          D ]}|dk    rdnd}|                    t%          |||         ||         ||         |	|||                     |rdnd|z  }t          fd|D                       }| xj        t'          ||         |d	| 
          gz  c_        ||         }t)          j        | | _        |d         x| _        | _        t)          j        |          | _        t7          |d         ||
          | _        d S )NFr   c                     g | ]
\  }}||z  S  r   ).0ips      r+   
<listcomp>z#RepVit.__init__.<locals>.<listcomp>  s     [[[tq!AF[[[r,   T)r   rv   r^   r   r\   r/   r   c                 &    g | ]}|d z
  z  d z   S )r   r   )r   rstage_strides     r+   r   z#RepVit.__init__.<locals>.<listcomp>0  s'    PPPQ< 7! ;PPPr,   zstages.)num_chs	reductionmodulerP   )r   r   r   grad_checkpointingglobal_pool	embed_dimr   r   stemr%   tuplezipr   lenr
   feature_infor   r   r   dictr   r   stagesnum_featureshead_hidden_sizer   r   r   r   )r!   in_chansimg_sizer   r   r   r   r^   r   rv   r   	drop_rater\   r"   r%   
resolution
num_stages
mlp_ratiosr   r   r   r   r*   s                        @r+   r   zRepVit.__init__  s
    	fd$$&&&"'&"&1x;;	![[s9X3F3F	RXHYHY/Z/Z[[[\\
^^
*Yz**955
z"" 	" 	"A!"aUJMMaL!H(m' +)!	 	 	   !+111Ll"FPPPPZPPPQQJ$y|vVc`aVcVc"d"d"d!eeq\FFmV,4=bMAD1I..$Yr]KNN			r,   c                 ,    t          dddg          }|S )Nz^stem)z^blocks\.(\d+)N)z^norm)i )r   r   )r   )r!   coarsematchers      r+   group_matcherzRepVit.group_matcher9  s     H.GI]-^___r,   c                     || _         d S r`   )r   r!   enables     r+   set_grad_checkpointingzRepVit.set_grad_checkpointing>  s    "(r,   returnc                     | j         S r`   )r   )r!   s    r+   get_classifierzRepVit.get_classifierB  s
    yr,   Nr   r   r   c                 h    || _         ||| _        t          | j        d         ||          | _        d S )NrP   )r   r   r   r   r   )r!   r   r   r   s       r+   reset_classifierzRepVit.reset_classifierF  s6    &"*D$T^B%7lSS			r,   c                     || j         _        d S r`   )r   r   r   s     r+   set_distilled_trainingzRepVit.set_distilled_trainingL  s    '-	$$$r,   c                     |                      |          }| j        r4t          j                                        st          | j        |          }n|                     |          }|S r`   )r   r   rC   r   r   r   r   ra   s     r+   forward_featureszRepVit.forward_featuresP  sX    IIaLL" 	59+A+A+C+C 	t{A..AAAAr,   
pre_logitsc                     | j         dk    r|                    dd          }|                     |          }|r|S |                     |          S )Nr   )r/   r   F)keepdim)r   meanr   r   )r!   rb   r   s      r+   forward_headzRepVit.forward_headX  sS    u$$vu--ANN1 	Hyy||r,   c                 Z    |                      |          }|                     |          }|S r`   )r   r   ra   s     r+   rc   zRepVit.forward`  s-    !!!$$a  r,   c                 (    fd |            d S )Nc                     |                                  D ]Q\  }}t          |d          r1|                                }t          | ||            |           F |           Rd S )Nr>   )named_childrenhasattrr>   setattr)net
child_namechildfusedfuse_childrens       r+   r   z"RepVit.fuse.<locals>.fuse_childreng  s    %(%7%7%9%9 ) )!
E5&)) )!JJLLECU333!M%((((!M%(((() )r,   r   )r!   r   s    @r+   r>   zRepVit.fusee  s4    	) 	) 	) 	) 	) 	dr,   rn   )T)NF)r@   rA   rB   r   GELUr   rC   r   ignorer   r   Moduler   intr   strboolr   r   r   r   rc   rD   r>   rE   rF   s   @r+   r   r      s        '6O 6O 6O 6O 6O 6Op Y    Y) ) ) ) Y	    T TC Thsm Tbf T T T T Y. . . .   $      
 U]__
 
 _
 
 
 
 
r,    c                 4    | dddddt           t          ddd
|S )	Nr   )r   r   r   )   r   gffffff?bicubiczstem.conv1.c)zhead.head.lzhead.head_dist.l)
urlr   
input_size	pool_sizecrop_pctinterpolationr   rL   
first_conv
classifierr   )r  kwargss     r+   _cfgr	  s  s9    #"%#$9   r,   ztimm/)	hf_hub_id)zrepvit_m1.dist_in1kzrepvit_m2.dist_in1kzrepvit_m3.dist_in1kzrepvit_m0_9.dist_300e_in1kzrepvit_m0_9.dist_450e_in1kzrepvit_m1_0.dist_300e_in1kzrepvit_m1_0.dist_450e_in1kzrepvit_m1_1.dist_300e_in1kzrepvit_m1_1.dist_450e_in1kzrepvit_m1_5.dist_300e_in1kzrepvit_m1_5.dist_450e_in1kzrepvit_m2_3.dist_300e_in1kzrepvit_m2_3.dist_450e_in1kFc                 |    |                     dd          }t          t          | |fdt          d|          i|}|S )Nout_indices)r   r   r/   r   feature_cfgT)flatten_sequentialr  )popr   r   r   )variant
pretrainedr  r  models        r+   _create_repvitr    sZ    **]L99K   DkJJJ	
  E Lr,   c           	      X    t          ddd          }t          dd| it          |fi |S )z&
    Constructs a RepViT-M1 model
    r   `      i  r/   r/      r/   Tr   r   r\   	repvit_m1r  )r  r   r  r  r  
model_argss      r+   r  r    sD    
  2-PTUUUJ[[*[Z@Z@ZSY@Z@Z[[[r,   c           	      X    t          ddd          }t          dd| it          |fi |S )z&
    Constructs a RepViT-M2 model
    @         i   r/   r/      r/   Tr  	repvit_m2r  )r&  r  r  s      r+   r&  r&    D    
  3=QUVVVJ[[*[Z@Z@ZSY@Z@Z[[[r,   c           	      X    t          ddd          }t          dd| it          |fi |S )z&
    Constructs a RepViT-M3 model
    r   )r   r      r/   Tr  	repvit_m3r  )r*  r  r  s      r+   r*  r*    r'  r,   c           	      V    t          dd          }t          dd| it          |fi |S )z(
    Constructs a RepViT-M0.9 model
    r  r  r   r   repvit_m0_9r  )r-  r  r  s      r+   r-  r-    sA    
  2-HHHJ]]J]$zB\B\U[B\B\]]]r,   c           	      V    t          dd          }t          dd| it          |fi |S )z(
    Constructs a RepViT-M1.0 model
    )8   p   r   i  r  r,  repvit_m1_0r  )r1  r  r  s      r+   r1  r1    A    
  3=IIIJ]]J]$zB\B\U[B\B\]]]r,   c           	      V    t          dd          }t          dd| it          |fi |S )z(
    Constructs a RepViT-M1.1 model
    r   r$  r,  repvit_m1_1r  )r4  r  r  s      r+   r4  r4    r2  r,   c           	      V    t          dd          }t          dd| it          |fi |S )z(
    Constructs a RepViT-M1.5 model
    r   )r   r      r   r,  repvit_m1_5r  )r7  r  r  s      r+   r7  r7    r2  r,   c           	      V    t          dd          }t          dd| it          |fi |S )z(
    Constructs a RepViT-M2.3 model
    )P      i@  i  )   r;  "   r/   r,  repvit_m2_3r  )r=  r  r  s      r+   r=  r=    r2  r,   )r   rn   )-__doc____all__typingr   rC   torch.nnr   	timm.datar   r   timm.layersr   r	   r
   r   _builderr   _manipulater   	_registryr   r   r   r   rH   r   rU   rq   rz   r   r   r   r   r   r	  default_cfgsr  r  r&  r*  r-  r1  r4  r7  r=  r   r,   r+   <module>rH     s7   " *              A A A A A A A A J J J J J J J J J J J J * * * * * * ' ' ' ' ' ' < < < < < < < <    r}   :       22 2 2 2 2ry 2 2 2j3 3 3 3 3	 3 3 3    ")    	4 	4 	4 	4 	4 	4 	4 	4    ry   "$ $ $ $ $ry $ $ $N    ")   .p p p p pRY p p pf     %$#t 
  
  
  $t 
  
  
  $t 
  
  
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
 '+d'
 '
 '
K( (* *Z	 	 	 	 \ \ \ \ \ \ \ \ \ \ \ \ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^r,   