
    Ngf                         d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z
 ddlmZmZmZmZmZmZmZmZmZmZmZ g dZeej                 Zdee         defd	Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z  G d dej                  Z! G d dej                  Z" G d de          Z# G d dej                  Z$dS )z[ EfficientNet, MobileNetV3, etc Blocks

Hacked together by / Copyright 2019, Ross Wightman
    )CallableDictOptionalTypeN)
functional)create_conv2dDropPathmake_divisiblecreate_act_layer	create_aa	to_2tuple	LayerTypeConvNormActget_norm_act_layerMultiQueryAttention2dAttention2d)SqueezeExcite	ConvBnActDepthwiseSeparableConvInvertedResidualCondConvResidualEdgeResidualUniversalInvertedResidualMobileAttention
group_sizechannelsc                 *    | sdS || z  dk    sJ || z  S )N   r    )r   r   s     \/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/_efficientnet_blocks.py
num_groupsr!      s2     &q *$)))):%%    c                        e Zd ZdZddej        ej        ddfdedede	e         de
de
d	e	e
         d
e	e         f fdZd Z xZS )r   a   Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family

    Args:
        in_chs (int): input channels to layer
        rd_ratio (float): ratio of squeeze reduction
        act_layer (nn.Module): activation layer of containing block
        gate_layer (Callable): attention gate function
        force_act_layer (nn.Module): override block's activation fn if this is set/bound
        rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs
    g      ?Nin_chsrd_ratiord_channels	act_layer
gate_layerforce_act_layerrd_round_fnc                 X   t          t          |                                            ||pt          } |||z            }|p|}t	          j        ||dd          | _        t          |d          | _        t	          j        ||dd          | _	        t          |          | _
        d S )Nr   T)biasinplace)superr   __init__roundnnConv2dconv_reducer   act1conv_expandgate)	selfr$   r%   r&   r'   r(   r)   r*   	__class__s	           r    r0   zSqueezeExcite.__init__+   s     	mT""++---%.K%+fx&788K#0y	9V[!$GGG$Y===	9[&!$GGG$Z00			r"   c                     |                     dd          }|                     |          }|                     |          }|                     |          }||                     |          z  S )N)      T)keepdim)meanr4   r5   r6   r7   )r8   xx_ses      r    forwardzSqueezeExcite.forward?   sa    vvfdv++%%yy%%499T??""r"   )__name__
__module____qualname____doc__r2   ReLUSigmoidintfloatr   r   r   r0   rA   __classcell__r9   s   @r    r   r      s        	 	 #)-#%7$&J37.21 11 1 "#	1
 !1 "1 &i01 "(+1 1 1 1 1 1(# # # # # # #r"   r   c                        e Zd ZdZdddddej        ej        ddf	ded	ed
ededededede	de
de
dee
         def fdZd Zd Z xZS )r   z@ Conv + Norm Layer + Activation w/ optional skip connection
    r   r    FN        r$   out_chskernel_sizestridedilationr   pad_typeskipr'   
norm_layeraa_layerdrop_path_ratec           	         t          t          |                                            t          |
|	          }t	          ||          }|o|dk    o||k    | _        |d uo|dk    }t          ||||rdn||||          | _         ||d          | _        t          ||||          | _
        |rt          |          nt          j                    | _        d S )Nr   rQ   rR   groupspaddingTr-   r   rQ   enable)r/   r   r0   r   r!   has_skipr   convbn1r   aar	   r2   Identity	drop_path)r8   r$   rO   rP   rQ   rR   r   rS   rT   r'   rU   rV   rW   norm_act_layerrZ   use_aar9   s                   r    r0   zConvBnAct.__init__J   s     	i'')))+J	BBJ//B1B71B%4&1*!G[*11Ffh@ @ @	 ">'4888HwvfUUU5CV.111r"   c                 |    |dk    rt          dd| j        j                  S t          d| j        j                  S )N	expansionr`   rA   module	hook_typenum_chsrM   ri   rk   )dictr_   out_channelsr8   locations     r    feature_infozConvBnAct.feature_infog   s?    {""u	49CYZZZZr49+ABBBBr"   c                     |}|                      |          }|                     |          }|                     |          }| j        r|                     |          |z   }|S N)r_   r`   ra   r^   rc   r8   r?   shortcuts      r    rA   zConvBnAct.forwardm   sY    IIaLLHHQKKGGAJJ= 	-q!!H,Ar"   rB   rC   rD   rE   r2   rF   BatchNorm2drH   strboolr   r   rI   r0   rq   rA   rJ   rK   s   @r    r   r   G   s         #%7$&N,0$&W WW W 	W
 W W W W W !W "W y)W "W W W W W W:C C C      r"   r   c            !            e Zd ZdZdddddddddej        ej        dddfd	ed
edededededede	dede	dede
de
dee
         dee         def  fdZd Zd Z xZS )r   z Depthwise-separable block
    Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion
    (factor of 1.0). This is an alternative to having a IR with an optional first pw conv.
    r<   r   rM   Fr   NrN   r$   rO   dw_kernel_sizerQ   rR   r   rS   noskippw_kernel_sizepw_acts2dr'   rU   rV   se_layerrW   c           	      D   t          t          |                                            t          ||          }|dk    o||k    o| | _        |
| _        |d uo|dk    }|dk    rSt          |dz            }t          ||ddd          | _         |||          | _	        |dz   dz  }|dk    rdn|}|}d}nd | _        d | _	        |}t          ||          }t          ||||rdn||||          | _         ||d	          | _        t          ||||
          | _        |r |||          nt          j                    | _        t          |||	|          | _         ||d| j                  | _        |rt)          |          nt          j                    | _        d S )Nr      r;   samerP   rQ   r[   F)rQ   rR   r[   rZ   Tr-   r\   r'   r[   )r.   	apply_act)r/   r   r0   r   r^   
has_pw_actrH   r   conv_s2dbn_s2dr!   conv_dwr`   r   ra   r2   rb   seconv_pwbn2r	   rc   )r8   r$   rO   r{   rQ   rR   r   rS   r|   r}   r~   r   r'   rU   rV   r   rW   rd   re   sd_chsdw_pad_typerZ   r9   s                         r    r0   zDepthwiseSeparableConv.__init__|   s   & 	$d++44666+J	BB1:7):JF
 %4&1* !88!__F)&&aPQ[abbbDM(.88DK,q0Q6N$2a$7$7&&XKFFF DMDK"KJ//$FN*11F{6C C C ">&$777HwvfUUU <DV((6Y7777$VWnhWWW!>'44?SSS5CV.111r"   c                 |    |dk    rt          dd| j        j                  S t          d| j        j                  S Nrg   r   forward_prerh   rM   rl   rm   r   in_channelsrn   ro   s     r    rq   z#DepthwiseSeparableConv.feature_info   ?    {""yM4<Kcddddr4<+DEEEEr"   c                    |}| j         *|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        r| 	                    |          |z   }|S rs   )
r   r   r   r`   ra   r   r   r   r^   rc   rt   s      r    rA   zDepthwiseSeparableConv.forward   s    =$a  AAALLOOHHQKKGGAJJGGAJJLLOOHHQKK= 	-q!!H,Ar"   )rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   r   r   
ModuleTyperI   r0   rq   rA   rJ   rK   s   @r    r   r   w   s]         #$ "# #%7$&N,0-1$&#5W 5W5W 5W  	5W
 5W 5W 5W 5W 5W  5W 5W 5W !5W "5W y)5W  z*!5W" "#5W 5W 5W 5W 5W 5WnF F F      r"   r   c            %            e Zd ZdZddddddddddej        ej        dddd	fd
ededededededede	de
dedededededee         dee         dee         de
f$ fdZd Zd Z xZS )r   a   Inverted residual block w/ optional SE

    Originally used in MobileNet-V2 - https://arxiv.org/abs/1801.04381v4, this layer is often
    referred to as 'MBConv' for (Mobile inverted bottleneck conv) and is also used in
      * MNasNet - https://arxiv.org/abs/1807.11626
      * EfficientNet - https://arxiv.org/abs/1905.11946
      * MobileNet-V3 - https://arxiv.org/abs/1905.02244
    r<   r   rM   F      ?r   NrN   r$   rO   r{   rQ   rR   r   rS   r|   	exp_ratioexp_kernel_sizer}   r   r'   rU   rV   r   conv_kwargsrW   c                    t          t          |                                            t          ||          }|pi }||k    o|dk    o| | _        |d uo|dk    }|dk    rSt          |dz            }t          ||ddd          | _         |||          | _        |dz   dz  }|dk    rdn|}|}d}nd | _        d | _        |}t          ||	z            }t          ||          }t          |||
fd|i|| _         ||d	          | _        t          |||f|rdn||||d
|| _         ||d	          | _        t          ||||          | _        |r |||          nt#          j                    | _        t          |||fd|i|| _         ||d          | _        |rt-          |          nt#          j                    | _        d S )Nr   r   r;   r   r   Fr[   Tr-   rY   r\   r   r   )r/   r   r0   r   r^   rH   r   r   r   r
   r!   r   r`   r   r   r   ra   r2   rb   r   conv_pwlbn3r	   rc   )r8   r$   rO   r{   rQ   rR   r   rS   r|   r   r   r}   r   r'   rU   rV   r   r   rW   rd   re   r   r   mid_chsrZ   r9   s                            r    r0   zInvertedResidual.__init__   sL   * 	%%..000+J	BB!'R7*:v{JF
%4&1* !88!__F)&&aPQ[abbbDM(.88DK,q0Q6N$2a$7$7&&XKFFF DMDK"K )!344J00 %VWoggxg[fgg!>'4888 %WnR*11FfkR R FQR R ">'4888HwvfUUU =EW((7i8888"+-- &gwhhPXh\ghh!>'U;;;5CV.111r"   c                 |    |dk    rt          dd| j        j                  S t          d| j        j                  S Nrg   r   r   rh   rM   rl   rm   r   r   rn   ro   s     r    rq   zInvertedResidual.feature_info  ?    {""z]DMLeffffr4=+EFFFFr"   c                    |}| j         *|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| 	                    |          }| j
        r|                     |          |z   }|S rs   )r   r   r   r`   r   r   ra   r   r   r   r^   rc   rt   s      r    rA   zInvertedResidual.forward  s    =$a  AAALLOOHHQKKLLOOHHQKKGGAJJGGAJJMM!HHQKK= 	-q!!H,Ar"   rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   rI   r   r   r   r   r0   rq   rA   rJ   rK   s   @r    r   r      s}         #$ "#$"##%7$&N,0-1*.$&'>W >W>W >W  	>W
 >W >W >W >W >W >W !>W  >W >W !>W ">W  y)!>W" z*#>W$ "$%>W& "'>W >W >W >W >W >W@G G G      r"   r   c                   4     e Zd Zddededef fdZd Z xZS )	LayerScale2dh㈵>Fdiminit_valuesr.   c                     t                                                       || _        t          j        |t          j        |          z            | _        d S rs   )r/   r0   r.   r2   	Parametertorchonesgamma)r8   r   r   r.   r9   s       r    r0   zLayerScale2d.__init__,  sB    \+
3"?@@


r"   c                 ~    | j                             dddd          }| j        r|                    |          n||z  S )Nr   )r   viewr.   mul_)r8   r?   r   s      r    rA   zLayerScale2d.forward1  s:    
2q!,, $;qvve}}}!e);r"   )r   F)	rB   rC   rD   rH   rI   ry   r0   rA   rJ   rK   s   @r    r   r   +  sp        A AC Ae AT A A A A A A
< < < < < < <r"   r   c            &            e Zd ZdZdddddddddej        ej        dddd	d
fdededededededededede	de
dededee         dee         dee         de
dee
         f$ fdZd Zd Z xZS ) r   a   Universal Inverted Residual Block (aka Universal Inverted Bottleneck, UIB)

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L778
    r   r<   r   rM   Fr   NrN   r   r$   rO   dw_kernel_size_startdw_kernel_size_middw_kernel_size_endrQ   rR   r   rS   r|   r   r'   rU   rV   r   r   rW   layer_scale_init_valuec                    t          t          |                                            |pi }||k    o|dk    o|
 | _        |dk    r|s|s|sJ |r4|s|nd}t	          ||          }t          |||f||||	d|||d|| _        nt          j                    | _        t          ||z            }t          ||df|	||d|| _
        |r-t	          ||          }t          |||f||||	|||d|| _        nt          j                    | _        |r |||          nt          j                    | _        t          ||df|	d||d|| _        |r?|s|s|nd}t	          ||          }|dk    r|rJ t          |||f||||	d||d|| _        nt          j                    | _        |t          ||          | _        nt          j                    | _        |rt#          |          nt          j                    | _        d S )	Nr   F)rQ   rR   rZ   r[   r   r'   rU   rV   )r[   r'   rU   )rQ   rR   rZ   r[   r'   rU   rV   r   )r[   r   r'   rU   )rQ   rR   rZ   r[   r   r'   rU   )r/   r   r0   r^   r!   r   dw_startr2   rb   r
   pw_expdw_midr   pw_projdw_endr   layer_scaler	   rc   )r8   r$   rO   r   r   r   rQ   rR   r   rS   r|   r   r'   rU   rV   r   r   rW   r   dw_start_stridedw_start_groupsr   rZ   dw_end_stridedw_end_groupsr9   s                            r    r0   z"UniversalInvertedResidual.__init__=  s   * 	'..77999!'R7*:v{JF
A::'S+=SASSSS   	*,>EffAO(V<<O' 4&!& #%!   DMM KMMDM !)!344!GQ
!	
 

 
 
  	(
G44F%"4
! #%!
 
 
 
DKK +--DK =EW((7i8888"+-- #Wa
!
 
 
 
  	(*>`GY`FF_`M&z7;;Mq  ####%"4
$!$ #%
 
 
 
DKK +--DK!-+G5KLLD!{}}D5CV.111r"   c                     |dk    r!t          dd| j        j        j                  S t          d| j        j        j                  S )Nrg   zpw_proj.convr   rh   rM   rl   )rm   r   r_   r   rn   ro   s     r    rq   z&UniversalInvertedResidual.feature_info  sG    {""~PTP\PaPmnnnnr4<+<+IJJJJr"   c                 n   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        r|                     |          |z   }|S rs   )	r   r   r   r   r   r   r   r^   rc   rt   s      r    rA   z!UniversalInvertedResidual.forward  s    MM!KKNNKKNNGGAJJLLOOKKNNQ= 	-q!!H,Ar"   r   rK   s   @r    r   r   6  s         )*&'&' "#%7$&N,0-1*.$&6:'nW nWnW nW #&	nW
 !$nW !$nW nW nW nW nW nW nW !nW "nW y)nW  z*!nW" "$#nW$ "%nW& %-UO'nW nW nW nW nW nW`K K K      r"   r   c            1            e Zd ZdZdddddddddddddej        ej        d	d
d
d
dddfdedededededededededede	dededede	de
de
dee
         deded ed!ee         d"e	d#e	f0 fd$Zd% Zd& Z xZS )'r   z Mobile Attention Block

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L1504
    r   r<   rM      @   F)r   r   NrN   r   r$   rO   rQ   r{   rR   r   rS   	num_headskey_dim	value_dimuse_multi_queryquery_strides	kv_stridecpe_dw_kernel_sizer|   r'   rU   rV   rW   	attn_drop	proj_dropr   use_biasuse_cpec                    t          t          |                                            t          ||          }|dk    o||k    o| | _        t          |          | _        || _        t          d | j        D                       | _	        |rt          ||||dd          | _        nd | _         ||d          | _        |||	z  dk    sJ ||	z  }|r!t          ||||	|
|||||||          | _        nt          ||||||	          | _        |t!          ||          | _        nt%          j                    | _        |rt)          |          nt%          j                    | _        d S )
Nr   c                     g | ]}|d k    	S )r   r   ).0ss     r    
<listcomp>z,MobileAttention.__init__.<locals>.<listcomp>  s    $G$G$GqQU$G$G$Gr"   T)rP   rR   	depthwiser,   Fr   r   )dim_outr   r   r   r   r   rR   r[   r{   r   r   )r   r   r   r   r,   )r/   r   r0   r   r^   r   r   r   anyhas_query_strider   conv_cpe_dwnormr   attnr   r   r   r2   rb   r	   rc   )r8   r$   rO   rQ   r{   rR   r   rS   r   r   r   r   r   r   r   r|   r'   rU   rV   rW   r   r   r   r   r   rd   r9   s                             r    r0   zMobileAttention.__init__  s   6 	ot$$--///+J	BB1:7):JF
&}55" #$G$GD4F$G$G$G H H  		$,.!     D  $D"N6U;;;	G#q((((')I 	-##+#! -##  DII  $###  DI "-+G5KLLD!{}}D5CV.111r"   c                 |    |dk    rt          dd| j        j                  S t          d| j        j                  S r   r   ro   s     r    rq   zMobileAttention.feature_info   r   r"   c                    | j         |                      |          }||z   }|}|                     |          }|                     |          }|                     |          }| j        r|                     |          |z   }|S rs   )r   r   r   r   r^   rc   )r8   r?   x_cperu   s       r    rA   zMobileAttention.forward&  s    '$$Q''EE	AIIaLLIIaLLQ= 	-q!!H,Ar"   rv   rK   s   @r    r   r     s         "#$)!'&' #%7$&N,0$&""6:"!3WW WWWW WW 	WW
  WW WW WW WW WW WW WW "WW WW WW !$WW  !WW" !#WW$ "%WW& y)'WW( ")WW* +WW, -WW. %-UO/WW0 1WW2 3WW WW WW WW WW WWrF F F      r"   r   c            #            e Zd ZdZdddddddddej        ej        dddd	fd
ededededededede	de
dededededee         dee         dede
f" fdZd Z xZS )r   z, Inverted residual block w/ CondConv routingr<   r   rM   Fr   Nr   rN   r$   rO   r{   rQ   rR   r   rS   r|   r   r   r}   r'   rU   rV   r   num_expertsrW   c                     || _         t          | j                   }t          t          |                               |||||||||	|
|||||||           t          j        || j                   | _        d S )N)r   )r{   rQ   rR   r   rS   r|   r   r   r}   r'   rU   rV   r   r   rW   )r   rm   r/   r   r0   r2   Linear
routing_fn)r8   r$   rO   r{   rQ   rR   r   rS   r|   r   r   r}   r'   rU   rV   r   r   rW   r   r9   s                      r    r0   zCondConvResidual.__init__8  s    * 't'7888%%..)!+)!#)# 	/ 	
 	
 	
& )FD,<==r"   c                    |}t          j        |d                              d          }t          j        |                     |                    }|                     ||          }|                     |          }|                     ||          }| 	                    |          }| 
                    |          }|                     ||          }|                     |          }| j        r|                     |          |z   }|S )Nr   )Fadaptive_avg_pool2dflattenr   sigmoidr   r   r`   r   r   r   r   r   r^   rc   )r8   r?   ru   pooled_inputsrouting_weightss        r    rA   zCondConvResidual.forwardd  s    -a33;;A>>-(F(FGGLLO,,HHQKKLLO,,HHQKKGGAJJMM!_--HHQKK= 	-q!!H,Ar"   )rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   rI   r   r   r   r0   rA   rJ   rK   s   @r    r   r   5  s=       66 #$ "#$"##%7$&N,0-1 $&%*> *>*> *>  	*>
 *> *> *> *> *> *> !*>  *> !*> "*> y)*>  z*!*>" #*>$ "%*> *> *> *> *> *>X      r"   r   c            !            e Zd ZdZdddddddddej        ej        ddd	fd
edededededededede	de
dedededee         dee         de
f  fdZd Zd Z xZS )r   a(   Residual block with expansion convolution followed by pointwise-linear w/ stride

    Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML`
        - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html

    This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers
      * MobileDet - https://arxiv.org/abs/2004.14525
      * EfficientNet-X - https://arxiv.org/abs/2102.05610
      * EfficientNet-V2 - https://arxiv.org/abs/2104.00298
    r<   r   r   rM   Fr   NrN   r$   rO   r   rQ   rR   r   rS   force_in_chsr|   r   r}   r'   rU   rV   r   rW   c           	         t          t          |                                            t          ||          }|dk    rt	          ||
z            }nt	          ||
z            }t          ||          }||k    o|dk    o|	 | _        |d uo|dk    }t          ||||rdn||||          | _         ||d          | _	        t          ||||          | _        |r |||          nt          j                    | _        t          ||||          | _         ||d	
          | _        |rt#          |          nt          j                    | _        d S )Nr   r   rY   Tr-   r\   r   r   Fr   )r/   r   r0   r   r
   r!   r^   r   conv_expr`   r   ra   r2   rb   r   r   r   r	   rc   )r8   r$   rO   r   rQ   rR   r   rS   r   r|   r   r}   r'   rU   rV   r   rW   rd   r   rZ   re   r9   s                        r    r0   zEdgeResidual.__init__  s   & 	lD!!**,,,+J	BB!$\I%=>>GG$Vi%788GJ007*:v{JF
%4&1* &G_*11Ffh@ @ @ ">'4888HwvfUUU =EW((7i8888"+-- &gwPXYYY!>'U;;;5CV.111r"   c                 |    |dk    rt          dd| j        j                  S t          d| j        j                  S r   r   ro   s     r    rq   zEdgeResidual.feature_info  r   r"   c                 D   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        r|                     |          |z   }|S rs   )r   r`   ra   r   r   r   r^   rc   rt   s      r    rA   zEdgeResidual.forward  s    MM!HHQKKGGAJJGGAJJMM!HHQKK= 	-q!!H,Ar"   )rB   rC   rD   rE   r2   rF   rw   rH   rx   ry   rI   r   r   r   r0   rq   rA   rJ   rK   s   @r    r   r   t  s]       	 	 $% ! "#$#%7$&N,0-1$&#,W ,W,W ,W !	,W
 ,W ,W ,W ,W ,W ,W ,W !,W !,W ",W y),W  z*!,W" "#,W ,W ,W ,W ,W ,W\G G G
 
 
 
 
 
 
r"   r   )%rE   typingr   r   r   r   r   torch.nnr2   r   r   timm.layersr   r	   r
   r   r   r   r   r   r   r   r   __all__Moduler   rH   r!   r   r   r   r   r   r   r   r   r   r   r"   r    <module>r      s    2 1 1 1 1 1 1 1 1 1 1 1        $ $ $ $ $ $H H H H H H H H H H H H H H H H H H H H H H H H H H  
 ")_
&8C= &C & & & &%# %# %# %# %#BI %# %# %#P- - - - -	 - - -`O O O O ORY O O Od_ _ _ _ _ry _ _ _D< < < < <29 < < <H H H H H	 H H HVq q q q qbi q q qh< < < < <' < < <~J J J J J29 J J J J Jr"   