
    Ng	!                     D   d Z ddlmZ ddlmZ ddlmZ ddlmZ  G d dej	                  Z
 G d	 d
ej	                  Z eeej        d          Z G d dej	                  Z G d dej	                  Z G d dej	                  Z G d dej	                  ZdS )zl MLP module w/ dropout and configurable activation layer

Hacked together by / Copyright 2020 Ross Wightman
    )partial)nn   )GlobalResponseNorm)	to_2tuplec                   B     e Zd ZdZddej        ddddf fd	Zd Z xZS )MlpzG MLP as used in Vision Transformer, MLP-Mixer and related networks
    NT        Fc	                 .   t                                                       |p|}|p|}t          |          }t          |          }	|rt          t          j        d          nt          j        }
 |
|||d                   | _         |            | _        t	          j	        |	d                   | _
        | ||          nt	          j                    | _         |
|||d                   | _        t	          j	        |	d                   | _        d S )Nr   kernel_sizer   bias)super__init__r   r   r   Conv2dLinearfc1actDropoutdrop1Identitynormfc2drop2)selfin_featureshidden_featuresout_features	act_layer
norm_layerr   dropuse_conv
drop_probslinear_layer	__class__s              K/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/mlp.pyr   zMlp.__init__   s     	#2{)8[t__
<DSwrya8888")<_47KKK9;;Z
1..
3=3IJJ///r{}}	<DGLLLZ
1..


    c                    |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|S N)r   r   r   r   r   r   r   xs     r'   forwardzMlp.forward)   sa    HHQKKHHQKKJJqMMIIaLLHHQKKJJqMMr(   	__name__
__module____qualname____doc__r   GELUr   r-   __classcell__r&   s   @r'   r	   r	      so         
 !g/ / / / / /2      r(   r	   c                   J     e Zd ZdZddej        dddddf fd	Zd Zd Z xZ	S )	GluMlpzi MLP w/ GLU style gating
    See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202
    NTr
   Fc
                 t   t                                                       |p|}|p|}|dz  dk    sJ t          |          }t          |          }
|rt          t          j        d          nt          j        }|rdnd| _        |	| _         ||||d                   | _	         |            | _
        t	          j        |
d                   | _        | ||dz            nt	          j                    | _         ||dz  ||d                   | _        t	          j        |
d                   | _        d S )N   r   r   r   r   )r   r   r   r   r   r   r   	chunk_dim	gate_lastr   r   r   r   r   r   r   r   )r   r   r   r   r    r!   r   r"   r#   r<   r$   r%   r&   s               r'   r   zGluMlp.__init__7   s4    	#2{)8["a''''t__
<DSwrya8888")&.B"<_47KKK9;;Z
1..
8B8NJJ!3444TVT_TaTa	<1 4laQQQZ
1..


r(   c                    | j         j        j        d         dz  }t          j                            | j         j        |d                     t          j                            | j         j        |d          d           d S )Nr   r9   ư>std)r   r   shaper   initones_normal_weight)r   fc1_mids     r'   init_weightszGluMlp.init_weightsT   sd    (-%a(A-
dhmGHH-...
1t<<<<<r(   c                    |                      |          }|                    d| j                  \  }}| j        r||                     |          z  n|                     |          |z  }|                     |          }|                     |          }|                     |          }|                     |          }|S )Nr9   )dim)	r   chunkr;   r<   r   r   r   r   r   )r   r,   x1x2s       r'   r-   zGluMlp.forwardZ   s    HHQKK//B!%FB"TXXb\\B5FJJqMMIIaLLHHQKKJJqMMr(   )
r/   r0   r1   r2   r   Sigmoidr   rG   r-   r4   r5   s   @r'   r7   r7   3   s          !j/ / / / / /:= = =      r(   r7   F)r    r<   c                   F     e Zd ZdZddej        dddf fd	Zd Zd Z xZ	S )SwiGLUz SwiGLU
    NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
    better matches some other common impl which makes mapping checkpoints simpler.
    NTr
   c                 D   t                                                       |p|}|p|}t          |          }t          |          }t          j        |||d                   | _        t          j        |||d                   | _         |            | _        t          j        |d                   | _	        | ||          nt          j
                    | _        t          j        |||d                   | _        t          j        |d                   | _        d S )Nr   r   r   )r   r   r   r   r   fc1_gfc1_xr   r   r   r   r   r   r   )
r   r   r   r   r    r!   r   r"   r$   r&   s
            r'   r   zSwiGLU.__init__m   s     	#2{)8[t__
Y{O$q'JJJ
Y{O$q'JJJ
9;;Z
1..
3=3IJJ///r{}}	9_laIIIZ
1..


r(   c                     t           j                            | j        j                   t           j                            | j        j        d           d S )Nr>   r?   )r   rB   rC   rQ   r   rD   rE   )r   s    r'   rG   zSwiGLU.init_weights   s=    
djo&&&

)t44444r(   c                 2   |                      |          }|                     |          }|                     |          |z  }|                     |          }|                     |          }|                     |          }|                     |          }|S r*   )rQ   rR   r   r   r   r   r   )r   r,   x_gates      r'   r-   zSwiGLU.forward   sw    AJJqMMHHVq JJqMMIIaLLHHQKKJJqMMr(   )
r/   r0   r1   r2   r   SiLUr   rG   r-   r4   r5   s   @r'   rO   rO   h   s{          !g/ / / / / /05 5 5
      r(   rO   c                   B     e Zd ZdZddej        ddddf fd	Zd Z xZS )GatedMlpz MLP as used in gMLP
    NTr
   c	                 v   t                                                       |p|}|p|}t          |          }t          |          }	t          j        |||d                   | _         |            | _        t          j        |	d                   | _        |!|dz  dk    sJ  ||          | _	        |dz  }nt          j
                    | _	        | ||          nt          j
                    | _        t          j        |||d                   | _        t          j        |	d                   | _        d S )Nr   r   r9   r   )r   r   r   r   r   r   r   r   r   gater   r   r   r   )r   r   r   r   r    r!   
gate_layerr   r"   r$   r&   s             r'   r   zGatedMlp.__init__   s    	#2{)8[t__
9[/QHHH9;;Z
1..
!"Q&!++++"
?33DI-2OODI3=3IJJ///r{}}	9_laIIIZ
1..


r(   c                 ,   |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|S r*   )r   r   r   rZ   r   r   r   r+   s     r'   r-   zGatedMlp.forward   sp    HHQKKHHQKKJJqMMIIaLLIIaLLHHQKKJJqMMr(   r.   r5   s   @r'   rX   rX      so         
 !g/ / / / / /<      r(   rX   c                   @     e Zd ZdZddej        dddf fd	Zd Z xZS )ConvMlpz1 MLP using 1x1 convs that keeps spatial dims
    NTr
   c                    t                                                       |p|}|p|}t          |          }t          j        ||d|d                   | _        |r ||          nt          j                    | _         |            | _        t          j	        |          | _
        t          j        ||d|d                   | _        d S )Nr   r   )r   r   )r   r   r   r   r   r   r   r   r   r   r"   r   )	r   r   r   r   r    r!   r   r"   r&   s	           r'   r   zConvMlp.__init__   s     	#2{)8[9[/qtTUwWWW3=PJJ///2;==	9;;Jt$$	9_lPTUVPWXXXr(   c                     |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|S r*   )r   r   r   r"   r   r+   s     r'   r-   zConvMlp.forward   sR    HHQKKIIaLLHHQKKIIaLLHHQKKr(   )	r/   r0   r1   r2   r   ReLUr   r-   r4   r5   s   @r'   r^   r^      sr         
 !gY Y Y Y Y Y*      r(   r^   c                   @     e Zd ZdZddej        dddf fd	Zd Z xZS )GlobalResponseNormMlpzG MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d
    NTr
   Fc                    t                                                       |p|}|p|}t          |          }t          |          }|rt          t          j        d          nt          j        }	 |	|||d                   | _         |            | _        t	          j	        |d                   | _
        t          ||           | _         |	|||d                   | _        t	          j	        |d                   | _        d S )Nr   r   r   r   )channels_last)r   r   r   r   r   r   r   r   r   r   r   r   grnr   r   )r   r   r   r   r    r   r"   r#   r$   r%   r&   s             r'   r   zGlobalResponseNormMlp.__init__   s     	#2{)8[t__
<DSwrya8888")<_47KKK9;;Z
1..
%o\RRR<DGLLLZ
1..


r(   c                    |                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|S r*   )r   r   r   rf   r   r   r+   s     r'   r-   zGlobalResponseNormMlp.forward   sa    HHQKKHHQKKJJqMMHHQKKHHQKKJJqMMr(   r.   r5   s   @r'   rc   rc      sl         
 !g/ / / / / /0      r(   rc   N)r2   	functoolsr   torchr   rf   r   helpersr   Moduler	   r7   rV   SwiGLUPackedrO   rX   r^   rc    r(   r'   <module>rn      s                # # # # # #      # # # # #") # # #L/ / / / /RY / / /d wvEBBB* * * * *RY * * *Z) ) ) ) )ry ) ) )X    bi   B" " " " "BI " " " " "r(   