
    Ng                        d Z ddlZddlmZ ddlmZ d)defdZ G d d	ej                  Z	d)defd
Z
 G d dej                  Zd)defdZ G d dej                  Zd)defdZ G d dej                  Zd)defdZ G d dej                  Zd)defdZ G d dej                  Zd)defdZ G d dej                  Z G d dej                  Zd)dej        dedej        fd Z G d! d"ej                  Zd)dej        dedej        fd#Z G d$ d%ej                  Zd)dej        dedej        fd&Z G d' d(ej                  ZdS )*z Activations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalFinplacec                     |r'|                      |                                           n&|                     |                                           S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mul_sigmoidmulxr   s     S/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/activations.pyswishr      s<     #*A166!))++quuQYY[[/A/AA    c                   ,     e Zd Zddef fdZd Z xZS )SwishFr   c                 d    t          t          |                                            || _        d S N)superr   __init__r   selfr   	__class__s     r   r   zSwish.__init__   s*    eT##%%%r   c                 ,    t          || j                  S r   )r   r   r   r   s     r   forwardzSwish.forward   s    Q%%%r   F__name__
__module____qualname__boolr   r   __classcell__r   s   @r   r   r      sX               & & & & & & &r   r   c                 t    |                      t          j        |                                                     S )zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    NOTE: I don't have a working inplace variant
    )r	   Fsoftplustanhr
   s     r   mishr'      s*     55A##%%&&&r   c                   0     e Zd ZdZddef fdZd Z xZS )MishziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    Fr   c                 V    t          t          |                                            d S r   )r   r)   r   r   s     r   r   zMish.__init__'   %    dD""$$$$$r   c                      t          |          S r   )r'   r   s     r   r   zMish.forward*   s    Awwr   r   )r   r   r   __doc__r    r   r   r!   r"   s   @r   r)   r)   $   sb         % % % % % % % %      r   r)   c                 V    |r|                                  n|                                 S r   )sigmoid_r   r
   s     r   r   r   .   s!    "31::<<<		3r   c                   ,     e Zd Zddef fdZd Z xZS )SigmoidFr   c                 d    t          t          |                                            || _        d S r   )r   r1   r   r   r   s     r   r   zSigmoid.__init__4   s*    gt%%'''r   c                 `    | j         r|                                n|                                S r   )r   r/   r   r   s     r   r   zSigmoid.forward8   s#    #|<qzz|||<r   r   r   r"   s   @r   r1   r1   3   sX               = = = = = = =r   r1   c                 V    |r|                                  n|                                 S r   )tanh_r&   r
   s     r   r&   r&   <   s!    -177999QVVXX-r   c                   ,     e Zd Zddef fdZd Z xZS )TanhFr   c                 d    t          t          |                                            || _        d S r   )r   r7   r   r   r   s     r   r   zTanh.__init__B   s*    dD""$$$r   c                 `    | j         r|                                n|                                S r   )r   r5   r&   r   s     r   r   zTanh.forwardF   s#     L6qwwyyyaffhh6r   r   r   r"   s   @r   r7   r7   A   sX               7 7 7 7 7 7 7r   r7   c                     t          j        | dz                                 d          }|r|                     |          n|                     |          S )N      @      @)r$   relu6div_r   r	   )r   r   inners      r   
hard_swishr@   J   sD    GAFOO  $$E#5166%===u5r   c                   ,     e Zd Zddef fdZd Z xZS )	HardSwishFr   c                 d    t          t          |                                            || _        d S r   )r   rB   r   r   r   s     r   r   zHardSwish.__init__P   s*    i'')))r   c                 ,    t          || j                  S r   )r@   r   r   s     r   r   zHardSwish.forwardT   s    !T\***r   r   r   r"   s   @r   rB   rB   O   sX               + + + + + + +r   rB   c                     |r<|                      d                              dd                              d          S t          j        | dz             dz  S )Nr;   g        r<   )add_clamp_r>   r$   r=   r
   s     r   hard_sigmoidrH   X   sO     $vvbzz  R((--b111wq2v##r   c                   ,     e Zd Zddef fdZd Z xZS )HardSigmoidFr   c                 d    t          t          |                                            || _        d S r   )r   rJ   r   r   r   s     r   r   zHardSigmoid.__init__`   s*    k4  ))+++r   c                 ,    t          || j                  S r   )rH   r   r   s     r   r   zHardSigmoid.forwardd   s    At|,,,r   r   r   r"   s   @r   rJ   rJ   _   sX               - - - - - - -r   rJ   c                     |r0|                      d| dz                       dd          z            S d| z  | dz                       dd          z  S )z Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      ?   r   )minmax)r   clampr
   s     r   	hard_mishrR   h   sZ    
  5vvcQUMMaQM777888Qw!a%1!4444r   c                   ,     e Zd Zddef fdZd Z xZS )HardMishFr   c                 d    t          t          |                                            || _        d S r   )r   rT   r   r   r   s     r   r   zHardMish.__init__t   s*    h&&(((r   c                 ,    t          || j                  S r   )rR   r   r   s     r   r   zHardMish.forwardx   s    DL)))r   r   r   r"   s   @r   rT   rT   s   sX               * * * * * * *r   rT   c            	       Z     e Zd ZdZddedededd	f fd
Zdej	        dej	        fdZ
 xZS )PReLUz)Applies PReLU (w/ dummy inplace arg)
             ?Fnum_parametersinitr   returnNc                 \    t          t          |                               ||           d S )N)r[   r\   )r   rX   r   )r   r[   r\   r   r   s       r   r   zPReLU.__init__   s,    eT##>#MMMMMr   inputc                 6    t          j        || j                  S r   )r$   preluweightr   r_   s     r   r   zPReLU.forward   s    wudk***r   )rY   rZ   F)r   r   r   r-   intfloatr    r   torchTensorr   r!   r"   s   @r   rX   rX   |   s         N Ns Ne NT N^b N N N N N N+U\ +el + + + + + + + +r   rX   r   r]   c                 *    t          j        |           S r   r$   gelur
   s     r   rj   rj      s    6!99r   c                   N     e Zd ZdZddef fdZdej        dej        fdZ xZ	S )	GELULApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    Fr   c                 V    t          t          |                                            d S r   )r   rl   r   r   s     r   r   zGELU.__init__   r+   r   r_   r]   c                 *    t          j        |          S r   ri   rc   s     r   r   zGELU.forward   s    ve}}r   r   
r   r   r   r-   r    r   rf   rg   r   r!   r"   s   @r   rl   rl      sy         % % % % % % % %U\ el        r   rl   c                 .    t          j        | d          S Nr&   )approximateri   r
   s     r   	gelu_tanhrt      s    6!((((r   c                   N     e Zd ZdZddef fdZdej        dej        fdZ xZ	S )	GELUTanhrm   Fr   c                 V    t          t          |                                            d S r   )r   rv   r   r   s     r   r   zGELUTanh.__init__   s%    h&&(((((r   r_   r]   c                 .    t          j        |d          S rr   ri   rc   s     r   r   zGELUTanh.forward   s    ve0000r   r   rp   r"   s   @r   rv   rv      sy         ) ) ) ) ) ) ) )1U\ 1el 1 1 1 1 1 1 1 1r   rv   c                 6    | t          j        d| z            z  S )NgZd;?)rf   r   r
   s     r   
quick_gelurz      s    u}UQY''''r   c                   N     e Zd ZdZddef fdZdej        dej        fdZ xZ	S )		QuickGELUrm   Fr   c                 V    t          t          |                                            d S r   )r   r|   r   r   s     r   r   zQuickGELU.__init__   s%    i'')))))r   r_   r]   c                      t          |          S r   )rz   rc   s     r   r   zQuickGELU.forward   s    %   r   r   rp   r"   s   @r   r|   r|      sy         * * * * * * * *!U\ !el ! ! ! ! ! ! ! !r   r|   r   )r-   rf   r   torch.nnr   r$   r    r   Moduler   r'   r)   r   r1   r&   r7   r@   rB   rH   rJ   rR   rT   rX   rg   rj   rl   rt   rv   rz   r|    r   r   <module>r      s           $ $ $ $ $ $B Bd B B B B& & & & &BI & & &' 'T ' ' ' '    29   4 4 4 4 4 4
= = = = =bi = = =. .T . . . .
7 7 7 7 729 7 7 76 64 6 6 6 6
+ + + + +	 + + +$ $T $ $ $ $- - - - -") - - -5 5$ 5 5 5 5* * * * *ry * * *+ + + + +BH + + + EL 4 EL        29   ) ) ) ) ) ) ) )1 1 1 1 1ry 1 1 1( (%, ( (%, ( ( ( (! ! ! ! !	 ! ! ! ! !r   