
    Ng                     P   d Z ddlZddlmZ ddlmZ d Zd Z G d dej        j	                  Z
d*d
Z G d dej                  Zd Zd Z G d dej        j	                  Zd*dZ G d dej                  Zd*defdZd Z G d dej        j	                  Zd*defdZ G d dej                  Zd Zd Z G d dej        j	                  Zd*d Z G d! d"ej                  Zd# Zd$ Z G d% d&ej        j	                  Z d*defd'Z! G d( d)ej                  Z"dS )+a~   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use
basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalc                 P    |                      t          j        |                     S N)multorchsigmoidxs    V/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/activations_me.py	swish_fwdr      s    55q!!"""    c                 L    t          j        |           }||d| d|z
  z  z   z  z  S N   )r   r	   )r   grad_output	x_sigmoids      r   	swish_bwdr      s/    a  I)q1I+>'>?@@r   c                   T    e Zd ZdZed             Zed             Zed             ZdS )SwishAutoFnz optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 X    |                      d||                      d|                    S )NMulSigmoid)op)gr   s     r   symboliczSwishAutoFn.symbolic   s&    ttE1add9a00111r   c                 J    |                      |           t          |          S r   )save_for_backwardr   ctxr   s     r   forwardzSwishAutoFn.forward#   s"    a   ||r   c                 <    | j         d         }t          ||          S Nr   )saved_tensorsr   r    r   r   s      r   backwardzSwishAutoFn.backward(   s    a K(((r   N)__name__
__module____qualname____doc__staticmethodr   r!   r&    r   r   r   r      so          2 2 \2   \ ) ) \) ) )r   r   Fc                 6    t                               |           S r   r   applyr   inplaces     r   swish_mer2   .   s    Qr   c                   ,     e Zd Zddef fdZd Z xZS )SwishMeFr1   c                 V    t          t          |                                            d S r   )superr4   __init__selfr1   	__class__s     r   r7   zSwishMe.__init__3   s%    gt%%'''''r   c                 6    t                               |          S r   r.   r9   r   s     r   r!   zSwishMe.forward6   s      ###r   Fr'   r(   r)   boolr7   r!   __classcell__r:   s   @r   r4   r4   2   sX        ( ( ( ( ( ( ( ($ $ $ $ $ $ $r   r4   c                 t    |                      t          j        t          j        |                               S r   )r   r   tanhFsoftplusr
   s    r   mish_fwdrF   :   s&    55AJqMM**+++r   c                     t          j        |           }t          j        |                                           }|                    || |z  d||z  z
  z  z             S r   )r   r	   rD   rE   rC   r   )r   r   r   	x_tanh_sps       r   mish_bwdrI   >   sT    a  I
1""$$I??9q9}I	<Q8Q'RRSSSr   c                   >    e Zd ZdZed             Zed             ZdS )
MishAutoFnz Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient variant of Mish
    c                 J    |                      |           t          |          S r   )r   rF   r   s     r   r!   zMishAutoFn.forwardH   s"    a   {{r   c                 <    | j         d         }t          ||          S r#   )r$   rI   r%   s      r   r&   zMishAutoFn.backwardM   s    a ;'''r   Nr'   r(   r)   r*   r+   r!   r&   r,   r   r   rK   rK   D   sT            \ ( ( \( ( (r   rK   c                 6    t                               |           S r   rK   r/   r0   s     r   mish_merQ   S   s    Ar   c                   ,     e Zd Zddef fdZd Z xZS )MishMeFr1   c                 V    t          t          |                                            d S r   )r6   rS   r7   r8   s     r   r7   zMishMe.__init__X   s%    fd$$&&&&&r   c                 6    t                               |          S r   rP   r<   s     r   r!   zMishMe.forward[   s    """r   r=   r>   rA   s   @r   rS   rS   W   sX        ' ' ' ' ' ' ' '# # # # # # #r   rS   r1   c                 \    | dz                        dd                              d          S N   r      minmax      @clampdivr0   s     r   hard_sigmoid_fwdra   _   s*    E==QA=&&**2...r   c                 V    t          j        |           | dk    | dk    z  z  dz  }||z  S )N            @r]   )r   	ones_liker   r   ms      r   hard_sigmoid_bwdrh   c   s2    qCxAG45:A?r   c                   :    e Zd Zed             Zed             ZdS )HardSigmoidAutoFnc                 J    |                      |           t          |          S r   )r   ra   r   s     r   r!   zHardSigmoidAutoFn.forwardi   s$    a   """r   c                 <    | j         d         }t          ||          S r#   )r$   rh   r%   s      r   r&   zHardSigmoidAutoFn.backwardn   s    a ;///r   N)r'   r(   r)   r+   r!   r&   r,   r   r   rj   rj   h   sH        # # \# 0 0 \0 0 0r   rj   c                 6    t                               |           S r   rj   r/   r0   s     r   hard_sigmoid_mero   t   s    ""1%%%r   c                   ,     e Zd Zddef fdZd Z xZS )HardSigmoidMeFr1   c                 V    t          t          |                                            d S r   )r6   rq   r7   r8   s     r   r7   zHardSigmoidMe.__init__y   s%    mT""++-----r   c                 6    t                               |          S r   rn   r<   s     r   r!   zHardSigmoidMe.forward|   s     &&q)))r   r=   r>   rA   s   @r   rq   rq   x   sX        . . . . . . . .* * * * * * *r   rq   c                 b    | | dz                        dd                              d          z  S rW   r^   r
   s    r   hard_swish_fwdru      s/    A}}}**..r2222r   c                     t          j        |           | dk    z  }t          j        | dk    | dk    z  | dz  dz   |          }||z  S )Nrd   rc         ?r   re   whererf   s      r   hard_swish_bwdrz      sL    a2g&AQ#X!r'*QVb[!<<A?r   c                   T    e Zd ZdZed             Zed             Zed             ZdS )HardSwishAutoFnz'A memory efficient HardSwish activationc                 J    |                      |           t          |          S r   )r   ru   r   s     r   r!   zHardSwishAutoFn.forward   s$    a   a   r   c                 <    | j         d         }t          ||          S r#   )r$   rz   r%   s      r   r&   zHardSwishAutoFn.backward   s    a a---r   c                 T   |                      d||                      dt          j        dt          j                                      }|                      d||                      dt          j        dt          j                            |                      dt          j        dt          j                                      }|                      d	||                      dt          j        dt          j                                      }|                      d
||          S )NAddConstantrX   )dtype)value_tClipr   rY   Divr   )r   r   tensorfloat)r   r9   input	hardtanh_s       r   r   zHardSwishAutoFn.symbolic   s   UD!$$z5<QVQ\;];];]$"^"^__DDZaW\WbAcAcAc(d(dfgfjfjku  @E  @L  MN  V[  Va  @b  @b  @bfj  gc  gc  d  d	DD	144
ELQRZ_ZeDfDfDf4+g+ghh	ttE4+++r   N)r'   r(   r)   r*   r+   r!   r&   r   r,   r   r   r|   r|      si        11! ! \! . . \. , , \, , ,r   r|   c                 6    t                               |           S r   r|   r/   r0   s     r   hard_swish_mer      s      ###r   c                   ,     e Zd Zddef fdZd Z xZS )HardSwishMeFr1   c                 V    t          t          |                                            d S r   )r6   r   r7   r8   s     r   r7   zHardSwishMe.__init__   s%    k4  ))+++++r   c                 6    t                               |          S r   r   r<   s     r   r!   zHardSwishMe.forward   s    $$Q'''r   r=   r>   rA   s   @r   r   r      sX        , , , , , , , ,( ( ( ( ( ( (r   r   c                 B    d| z  | dz                        dd          z  S )Nrw      r   rZ   )r_   r
   s    r   hard_mish_fwdr      s%    7a!e]]qa]0000r   c                     t          j        |           | dk    z  }t          j        | dk    | dk    z  | dz   |          }||z  S )Ng       g        g      ?rx   rf   s      r   hard_mish_bwdr      sH    a3h'AQ#X!r'*AFA66A?r   c                   >    e Zd ZdZed             Zed             ZdS )HardMishAutoFnz A memory efficient variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 J    |                      |           t          |          S r   )r   r   r   s     r   r!   zHardMishAutoFn.forward   s$    a   Qr   c                 <    | j         d         }t          ||          S r#   )r$   r   r%   s      r   r&   zHardMishAutoFn.backward   s    a Q,,,r   NrN   r,   r   r   r   r      sT              \  - - \- - -r   r   c                 6    t                               |           S r   r   r/   r0   s     r   hard_mish_mer      s    """r   c                   ,     e Zd Zddef fdZd Z xZS )
HardMishMeFr1   c                 V    t          t          |                                            d S r   )r6   r   r7   r8   s     r   r7   zHardMishMe.__init__   s%    j$((*****r   c                 6    t                               |          S r   r   r<   s     r   r!   zHardMishMe.forward   s    ##A&&&r   r=   r>   rA   s   @r   r   r      sX        + + + + + + + +' ' ' ' ' ' 'r   r   r=   )#r*   r   r   torch.nnr   rD   r   r   autogradFunctionr   r2   Moduler4   rF   rI   rK   rQ   rS   r?   ra   rh   rj   ro   rq   ru   rz   r|   r   r   r   r   r   r   r   r,   r   r   <module>r      s8  	 	        $ $ $ $ $ $# # #A A A
) ) ) ) )%.) ) ) )(       $ $ $ $ $bi $ $ $, , ,T T T( ( ( ( (( ( ( (   # # # # #RY # # #/ / / / / /  
	0 	0 	0 	0 	0/ 	0 	0 	0& & & & & &* * * * *BI * * *3 3 3  , , , , ,en- , , ,($ $ $ $( ( ( ( (") ( ( (1 1 1  - - - - -U^, - - - # #T # # # #' ' ' ' ' ' ' ' ' 'r   