
    Ng&6              	       *   d Z ddlmZmZ ddlZddlmZ ddlmc mZ	 ddl
mZ ddlmZ d(defd	Zd(defd
Zd(defdZd)deeee         f         defdZd*dededefdZd+dedededefdZd,dedefdZ G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d de          Z G d  d!ej                  Z G d" d#e          Z G d$ d%ej                  Z  G d& d'e           Z!dS )-a7   EvoNorm in PyTorch

Based on `Evolving Normalization-Activation Layers` - https://arxiv.org/abs/2004.02967
@inproceedings{NEURIPS2020,
 author = {Liu, Hanxiao and Brock, Andy and Simonyan, Karen and Le, Quoc},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {13539--13550},
 publisher = {Curran Associates, Inc.},
 title = {Evolving Normalization-Activation Layers},
 url = {https://proceedings.neurips.cc/paper/2020/file/9d4c03631b8b0c85ae08bf05eda37d0f-Paper.pdf},
 volume = {33},
 year = {2020}
}

An attempt at getting decent performing EvoNorms running in PyTorch.
While faster than other PyTorch impl, still quite a ways off the built-in BatchNorm
in terms of memory usage and throughput on GPUs.

I'm testing these modules on TPU w/ PyTorch XLA. Promising start but
currently working around some issues with builtin torch/tensor.var/std. Unlike
GPU, similar train speeds for EvoNormS variants and BatchNorm.

Hacked together by / Copyright 2020 Ross Wightman
    )SequenceUnionN   )create_act_layer)_asserth㈵>epsc                    |                                                      ddd                              |                                                              | j                  }|                    | j                  S )N      FTdimunbiasedkeepdim)floatvaraddsqrttodtypeexpandshapexr	   stds      P/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/evo_norm.pyinstance_stdr   $   sa    
''))--FUD-
A
A
E
Ec
J
J
O
O
Q
Q
T
TUVU\
]
]C::ag    c                     t          | d                              |                                          }|                    | j                  S )Nr   )r   )
manual_varr   r   r   r   r   s      r   instance_std_tpur"   )   sB    
QF
#
#
#
'
'
,
,
1
1
3
3C::agr   c                 &   |                                                                                      dd                              |                                                              | j                  }|                    | j                  S )Nr   Tr   r   )	r   squaremeanr   r   r   r   r   r   )r   r	   rmss      r   instance_rmsr(   /   sl    
''))




!
!fd
!
;
;
?
?
D
D
I
I
K
K
N
Nqw
W
WC::agr   Fr   diff_sqmc                     |                      |d          }|r4| | z                       |d          ||z  z
                      d          }n | |z
  | |z
  z                       |d          }|S )NTr$   r   )r&   clamp)r   r   r)   xmr   s        r   r!   r!   4   s|    	
C	&	&B @A||T|22b2g>EEaHHB1r6"((S$(??Jr       groupsflattenc                    | j         \  }}}}| j        }t          ||z  dk    d           |rz|                     ||d          } |                                                     ddd                              |                                                              |          }	n~|                     ||||z  ||          } |                                                     ddd                              |                                                              |          }	|		                    | j                                       ||||          S )	Nr    r   FTr   r   r      )
r   r   r   reshaper   r   r   r   r   r   )
r   r.   r	   r/   BCHWx_dtyper   s
             r   	group_stdr;   >   s%   JAq!QgGAJ!OR    eIIa$$ggiimmE4m@@DDSIINNPPSST[\\IIafa33ggiimm	E4mHHLLSQQVVXX[[\cdd::ag&&q!Q222r   c                    | j         \  }}}}t          ||z  dk    d           |r*|                     ||d          } t          | d|          }	n.|                     ||||z  ||          } t          | d|          }	|	                    |                                                              | j                                       ||||          S )Nr   r1   r2   )r   r)   r3   )r   r   r5   r!   r   r   r   )
r   r.   r	   r)   r/   r6   r7   r8   r9   r   s
             r   group_std_tpur=   K   s     JAq!QAJ!OR    >IIa$$X666IIafa33	H===773<<%%ag..66q!QBBBr   c                    | j         \  }}}}t          ||z  dk    d           | j        }|                     ||||z  ||          } |                                                                                     dd                              |                                          	                    |          }|
                    | j                                       ||||          S )Nr   r1   r3   Tr$   )r   r   r   r5   r   r%   r&   r   sqrt_r   r   )	r   r.   r	   r6   r7   r8   r9   r:   r'   s	            r   	group_rmsr@   Z   s    JAq!QAJ!OR   gG			!VQ&[!Q//A
''))




!
!i
!
>
>
B
B3
G
G
M
M
O
O
R
RSZ
[
[C::ag&&q!Q222r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dB0T皙?MbP?c                    t                                                       || _        || _        || _        t          j        t          j        |                    | _	        t          j        t          j
        |                    | _        |r&t          j        t          j        |                    nd | _        |                     dt          j        |                     |                                  d S Nrunning_var)super__init__	apply_actmomentumr	   nn	Parametertorchonesweightzerosbiasvregister_bufferreset_parametersselfnum_featuresrJ   rK   r	   _	__class__s         r   rI   zEvoNorm2dB0.__init__d   s    " l5:l#;#;<<L\!:!:;;	;DNej66777$]EJ|,D,DEEEr   c                     t           j                            | j                   t           j                            | j                   | j        &t           j                            | j                   d S d S NrL   initones_rP   zeros_rR   rS   rW   s    r   rU   zEvoNorm2dB0.reset_parameterso   Z    
dk"""
ty!!!6GMM$&!!!!! r   c                 
   t          |                                dk    d           |j        }d}| j        o| j        r|                                                    dd          }|                                |j        d         z  }| j	        
                    | j	        d| j        z
  z  |                                | j        z  ||dz
  z  z  z              n| j	        }|                    | j                                                                      |                              |                              |          }| j                            |                              |          }||z  t'          || j                  z   }||                    |          z  }|| j                            |                              |          z  | j                            |                              |          z   S Nr4   expected 4D inputr   r2   r   r   )r   r   r   F)r   r   r   )r   r   r   rS   trainingr   r   numelr   rG   copy_rK   detachr   r	   r?   r   view	expand_asr   maxrP   rR   )	rW   r   r:   v_shaper   nleftrS   rights	            r   forwardzEvoNorm2dB0.forwardu   s   11222'6} 'ggiimm	EmBBGGII
* &&$DM(9:JJLL4=0AQK@AB B B B &7748$$**,,//88==gFFPPQRSSD		'""''00AELDH555EDHHUOO#A4;>>'**//88849<<;P;P;U;UV];^;^^^r   )TrC   rD   __name__
__module____qualname__rI   rU   rr   __classcell__rZ   s   @r   rB   rB   c   sb        	  	  	  	  	  	 " " "_ _ _ _ _ _ _r   rB   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dB1TrC   r   c                    t                                                       || _        || _        || _        t          j        t          j        |                    | _	        t          j        t          j
        |                    | _        |                     dt          j        |                     |                                  d S rF   rH   rI   rJ   rK   r	   rL   rM   rN   rO   rP   rQ   rR   rT   rU   rV   s         r   rI   zEvoNorm2dB1.__init__       " l5:l#;#;<<L\!:!:;;	]EJ|,D,DEEEr   c                     t           j                            | j                   t           j                            | j                   d S r\   rL   r^   r_   rP   r`   rR   ra   s    r   rU   zEvoNorm2dB1.reset_parameters   4    
dk"""
ty!!!!!r   c                    t          |                                dk    d           |j        }d}| j        rN| j        r|                                                    dd          }|                                |j        d         z  }| j	        
                    | j	        d| j        z
  z  |                                                    | j	        j                  | j        z  ||dz
  z  z  z              n| j	        }|                    |                              |          }|                    | j                                                  }|dz   t%          || j                  z  }||                    |          z  }|| j                            |                              |          z  | j                            |                              |          z   S rd   r   r   r   rJ   rg   r   r   rh   r   rG   ri   rK   rj   r   rk   r   r	   r?   r(   rm   rP   rR   rW   r   r:   rn   r   ro   rp   rq   s           r   rr   zEvoNorm2dB1.forward   s   11222'> 	$} 'ggiimm	EmBBGGII
* &&$DM(9:JJLLOOD$4$:;;dmKqTUXYTY{[\] ] ] ] &&&//&&w//C7748$$**,,DUl1dh777EDHHUOO#A4;##G,,//88849>>';R;R;U;UV];^;^^^r   TrC   r   rs   rx   s   @r   rz   rz      b                   " " "_ _ _ _ _ _ _r   rz   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dB2TrC   r   c                    t                                                       || _        || _        || _        t          j        t          j        |                    | _	        t          j        t          j
        |                    | _        |                     dt          j        |                     |                                  d S rF   r|   rV   s         r   rI   zEvoNorm2dB2.__init__   r}   r   c                     t           j                            | j                   t           j                            | j                   d S r\   r   ra   s    r   rU   zEvoNorm2dB2.reset_parameters   r   r   c                    t          |                                dk    d           |j        }d}| j        rK| j        r|                                                    dd          }|                                |j        d         z  }| j	        
                    | j	        d| j        z
  z  |                                                    | j	        j                  | j        z  ||dz
  z  z  z              n| j	        }|                    |                              |          }|                    | j                                                  }t%          || j                  |z
  }||                    |          z  }|| j                            |                              |          z  | j                            |                              |          z   S rd   r   r   s           r   rr   zEvoNorm2dB2.forward   s   11222'> 	$} 'ggiimm	EmBBGGII
* &&$DM(9:JJLLOOD$4$:;;dmKqTUXYTY{[\] ] ] ] &&&//&&w//C7748$$**,,D DH--1EDHHUOO#A4;##G,,//88849>>';R;R;U;UV];^;^^^r   r   rs   rx   s   @r   r   r      r   r   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )	EvoNorm2dS0r-   NTr   c                    t                                                       || _        |r||z  dk    sJ ||z  | _        n|| _        || _        t          j        t          j        |                    | _	        t          j        t          j
        |                    | _        |r&t          j        t          j        |                    nd | _        |                                  d S Nr   )rH   rI   rJ   r.   r	   rL   rM   rN   rO   rP   rQ   rR   rS   rU   rW   rX   r.   
group_sizerJ   r	   rY   rZ   s          r   rI   zEvoNorm2dS0.__init__   s    " 	!*,1111&*4DKK DKl5:l#;#;<<L\!:!:;;	;DNej66777$r   c                     t           j                            | j                   t           j                            | j                   | j        &t           j                            | j                   d S d S r\   r]   ra   s    r   rU   zEvoNorm2dS0.reset_parameters   rb   r   c                    t          |                                dk    d           |j        }d}| j        c| j                            |                              |          }|||z                                  z  t          || j        | j	                  z  }|| j
                            |                              |          z  | j                            |                              |          z   S Nr4   re   rf   )r   r   r   rS   rk   r   sigmoidr;   r.   r	   rP   rR   )rW   r   r:   rn   rS   s        r   rr   zEvoNorm2dS0.forward   s    11222'6G$$''00AQUOO%%%	!T[$((K(KKA4;##G,,//88849>>';R;R;U;UV];^;^^^r   )r-   NTr   rs   rx   s   @r   r   r      sb                   " " "_ _ _ _ _ _ _r   r   c                   &     e Zd Zd fd	Zd Z xZS )EvoNorm2dS0ar-   NTrD   c                 T    t                                          |||||           d S )N)r.   r   rJ   r	   rH   rI   r   s          r   rI   zEvoNorm2dS0a.__init__   s?    J)Y\ 	 	^ 	^ 	^ 	^ 	^r   c                    t          |                                dk    d           |j        }d}t          || j        | j                  }| j        G| j                            |                              |          }|||z  	                                z  }||z  }|| j
                            |                              |          z  | j                            |                              |          z   S r   )r   r   r   r;   r.   r	   rS   rk   r   r   rP   rR   )rW   r   r:   rn   drS   s         r   rr   zEvoNorm2dS0a.forward   s    11222'adh//6G$$''00AQUOO%%%AE4;##G,,//88849>>';R;R;U;UV];^;^^^r   )r-   NTrD   rt   ru   rv   rI   rr   rw   rx   s   @r   r   r      sY        ^ ^ ^ ^ ^ ^	_ 	_ 	_ 	_ 	_ 	_ 	_r   r   c                   0     e Zd Z	 	 d fd	Zd Zd Z xZS )	EvoNorm2dS1r-   NTr   c                    t                                                       |pt          j        }|| _        ||rt          |          | _        nt          j                    | _        |r||z  dk    sJ ||z  | _        n|| _        || _	        d| _
        t          j        t          j        |                    | _        t          j        t          j        |                    | _        |                                  d S )Nr   F)rH   rI   rL   SiLUrJ   r   actIdentityr.   r	   pre_act_normrM   rN   rO   rP   rQ   rR   rU   	rW   rX   r.   r   rJ   	act_layerr	   rY   rZ   s	           r   rI   zEvoNorm2dS1.__init__  s     	(	" Y '	22DHH{}}DH 	!*,1111&*4DKK DK!l5:l#;#;<<L\!:!:;;	r   c                     t           j                            | j                   t           j                            | j                   d S r\   r   ra   s    r   rU   zEvoNorm2dS1.reset_parameters  r   r   c                    t          |                                dk    d           |j        }d}| j        r1|                     |          t          || j        | j                  z  }|| j        	                    |          
                    |          z  | j        	                    |          
                    |          z   S r   )r   r   r   rJ   r   r;   r.   r	   rP   rk   r   rR   rW   r   r:   rn   s       r   rr   zEvoNorm2dS1.forward      11222'> 	Bi4;AAAA4;##G,,//88849>>';R;R;U;UV];^;^^^r   r-   NTNr   rs   rx   s   @r   r   r      sh        6:04           *" " "_ _ _ _ _ _ _r   r   c                   *     e Zd Z	 	 d fd	Zd Z xZS )EvoNorm2dS1ar-   NTrD   c                 V    t                                          ||||||           d S N)r.   r   rJ   r   r	   r   r   s	           r   rI   zEvoNorm2dS1a.__init__$  D     	J)_hnq 	 	s 	s 	s 	s 	sr   c                 ~   t          |                                dk    d           |j        }d}|                     |          t	          || j        | j                  z  }|| j                            |          	                    |          z  | j
                            |          	                    |          z   S r   )r   r   r   r   r;   r.   r	   rP   rk   r   rR   r   s       r   rr   zEvoNorm2dS1a.forward*      11222'HHQKK)At{DH===4;##G,,//88849>>';R;R;U;UV];^;^^^r   r-   NTNrD   r   rx   s   @r   r   r   #  _        6:04s s s s s s_ _ _ _ _ _ _r   r   c                   0     e Zd Z	 	 d fd	Zd Zd Z xZS )	EvoNorm2dS2r-   NTr   c                    t                                                       |pt          j        }|| _        ||rt          |          | _        nt          j                    | _        |r||z  dk    sJ ||z  | _        n|| _        || _	        t          j
        t          j        |                    | _        t          j
        t          j        |                    | _        |                                  d S r   )rH   rI   rL   r   rJ   r   r   r   r.   r	   rM   rN   rO   rP   rQ   rR   rU   r   s	           r   rI   zEvoNorm2dS2.__init__3  s     	(	" Y '	22DHH{}}DH 	!*,1111&*4DKK DKl5:l#;#;<<L\!:!:;;	r   c                     t           j                            | j                   t           j                            | j                   d S r\   r   ra   s    r   rU   zEvoNorm2dS2.reset_parametersG  r   r   c                    t          |                                dk    d           |j        }d}| j        r1|                     |          t          || j        | j                  z  }|| j        	                    |          
                    |          z  | j        	                    |          
                    |          z   S r   )r   r   r   rJ   r   r@   r.   r	   rP   rk   r   rR   r   s       r   rr   zEvoNorm2dS2.forwardK  r   r   r   rs   rx   s   @r   r   r   2  sh        6:04           (" " "_ _ _ _ _ _ _r   r   c                   *     e Zd Z	 	 d fd	Zd Z xZS )EvoNorm2dS2ar-   NTrD   c                 V    t                                          ||||||           d S r   r   r   s	           r   rI   zEvoNorm2dS2a.__init__U  r   r   c                 ~   t          |                                dk    d           |j        }d}|                     |          t	          || j        | j                  z  }|| j                            |          	                    |          z  | j
                            |          	                    |          z   S r   )r   r   r   r   r@   r.   r	   rP   rk   r   rR   r   s       r   rr   zEvoNorm2dS2a.forward[  r   r   r   r   rx   s   @r   r   r   T  r   r   r   )r   )F)r-   r   F)r-   r   FF)r-   r   )"__doc__typingr   r   rN   torch.nnrL   torch.nn.functional
functionalF
create_actr   trace_utilsr   r   r   r"   r(   intboolr!   r;   r=   r@   ModulerB   rz   r   r   r   r   r   r   r    r   r   <module>r      s   2 # " " " " " " "                 ( ( ( ( ( (                 
 U         
 uS(3-/0 D    
3 
3 
3 
3t 
3 
3 
3 
3C CS CE CD C[_ C C C C3 3 3 3 3 3 3$_ $_ $_ $_ $_") $_ $_ $_N _  _  _  _  _")  _  _  _F _  _  _  _  _")  _  _  _F_ _ _ _ _") _ _ _>_ _ _ _ _; _ _ _" _  _  _  _  _")  _  _  _F_ _ _ _ _; _ _ __ _ _ _ _") _ _ _D_ _ _ _ _; _ _ _ _ _r   