
    NgzC                        d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ d dZ G d dej                  Z G d dej                  Zd!dZ G d dej        j                  Zd Zd Zd Z G d dej                   Z! G d dej                   Z" G d dej#                  Z$ G d dej#                  Z%dS )"aw   Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
    )UnionListOptionalAnyN)nn)
functional)FrozenBatchNorm2d   )create_act_layer)is_fast_normfast_group_normfast_layer_norm)_assertFTc                     |pi }|                     d|           d }|rt          | fi |}|t          j                    n|S )Ninplace)
setdefaultr   r   Identity)	act_layer
act_kwargsr   	apply_actacts        P/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/norm_act.py_create_actr      sX    !rJ)W---
C 8y77J77K2;===S0    c                   J     e Zd ZdZdddddej        dddddf fd	Zd Z xZS )BatchNormAct2dzBatchNorm + Activation

    This module performs BatchNorm + Activation in a manner that will remain backwards
    compatible with weights trained with separate bn, act. This is why we inherit from BN
    instead of composing it as a .bn member.
    h㈵>g?TNc                 T   	 ||d} t          t          |           j        |f||||d| n=# t          $ r0 t          t          |                               |||||           Y nw xY w|

 |
            nt	          j                    | _        t          |||	|          | _        d S )N)devicedtype)epsmomentumaffinetrack_running_statsr   r   r   )	superr   __init__	TypeErrorr   r   dropr   r   )selfnum_featuresr!   r"   r#   r$   r   r   r   r   
drop_layerr   r    factory_kwargs	__class__s                 r   r'   zBatchNormAct2d.__init__+   s    	(.??N0E.$''0!$7  !     	 	 	.$''00!$7 1     	 %/$:JJLLL	yZ\efffs   +/ 7A)(A)c           
      N   t          |j        dk    d|j         d           | j        d}n| j        }| j        rN| j        rG| j        @| j                            d           | j        dt          | j                  z  }n| j        }	 | j        rd}n| j        d u o| j	        d u }	 t          j        || j        r| j        r| j        nd | j        r| j        r| j	        nd | j        | j        ||| j                  }|                     |          }|                     |          }|S )N   zexpected 4D input (got zD input)g        r
   g      ?T)r   ndimr"   trainingr$   num_batches_trackedadd_floatrunning_meanrunning_varF
batch_normweightbiasr!   r)   r   )r*   xexponential_average_factorbn_trainings       r   forwardzBatchNormAct2d.forwardP   s\   !GqvGGGHHH
 = ),&&)-&= 	?T5 	?'3(--a000=(14uT=U7V7V1V..15.	 = 	UKK,4T4;Kt;SK	
 L%)]Xd6NXDTX$(MWT5MWDSWKI&H

 

 IIaLLHHQKKr   )	__name__
__module____qualname____doc__r   ReLUr'   r?   __classcell__r.   s   @r   r   r   $   s           $g#g #g #g #g #g #gJ0 0 0 0 0 0 0r   r   c                   <     e Zd Zdej        dej        f fdZ xZS )SyncBatchNormActr<   returnc                     t                                          |          }t          | d          r|                     |          }t          | d          r|                     |          }|S )Nr)   r   )r&   r?   hasattrr)   r   )r*   r<   r.   s     r   r?   zSyncBatchNormAct.forward   s_    GGOOA4   			!A4 	Ar   )r@   rA   rB   torchTensorr?   rE   rF   s   @r   rH   rH      sO        
 %,          r   rH   c                 "   | }t          | t          j        j        j        j                  r#t          | t                    rGt          | j        | j	        | j
        | j        | j        |          }| j        |_        | j        |_        n=t          j                            | j        | j	        | j
        | j        | j        |          }| j        rCt          j                    5  | j        |_        | j        |_        d d d            n# 1 swxY w Y   | j        |_        | j        |_        | j        |_        t-          | d          r| j        |_        |                                 D ])\  }}|                    |t5          ||                     *~ |S )N)process_groupqconfig)
isinstancerL   r   modules	batchnorm
_BatchNormr   rH   r+   r!   r"   r#   r$   r   r)   SyncBatchNormno_gradr:   r;   r6   r7   r3   rK   rP   named_children
add_moduleconvert_sync_batchnorm)modulerO   module_outputnamechilds        r   rY   rY      s   M&%(*4?@@  3fn-- 	,#
*+  M !'
M!'M "H22#
* M = 	1 1 1'-}$%+["1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 &,%8"$*$6!,2,F)69%% 	3$*NM!,,.. U Ue  '=e]'S'STTTTs    DD	D	c                        e Zd ZdZddej        dddfdedef fdZde	d	e
d
e	dedee
         dee
         dee
         f fdZdej        dej        fdZde
fdZ xZS )FrozenBatchNormAct2da$  
    BatchNormAct2d where the batch statistics and the affine parameters are fixed

    Args:
        num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
        eps (float): a value added to the denominator for numerical stability. Default: 1e-5
    r   TNr+   r!   c                    t                                                       || _        |                     dt	          j        |                     |                     dt	          j        |                     |                     dt	          j        |                     |                     dt	          j        |                     |
 |            nt          j                    | _	        t          ||||          | _        d S )Nr:   r;   r6   r7   r%   )r&   r'   r!   register_bufferrL   oneszerosr   r   r)   r   r   )	r*   r+   r!   r   r   r   r   r,   r.   s	           r   r'   zFrozenBatchNormAct2d.__init__   s     	Xuz,'?'?@@@VU[%>%>???^U[-F-FGGG]EJ|,D,DEEE$.$:JJLLL	yZ\efffr   
state_dictprefixlocal_metadatastrictmissing_keysunexpected_keys
error_msgsc           	      n    |dz   }||v r||= t                                          |||||||           d S )Nr3   )r&   _load_from_state_dict)
r*   rd   re   rf   rg   rh   ri   rj   num_batches_tracked_keyr.   s
            r   rl   z*FrozenBatchNormAct2d._load_from_state_dict   s[     #)+@"@"j0023%%oWa	
 	
 	
 	
 	
r   r<   rI   c                    | j                             dddd          }| j                            dddd          }| j                            dddd          }| j                            dddd          }||| j        z                                   z  }|||z  z
  }||z  |z   }|                     |                     |                    }|S )Nr
   )	r:   reshaper;   r7   r6   r!   rsqrtr   r)   )r*   r<   wbrvrmscaler;   s           r   r?   zFrozenBatchNormAct2d.forward   s     K2q!,,IaQ**%%aQ22&&q"a33R$(]))+++2:~IHHTYYq\\""r   c                 f    | j         j         d| j        j        d          d| j         d| j         dS )N(r   z, eps=z, act=))r.   r@   r:   shaper!   r   )r*   s    r   __repr__zFrozenBatchNormAct2d.__repr__   s>    .)ccDK,=a,@ccccX\X`ccccr   )r@   rA   rB   rC   r   rD   intr5   r'   dictstrboolr   rl   rL   rM   r?   r{   rE   rF   s   @r   r_   r_      s5         'g gg g g g g g g(

 
 	

 
 3i
 c
 I
 
 
 
 
 
$ %,    d# d d d d d d d dr   r_   c                    | }t          | t          t          f          rt          | j                  }| j        |_        | j        |_        | j        rt| j        j                                        	                                |j        _        | j
        j                                        	                                |j
        _        | j        j        |j        _        | j        j        |j        _        | j        |_        | j        |_        | j        |_        ngt          | t           j        j        j        j        t           j        j        j        j        f          rt-          | j                  }| j        |_        | j        |_        | j        rt| j        j                                        	                                |j        _        | j
        j                                        	                                |j
        _        | j        j        |j        _        | j        j        |j        _        | j        |_        nC|                                 D ].\  }}t1          |          }||ur|                    ||           /|S )a  
    Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
    of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )rQ   r   rH   r_   r+   r#   r:   dataclonedetachr;   r6   r7   r!   r)   r   rL   r   rR   rS   BatchNorm2drU   r	   rW   freeze_batch_norm_2drX   rZ   resr\   r]   	new_childs        r   r   r      s    C&>+;<== 0"6#677!.]
= 	>$m06688??AACJO"K,2244;;==CHM & 3 8%16*;*	FUX-7CUXEUE_Emn	o	o 0 344!.]
= 	>$m06688??AACJO"K,2244;;==CHM & 3 8%16*!0022 	0 	0KD%,U33I%%tY///Jr   c                 V   | }t          | t                    rt          | j                  }| j        rt| j        j                                                                        |j        _        | j	        j                                                                        |j	        _        | j
        j        |j
        _        | j        j        |j        _        | j        |_        | j        |_        | j        |_        n0t          | t                    rt           j                            | j                  }| j        rt| j        j                                                                        |j        _        | j	        j                                                                        |j	        _        | j
        j        |j
        _        | j        j        |j        _        | j        |_        nC|                                 D ].\  }}t)          |          }||ur|                    ||           /|S )a  
    Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
    of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
    recursively and submodules are converted in place.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )rQ   r_   r   r+   r#   r:   r   r   r   r;   r6   r7   r!   r)   r   r	   rL   r   r   rW   unfreeze_batch_norm_2drX   r   s        r   r   r   '  s    C&.// 0V011= 	>$m06688??AACJO"K,2244;;==CHM & 3 8%16*;*	F-	.	. 0h""6#677= 	>$m06688??AACJO"K,2244;;==CHM & 3 8%16*!0022 	0 	0KD%.u55I%%tY///Jr   c                 *    |r| |z  dk    sJ | |z  S |S )Nr    )num_channels
num_groups
group_sizes      r   _num_groupsr   P  s1     *j(A----z))r   c            	       B     e Zd Zdddddej        dddf	 fd	Zd Z xZS )GroupNormAct    r   TNc                    t          t          |                               t          |||          |||           |

 |
            nt	          j                    | _        t          |||	|          | _        t                      | _
        d S )Nr!   r#   r%   )r&   r   r'   r   r   r   r)   r   r   r   
_fast_norm)r*   r   r   r!   r#   r   r   r   r   r   r,   r.   s              r   r'   zGroupNormAct.__init__Y  s     	lD!!**j*==	 	+ 	
 	
 	
 %/$:JJLLL	yZ\efff&..r   c                    | j         r(t          || j        | j        | j        | j                  }n,t          j        || j        | j        | j        | j                  }|                     |          }| 	                    |          }|S N
r   r   r   r:   r;   r!   r8   
group_normr)   r   r*   r<   s     r   r?   zGroupNormAct.forwardq  n    ? 	S4?DKDHUUAAQdiRRAIIaLLHHQKKr   r@   rA   rB   r   rD   r'   r?   rE   rF   s   @r   r   r   W  sk        
 g) ) ) ) ) )0      r   r   c                   >     e Zd Zdddej        dddf fd	Zd Z xZS )GroupNorm1Actr   TNc	                     t          t          |                               d|||           |
 |            nt          j                    | _        t          ||||          | _        t                      | _	        d S )Nr
   r   r%   )
r&   r   r'   r   r   r)   r   r   r   r   
r*   r   r!   r#   r   r   r   r   r,   r.   s
            r   r'   zGroupNorm1Act.__init__|  sp     	mT""++A|V+TTT$.$:JJLLL	yZ\efff&..r   c                    | j         r(t          || j        | j        | j        | j                  }n,t          j        || j        | j        | j        | j                  }|                     |          }| 	                    |          }|S r   r   r   s     r   r?   zGroupNorm1Act.forward  r   r   r   rF   s   @r   r   r   {  se         g) ) ) ) ) )"      r   r   c                   l     e Zd Zdddej        dddfdeeee         ej	        f         f fdZ
d Z xZS )LayerNormActr   TNnormalization_shapec	                     t          t          |                               |||           |
 |            nt          j                    | _        t          ||||          | _        t                      | _	        d S N)r!   elementwise_affiner%   )
r&   r   r'   r   r   r)   r   r   r   r   )
r*   r   r!   r#   r   r   r   r   r,   r.   s
            r   r'   zLayerNormAct.__init__  sp     	lD!!**+>C\b*ccc$.$:JJLLL	yZ\efff&..r   c                    | j         r(t          || j        | j        | j        | j                  }n,t          j        || j        | j        | j        | j                  }|                     |          }| 	                    |          }|S r   )
r   r   normalized_shaper:   r;   r!   r8   
layer_normr)   r   r   s     r   r?   zLayerNormAct.forward  sr    ? 	Y4#8$+tyRVRZ[[AAQ 5t{DItxXXAIIaLLHHQKKr   )r@   rA   rB   r   rD   r   r|   r   rL   Sizer'   r?   rE   rF   s   @r   r   r     s         g) )!&sDIuz'A!B) ) ) ) ) )"      r   r   c                   >     e Zd Zdddej        dddf fd	Zd Z xZS )LayerNormAct2dr   TNc	                     t          t          |                               |||           |
 |            nt          j                    | _        t          ||||          | _        t                      | _	        d S r   )
r&   r   r'   r   r   r)   r   r   r   r   r   s
            r   r'   zLayerNormAct2d.__init__  so     	nd##,,\sW],^^^$.$:JJLLL	yZ\efff&..r   c                 p   |                     dddd          }| j        r(t          || j        | j        | j        | j                  }n,t          j        || j        | j        | j        | j                  }|                     dddd          }| 	                    |          }| 
                    |          }|S )Nr         r
   )permuter   r   r   r:   r;   r!   r8   r   r)   r   r   s     r   r?   zLayerNormAct2d.forward  s    IIaAq!!? 	Y4#8$+tyRVRZ[[AAQ 5t{DItxXXAIIaAq!!IIaLLHHQKKr   r   rF   s   @r   r   r     se         g) ) ) ) ) ) 	 	 	 	 	 	 	r   r   )NFTr   )&rC   typingr   r   r   r   rL   r   torch.nnr   r8   torchvision.ops.miscr	   
create_actr   	fast_normr   r   r   trace_utilsr   r   r   r   rU   rH   rY   Moduler_   r   r   r   	GroupNormr   r   	LayerNormr   r   r   r   r   <module>r      sb    . - - - - - - - - - - -        $ $ $ $ $ $ 2 2 2 2 2 2 ( ( ( ( ( ( E E E E E E E E E E            1 1 1 1\ \ \ \ \R^ \ \ \~    r'   ' ' ' 'T=d =d =d =d =d58? =d =d =d@) ) )X& & &R  ! ! ! ! !2< ! ! !H    BL   8    2<   8    R\     r   