
    Ng                         d Z ddlZddlmc mZ ddlmZ ddlZdej	        fdZ
dej	        fdZdeded	efd
Z G d de          ZdS )a3  
AdamP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/adamp.py

Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217
Code: https://github.com/clovaai/AdamP

Copyright (c) 2020-present NAVER Corp.
MIT license
    N)	Optimizerreturnc                 T    |                      |                     d          d          S )Nr   )reshapesizexs    L/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/optim/adamp.py_channel_viewr      s     99QVVAYY###    c                 .    |                      dd          S )N   r   )r   r	   s    r   _layer_viewr      s    99Qr   deltawd_ratioepsc                 t   d}ddt          | j                  dz
  z  z   }t          t          fD ]} ||           }	 ||          }
t	          j        |
|	d|                                          }|                                |t          j	        |	
                    d                    z  k     r| |	                    dd                              |                              |          z  }|| |||z                                d                              |          z  z  }|}||fc S ||fS )	N      ?)r   )r   r   )dimr      )pr   )r   )lenshaper   r   Fcosine_similarityabs_maxmathsqrtr   normadd_r   sum)r   gradperturbr   r   r   wdexpand_size	view_func
param_view	grad_view
cosine_simp_ns                r   
projectionr-      sA   	B$#ag,,"233K#[1 
 
	Yq\\
IdOO	(JA3OOOTTVV
 >>edi
0B0B&C&CCCCjooqo1166s;;CCKPPPCsYYsW}5599a9@@HHUUUUGBB;	 D B;r   c                   V     e Zd Z	 	 d
 fd	Z ej                    dd	            Z xZS )AdamPMbP?g?g+?:0yE>r   皙?Fc	           	          t          |||||||          }	t          t          |                               ||	           d S )N)lrbetasr   weight_decayr   r   nesterov)dictsuperr/   __init__)selfparamsr5   r6   r   r7   r   r   r8   defaults	__class__s             r   r;   zAdamP.__init__,   sO    Cl(X? ? ? 	eT##FH55555r   Nc                    d }|5t          j                    5   |            }d d d            n# 1 swxY w Y   | j        D ]}|d         D ]}|j        |j        }|d         \  }}|d         }| j        |         }	t          |	          dk    r3d|	d<   t          j        |          |	d<   t          j        |          |	d<   |	d         |	d         }}
|	dxx         dz  cc<   d||	d         z  z
  }d||	d         z  z
  }|
                    |                              |d|z
  	           |                    |          	                    ||d|z
  
           |
                                t          j
        |          z                      |d                   }|d         |z  }|r||
z  d|z
  |z  z   |z  }n|
|z  }d}t          |j                  dk    r)t          ||||d         |d         |d                   \  }}|d         dk    r*|                    d|d         |d         z  |z  z
             |                    || 	           |S )Nr=   r6   r8   r   stepexp_avg
exp_avg_sqr   )alpha)valuer   r5   r   r   r   r7   )torchenable_gradparam_groupsr$   stater   
zeros_likemul_r"   addcmul_r    r   r   r-   )r<   closurelossgroupr   r$   beta1beta2r8   rI   rB   rC   bias_correction1bias_correction2denom	step_sizer%   r   s                     r   rA   z
AdamP.step3   s   "$$ ! !wyy! ! ! ! ! ! ! ! ! ! ! ! ! ! ! & -	2 -	2E8_ ,2 ,26>v$W~u ,
1 u::??$%E&M','7':':E)$*/*:1*=*=E,' ',I&6l8Kf"#$uf'=#= #$uf'=#= U##((QY(???&&//d!e)/LLL#**TY7G-H-HHNNuUZ|\\!$K*::	 .$w!e)t1CCuLGG%oG qw<<!##(21dGU7^UZ[eUfhmnsht(u(u%GX (1,,FF2deN.C Ch NNOOO wyj1111Y,2\ s   /33)r0   r1   r2   r   r3   r3   F)N)__name__
__module____qualname__r;   rF   no_gradrA   __classcell__)r?   s   @r   r/   r/   +   sk        @DCH6 6 6 6 6 6 U]__5 5 5 _5 5 5 5 5r   r/   )__doc__rF   torch.nn.functionalnn
functionalr   torch.optim.optimizerr   r   Tensorr   r   floatr-   r/    r   r   <module>rc      s               + + + + + + $ $ $ $ $el      U    $> > > > >I > > > > >r   