
    Ng                     :    d Z ddlZddlmZ  G d de          ZdS )an   PyTorch LARS / LARC Optimizer

An implementation of LARS (SGD) + LARC in PyTorch

Based on:
  * PyTorch SGD: https://github.com/pytorch/pytorch/blob/1.7/torch/optim/sgd.py#L100
  * NVIDIA APEX LARC: https://github.com/NVIDIA/apex/blob/master/apex/parallel/LARC.py

Additional cleanup and modifications to properly support PyTorch XLA.

Copyright 2021 Ross Wightman
    N)	Optimizerc                   r     e Zd ZdZ	 	 	 	 	 	 	 	 	 d fd	Z fdZ ej                    dd
            Z xZ	S )Larsa   LARS for PyTorch
    
    Paper: `Large batch training of Convolutional Networks` - https://arxiv.org/pdf/1708.03888.pdf

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining parameter groups.
        lr (float, optional): learning rate (default: 1.0).
        momentum (float, optional): momentum factor (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        dampening (float, optional): dampening for momentum (default: 0)
        nesterov (bool, optional): enables Nesterov momentum (default: False)
        trust_coeff (float): trust coefficient for computing adaptive lr / trust_ratio (default: 0.001)
        eps (float): eps for division denominator (default: 1e-8)
        trust_clip (bool): enable LARC trust ratio clipping (default: False)
        always_adapt (bool): always apply LARS LR adapt, otherwise only when group weight_decay != 0 (default: False)
          ?r   FMbP?:0yE>c                 F   |dk     rt          d|           |dk     rt          d|           |dk     rt          d|           |r|dk    s|dk    rt          d          t          ||||||||	|
	  	        }t                                          ||           d S )Ng        zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: r   z8Nesterov momentum requires a momentum and zero dampening)	lrmomentum	dampeningweight_decaynesterovtrust_coeffeps
trust_clipalways_adapt)
ValueErrordictsuper__init__)selfparamsr
   r   r   r   r   r   r   r   r   defaults	__class__s               K/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/optim/lars.pyr   zLars.__init__#   s     88;r;;<<<c>>BBBCCC#JLJJKKK 	YQ)q..WXXX%#!%

 

 

 	*****    c                     t                                          |           | j        D ]}|                    dd           d S )Nr   F)r   __setstate__param_groups
setdefault)r   stategroupr   s      r   r   zLars.__setstate__F   sP    U###& 	0 	0EZ////	0 	0r   Nc           
         d}|5t          j                    5   |            }ddd           n# 1 swxY w Y   | j        d         d         d         j        }t          j        d|          }| j        D ]}|d         }|d         }|d         }|d	         }	|d
         }
|d         }|d         D ]}|j        |j        }|dk    s|d         r|                    d          }|                    d          }|
|z  |||z  z   |z   z  }t          j        |dk    t          j        |dk    ||          |          }|d         rt          j        ||d         z  |          }|	                    ||           |
                    |           |dk    r| j        |         }d|vr,t          j        |                                          x}|d<   n5|d         }|
                    |          	                    |d|z
             |	r|                    ||          }n|}|	                    ||d                     |S )zPerforms a single optimization step.

        Args:
            closure (callable, optional): A closure that reevaluates the model and returns the loss.
        Nr   r   r   )devicer   r   r   r   r   r   r   g       @r   r
   )alphamomentum_buffer)torchenable_gradr   r$   tensorgradnormwhereminimumadd_mul_r!   clonedetachadd)r   closurelossr$   
one_tensorr"   r   r   r   r   r   r   pr*   w_normg_normtrust_ratioparam_statebufs                      r   stepz	Lars.stepK   s    "$$ ! !wyy! ! ! ! ! ! ! ! ! ! ! ! ! ! ! "1%h/29\#f555
& +	1 +	1E 0LZ(Hk*IZ(H.K,C8_ #1 #16>v  1$$n(=$VVC[[F!YYs^^F"-"6&6LCX:X[^:^"_K"'+
FQJZHH"# #K
 \* [&+mK%+4Mz&Z&ZIIa|I444IIk*** q=="&*Q-K(;;?D{4?P?P?W?W?Y?YYk*;<<)*;<**//BN/KKK ##xx8x<<"tE$K<0000G#1J s   /33)	r   r   r   r   Fr   r   FF)N)
__name__
__module____qualname____doc__r   r   r'   no_gradr<   __classcell__)r   s   @r   r   r      s         ( !+ !+ !+ !+ !+ !+F0 0 0 0 0
 U]__; ; ; _; ; ; ; ;r   r   )r@   r'   torch.optim.optimizerr   r    r   r   <module>rE      sj      + + + + + +v v v v v9 v v v v vr   