
    Ng                        d Z ddlmZ ddlZddlmZ  G d de          Z	 	 ddeej                 d	eej                 d
eej                 dedede	de	de	de	fdZ
deej                 d	eej                 d
eej                 de	de	de	de	defdZdeej                 d	eej                 d
eej                 de	de	de	de	defdZdS )z Lion Optimizer
Paper: `Symbolic Discovery of Optimization Algorithms` - https://arxiv.org/abs/2302.06675
Original Impl: https://github.com/google/automl/tree/master/lion
    )ListN)	Optimizerc                   j     e Zd ZdZ	 	 	 	 	 d
 fd	Z fdZ ej                    dd	            Z xZ	S )LionzImplements Lion algorithm.-C6?g?gGz?        FNc                    d|k    s"t          d                    |                    d|d         cxk    rdk     s*n t          d                    |d                             d|d         cxk    rdk     s*n t          d                    |d                             t          |||||          }t                                          ||           d	S )
a  Initialize the hyperparameters.

        Args:
          params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
          lr (float, optional): learning rate (default: 1e-4)
          betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.99))
          weight_decay (float, optional): weight decay coefficient (default: 0)
        r	   zInvalid learning rate: {}r   g      ?z%Invalid beta parameter at index 0: {}   z%Invalid beta parameter at index 1: {})lrbetasweight_decayforeachmaximizeN)
ValueErrorformatdictsuper__init__)	selfparamsr   r   r   r   r   defaults	__class__s	           K/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/optim/lion.pyr   zLion.__init__   s    ( byy8??CCDDDeAh$$$$$$$$DKKERSHUUVVVeAh$$$$$$$$DKKERSHUUVVV%
 
 
 	*****    c                     t                                          |           | j        D ].}|                    dd           |                    dd            /d S )Nr   Fr   )r   __setstate__param_groups
setdefault)r   stategroupr   s      r   r   zLion.__setstate__?   sf    U###& 	. 	.EZ///Y----	. 	.r   c                    d}|5t          j                    5   |            }ddd           n# 1 swxY w Y   | j        D ]}g }g }g }|d         \  }}|d         D ]}	|	j        
|                    |	           |	j        j        rt          d          |                    |	j                   | j        |	         }
t          |
          dk    r#t          j	        |	t           j
                  |
d<   |                    |
d                    t          ||||||d         |d	         |d
         |d         	  	         |S )zPerforms a single optimization step.

        Args:
          closure (callable, optional): A closure that reevaluates the model
            and returns the loss.

        Returns:
          the loss.
        Nr   r   z&Lion does not support sparse gradientsr   )memory_formatexp_avgr   r   r   r   )beta1beta2r   r   r   r   )torchenable_gradr   gradappend	is_sparseRuntimeErrorr    len
zeros_likepreserve_formatlion)r   closurelossr!   params_with_gradgradsexp_avgsr%   r&   pr    s              r   stepz	Lion.stepE   s    "$$ ! !wyy! ! ! ! ! ! ! ! ! ! ! ! ! ! ! &  	  	E!EH >LE58_ 2 26> ''***6# Q&'OPPPQV$$$
1 u::??','7I^'_'_'_E)$i 01111 ;">2z*i(
 
 
 
 
 s   /33)r   r   r	   FN)N)
__name__
__module____qualname____doc__r   r   r'   no_gradr7   __classcell__)r   s   @r   r   r      s        %%
 !+ !+ !+ !+ !+ !+F. . . . . U]__1 1 1 _1 1 1 1 1r   r   Fr   r4   r5   r   r   r%   r&   r   r   c          
          |d}|r-t           j                                        rt          d          |r&t           j                                        st          }	nt
          }	 |	| |||||||           dS )z=Functional API that performs Lion algorithm computation.
    NFz6torch.jit.script not supported with foreach optimizers)r%   r&   r   r   r   )r'   jitis_scriptingr,   _multi_tensor_lion_single_tensor_lion)
r   r4   r5   r   r   r%   r&   r   r   funcs
             r   r0   r0   z   s       U59))++ USTTT #uy--// #!"D!	 	 	 	 	 	r   c                   t          |           D ]\  }}	|s||         n||          }
||         }t          j        |	          r<t          j        |
          }
t          j        |          }t          j        |	          }	|	                    d||z  z
             |                    |                              |
d|z
            }|	                    t          j        |          |            |                    |
d|z
             d S )Nr   alpha)		enumerater'   
is_complexview_as_realmul_muladd_signlerp_)r   r4   r5   r%   r&   r   r   r   iparamr)   r$   updates                r   rB   rB      s    f%% ' '5'6uQxxeAhY1+E"" 	.%d++D(11G&u--E 	

1rL(())) U##((QY(??

5:f%%bS
111 	dAI&&&&#' 'r   c                   t          |           dk    rd S |r!t          j        t          |                    }d |D             }d |D             }d | D             } t          j        | d||z  z
             t          j        ||          }t          j        ||d|z
             d |D             }t          j        | ||            t          j        ||           t          j        ||d|z
             d S )Nr   c                 b    g | ],}t          j        |          rt          j        |          n|-S  r'   rH   rI   .0xs     r   
<listcomp>z&_multi_tensor_lion.<locals>.<listcomp>   s8    PPPQe&6q&9&9@U"""qPPPr   c                 b    g | ],}t          j        |          rt          j        |          n|-S rT   rU   rV   s     r   rY   z&_multi_tensor_lion.<locals>.<listcomp>   s8    VVV)9!)<)<C"1%%%!VVVr   c                 b    g | ],}t          j        |          rt          j        |          n|-S rT   rU   rV   s     r   rY   z&_multi_tensor_lion.<locals>.<listcomp>   s8    RRRau'7':':Ae ###RRRr   r   rE   c                 6    g | ]}|                                 S rT   )rM   )rW   us     r   rY   z&_multi_tensor_lion.<locals>.<listcomp>   s     )))Aqvvxx)))r   )r-   r'   _foreach_negtuple_foreach_mul__foreach_mul_foreach_add_)	r   r4   r5   r%   r&   r   r   r   updatess	            r   rA   rA      s$    6{{a 1"5<<00PP%PPPEVVXVVVHRR6RRRF 
B$5 5666  511G	a%i8888)))))G	s3333 
%(((	%q5y999999r   )FN)r;   typingr   r'   torch.optim.optimizerr   r   Tensorboolfloatr0   rB   rA   rT   r   r   <module>ri      s   $        + + + + + +^ ^ ^ ^ ^9 ^ ^ ^N % %U\"%EL!% u|$% % % % % % % % % %P'U\"'EL!' u|$'
 ' ' ' ' ' ' ' '>!:U\"!:EL!!: u|$!:
 !: !: !: !: !: !: !: !: !: !:r   