
    Ng/                     ~    d Z ddlZddlZddlZddlZddlmZ ddlm	Z	  ej
        e          Z G d de	          ZdS )z Cosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
    N)List   )	Schedulerc                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        j        dedededededdf fdZ	dede
e         fdZddZ xZS )CosineLRSchedulera\  
    Cosine decay with restarts.
    This is described in the paper https://arxiv.org/abs/1608.03983.

    Inspiration from
    https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

    k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
                  ?r   r   FTNq=
ףp?*   	optimizer	t_initiallr_min	cycle_mulcycle_decaycycle_limitreturnc           
          t                                          |d|
|||||           |dk    sJ |dk    sJ |dk    r&|dk    r |dk    rt                              d           | _        | _        | _        | _        | _        | _	         _
        |	 _        | _         j	        rA fd j        D              _        t                                           j
                   d S d  j        D              _        d S )Nlr)param_group_fieldt_in_epochsnoise_range_t	noise_pct	noise_std
noise_seed
initializer   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.c                 *    g | ]}|z
  j         z  S  )warmup_t).0vselfwarmup_lr_inits     T/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/scheduler/cosine_lr.py
<listcomp>z.CosineLRScheduler.__init__.<locals>.<listcomp>L   s%     ` ` `!!n"4!E ` ` `    c                     g | ]}d S )r   r   )r   _s     r#   r$   z.CosineLRScheduler.__init__.<locals>.<listcomp>O   s     = = =q = = =r%   )super__init___loggerwarningr   r   r   r   r   r   r"   warmup_prefixk_decaybase_valueswarmup_stepsupdate_groups)r!   r   r   r   r   r   r   r   r"   r,   r   r   r   r   r   r-   r   	__class__s   `       `        r#   r)   zCosineLRScheduler.__init__   sC   & 	"#'!! 	 		
 		
 		
 1}}}}{{{{>>i1nn1A1AOO>? ? ? #"&& ,*= 	> ` ` ` ` `tO_ ` ` `DGG!!$"566666 = =D,< = = =Dr%   tc                 `     j         k     r fd j        D             }n j        r
 j         z
   j        dk    rxt	          j        t	          j        d j        z  d j        z
  z  z
   j                            } j        |z   j        z  d j        |z  z
  d j        z
  z   j        z  z
  n j        z  } j         j        |z  z
   j        |z  fd j	        D             } j
        | j        k     r fd|D             }n fd j	        D             }|S )Nc                 *    g | ]}j         |z  z   S r   )r"   )r   sr!   r2   s     r#   r$   z-CosineLRScheduler._get_lr.<locals>.<listcomp>S   s%    JJJ14&Q.JJJr%   r   c                     g | ]}|z  S r   r   )r   r    gammas     r#   r$   z-CosineLRScheduler._get_lr.<locals>.<listcomp>b   s    AAA1QYAAAr%   c           
          g | ]E}j         d |j         z
  z  dt          j        t          j        z  z  z  z            z   z  z   FS )g      ?r   )r   mathcospi)r   lr_maxkr!   t_currt_is     r#   r$   z-CosineLRScheduler._get_lr.<locals>.<listcomp>f   sj        K#$+)=">!dhtwY_cdYdOdgjnogoOoFpFpBp"qq  r%   c                     g | ]	}j         
S r   )r   )r   r'   r!   s     r#   r$   z-CosineLRScheduler._get_lr.<locals>.<listcomp>k   s    ===qt{===r%   )r   r/   r,   r   r9   floorlogr   r   r.   r-   r   )	r!   r2   lrsilr_max_valuesr7   r=   r>   r?   s	   ``   @@@@r#   _get_lrzCosineLRScheduler._get_lrQ   s   t}JJJJJ8IJJJCC! &%~""JtxA,>!dnBT,U(UW[Weffggn)DN:a$.A"55!dn:LMPTP^^^'ndnq01$)EAAAA0@AAAMA4###      "/  
 >===D,<===
r%   c                     t          d|p| j                  }| j        dk    r
| j        |z  S t	          t          j        | j         | j        |z  dz
  z  d| j        z
  z                      S )Nr   r	   )maxr   r   r   intr9   rA   )r!   cycless     r#   get_cycle_lengthz"CosineLRScheduler.get_cycle_lengtho   sp    Q2$"233>S  >F**tz4>/T^v5MPQ5Q"RVWZ^ZhVh"ijjkkkr%   )r   r	   r	   r   r   r   FTNr
   r	   r   r	   T)r   )__name__
__module____qualname____doc__torchoptim	OptimizerrI   floatr)   r   rF   rK   __classcell__)r1   s   @r#   r   r      s         !!# #1> 1>{,1> 1> 	1>
 1> 1> 1>$ 
%1> 1> 1> 1> 1> 1>f e    <l l l l l l l lr%   r   )rO   loggingr9   numpynprP   typingr   	schedulerr   	getLoggerrL   r*   r   r   r%   r#   <module>r[      s                              '
H
%
%al al al al al	 al al al al alr%   