
    Ng                     |    d Z ddlZddlZddlZddlZddlZddlmZmZ  ej	        e
          Z G d d          ZdS )z Checkpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
    N   )unwrap_modelget_state_dictc            
       P    e Zd Zdddddddddef
dZddZdd	ZddZddZd Z	dS )CheckpointSaverN
checkpointrecovery F
   c                 Z   || _         || _        || _        || _        || _        g | _        d | _        d | _        d| _        d| _	        || _
        |	| _        || _        || _        d| _        |
| _        |
rt           j        nt           j        | _        || _        || _        | j        dk    sJ d S )Nr
   z.pth.tarr   )model	optimizerargs	model_ema
amp_scalercheckpoint_files
best_epochbest_metriccurr_recovery_filelast_recovery_filecheckpoint_dirrecovery_dirsave_prefixrecovery_prefix	extension
decreasingoperatorltgtcmpmax_history	unwrap_fn)selfr   r   r   r   r   checkpoint_prefixr   r   r   r   r!   r"   s                W/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/utils/checkpoint_saver.py__init__zCheckpointSaver.__init__   s      
"	"$ !#"$"$ -(,.#$",=8;;(+&"1$$$$$$    c                    |dk    sJ t           j                            | j        d| j        z             }t           j                            | j        d| j        z             }|                     |||           t           j                            |          rt          j        |           t          j        ||           | j	        r| j	        d         nd }t          | j	                  | j        k     s||                     ||d                   rt          | j	                  | j        k    r|                     d           d                    | j        t          |          g          | j        z   }t           j                            | j        |          }t          j        ||           | j	                            ||f           t%          | j	        d | j                   | _	        d	}| j	        D ]}	|d
                    |	          z  }t*                              |           || j        |                     || j                  r|| _        || _        t           j                            | j        d| j        z             }
t           j                            |
          rt          j        |
           t          j        ||
           | j        dn| j        | j        fS )Nr   tmplastr   -c                     | d         S )Nr    )xs    r%   <lambda>z1CheckpointSaver.save_checkpoint.<locals>.<lambda>Q   s
    QqT r'   )keyreversezCurrent checkpoints:
z {}

model_best)NN)ospathjoinr   r   _saveexistsunlinkrenamer   lenr!   r    _cleanup_checkpointsr   strlinkappendsortedr   format_loggerinfor   r   )r#   epochmetrictmp_save_pathlast_save_path
worst_filefilename	save_pathcheckpoints_strcbest_save_paths              r%   save_checkpointzCheckpointSaver.save_checkpoint?   s   zzzzT%8%$.:PQQd&96DN;RSS

=%0007>>.)) 	&In%%%
	-000262GQT*2..T
%&&)999>TXXfjm%D%D>4())T-===))!,,,xx!13u:: >??$.PHT%8(CCIGNI...!(()V)<===$*%>> O+%- %- %-D! 7O* 5 57>>!#4#44LL)))!t'7'?488FTXTdCeCe'?"'#) !#d.A<RVR`C`!a!a7>>.11 .In---777#/7||d>NPTP_=``r'   c                    |t          | j                  j                                        t	          | j        | j                  | j                                        dd}| j        | j        j        |d<   | j        |d<   | j	        &| j	                                        || j	        j
        <   | j        t	          | j        | j                  |d<   |||d<   t          j        ||           d S )N   )rD   arch
state_dictr   versionrQ   r   state_dict_emarE   )typer   __name__lowerr   r"   r   rR   r   r   state_dict_keyr   torchsave)r#   rJ   rD   rE   
save_states        r%   r7   zCheckpointSaver._savec   s    $$-3355(T^DD2244
 

 9 !%Jv!%Jv?&9=9S9S9U9UJt56>%+9$.$.+Y+YJ'(#)Jx 
:y)))))r'   r   c                    t          t          | j                  |          }| j        |z
  }|dk     st          | j                  |k    rd S | j        |d          }|D ]}	 t                              d                    |                     t          j        |d                    K# t          $ r7}t          
                    d                    |                     Y d }~d }~ww xY w| j        d |         | _        d S )Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)minr;   r   r!   rB   debugrA   r4   remove	Exceptionerror)r#   trimdelete_index	to_deletedes         r%   r<   z$CheckpointSaver._cleanup_checkpointsv   s   3t,--t44'$.!s4#899\IIF),--8	 	T 	TAT7>>qAABBB	!A$ T T THOOPQRRSSSSSSSST $ 5m|m Ds   !AB))
C*3-C%%C*c                    |dk    sJ d                     | j        t          |          t          |          g          | j        z   }t          j                             | j        |          }|                     ||           t          j                            | j	                  r	 t                              d                    | j	                             t	          j        | j	                   nJ# t          $ r=}t                              d                    || j	                             Y d }~nd }~ww xY w| j        | _	        || _        d S )Nr   r,   zCleaning recovery: {}z Exception '{}' while removing {})r6   r   r=   r   r4   r5   r   r7   r8   r   rB   r^   rA   r_   r`   ra   r   )r#   rD   	batch_idxrI   rJ   rf   s         r%   save_recoveryzCheckpointSaver.save_recovery   s6   zzzz88T13u::s9~~NOORVR``GLL!2H==	

9e$$$7>>$122 	ee5<<T=TUUVVV	$12222 e e e@GG4Kbccdddddddde"&"9"+s   (AC4 4
D;>3D66D;c                     t           j                            | j        | j                  }t          j        |dz   | j        z             }t          |          }t          |          r|d         ndS )N*r   r
   )	r4   r5   r6   r   r   globr   r@   r;   )r#   recovery_pathfiless      r%   find_recoveryzCheckpointSaver.find_recovery   s\    T%68LMM	-#->??uu::-uQxx2-r'   )N)r   )
rV   
__module____qualname__r   r&   rN   r7   r<   ri   ro   r.   r'   r%   r   r      s        
 *&"'% '% '% '%R"a "a "a "aH* * * *&E E E E, , , ,. . . . .r'   r   )__doc__rl   r   r4   loggingrY   r   r   r   	getLoggerrV   rB   r   r.   r'   r%   <module>ru      s       				   / / / / / / / / '
H
%
%A. A. A. A. A. A. A. A. A. A.r'   