
    g                         d dl Z d dlZddlmZmZ ddlmZmZmZm	Z	  e	            r	d dl
mc mZ d Z G d dej        j                  Zdefd	ZdS )
    N   )AcceleratorStateGradientState)DistributedType
honor_typeis_lomo_availableis_torch_xla_availablec                 l   t          | t          t          f          rt          | fd| D                       S t          | t                    r6 t          |           fd|                                 D                       S t          | t          j                  r| 	                              S | S )Nc              3   8   K   | ]}t          |          V  d S Nmove_to_device).0tdevices     P/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/optimizer.py	<genexpr>z!move_to_device.<locals>.<genexpr>   s-      !K!K.F";";!K!K!K!K!K!K    c                 8    i | ]\  }}|t          |          S  r   )r   kvr   s      r   
<dictcomp>z"move_to_device.<locals>.<dictcomp>   s)    SSSTQA~a88SSSr   )

isinstancelisttupler   dicttypeitemstorchTensorto)stater   s    `r   r   r      s    %$''  %!K!K!K!KU!K!K!KLLL	E4	 	   tE{{SSSSU[[]]SSSTTT	E5<	(	(  xxLr   c                      e Zd ZdZddZed             Zej        d             Zed             Zej        d             Zed	             Z	e	j        d
             Z	d Z
d Zd ZddZd Zd ZddZd Zed             Zd Zd ZdS )AcceleratedOptimizera  
    Internal wrapper around a torch optimizer.

    Conditionally will perform `step` and `zero_grad` if gradients should be synchronized when performing gradient
    accumulation.

    Args:
        optimizer (`torch.optim.optimizer.Optimizer`):
            The optimizer to wrap.
        device_placement (`bool`, *optional*, defaults to `True`):
            Whether or not the optimizer should handle device placement. If so, it will place the state dictionary of
            `optimizer` on the right device.
        scaler (`torch.cuda.amp.grad_scaler.GradScaler`, *optional*):
            The scaler to use in the step function if training with mixed precision.
    TNc                    || _         || _        t                      | _        t	                      | _        || _        d| _        | j        7d| _        | j         j	        | _
        t          | | j         j	                  | _        |r| j                                         }| j        j        t          j        k    r t#          j        || j        j                   nt)          || j        j                  }| j                             |           d S d S NF)	optimizerscalerr   accelerator_stater   gradient_statedevice_placement_is_overflow_accelerate_step_calledstep_optimizer_original_step_methodpatch_optimizer_step_optimizer_patched_step_method
state_dictdistributed_typer   XLAxmsend_cpu_data_to_devicer   r   load_state_dict)selfr(   r,   r)   r3   s        r   __init__zAcceleratedOptimizer.__init__6   s    "!1!3!3+oo 0!;"+0D(37>3FD02FtT^M`2a2aD/  	72244J%6/:MMM*:t7M7TUUUU+J8N8UVV
N**:66666	7 	7r   c                     | j         j        S r   r(   r#   r9   s    r   r#   zAcceleratedOptimizer.stateL   s    ~##r   c                     || j         _        d S r   r<   r9   r#   s     r   r#   zAcceleratedOptimizer.stateP   s    $r   c                     | j         j        S r   r(   param_groupsr=   s    r   rB   z!AcceleratedOptimizer.param_groupsT   s    ~**r   c                     || j         _        d S r   rA   )r9   rB   s     r   rB   z!AcceleratedOptimizer.param_groupsX   s    &2###r   c                     | j         j        S r   r(   defaultsr=   s    r   rF   zAcceleratedOptimizer.defaults\   s    ~&&r   c                     || j         _        d S r   rE   )r9   rF   s     r   rF   zAcceleratedOptimizer.defaults`   s    "*r   c                 :    | j                             |           d S r   )r(   add_param_group)r9   param_groups     r   rI   z$AcceleratedOptimizer.add_param_groupd   s    &&{33333r   c                     | j         j        t          j        k    r&| j        rt          j        || j         j                   | j        	                    |           d S r   )
r*   r4   r   r5   r,   r6   r7   r   r(   r8   )r9   r3   s     r   r8   z$AcceleratedOptimizer.load_state_dictg   sS    !2o6IIIdNcI&z43I3PQQQ&&z22222r   c                 4    | j                                         S r   )r(   r3   r=   s    r   r3   zAcceleratedOptimizer.state_dictl   s    ~((***r   c                    | j         j        rtdt          j        | j        j                  j        v }|r!|d}| j                            |           d S |t          d          | j                                         d S d S )Nset_to_noneT)rN   zJ`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.)r+   sync_gradientsinspect	signaturer(   	zero_grad
parameters
ValueError)r9   rN   
accept_args      r   rR   zAcceleratedOptimizer.zero_grado   s    - 		+&'*;DN<T*U*U*``J +&"&K(([(AAAAA*$%qrrr((*****		+ 		+r   c                     t          | j        d          r4t          | j        j                  r| j                                         dS dS dS )z`
        Sets the optimizer to "train" mode. Useful for optimizers like `schedule_free`
        trainN)hasattrr(   callablerW   r=   s    r   rW   zAcceleratedOptimizer.train{   sZ     4>7++ 	#9M0N0N 	#N  """""	# 	# 	# 	#r   c                     t          | j        d          r4t          | j        j                  r| j                                         dS dS dS )z_
        Sets the optimizer to "eval" mode. Useful for optimizers like `schedule_free`
        evalN)rX   r(   rY   r[   r=   s    r   r[   zAcceleratedOptimizer.eval   sZ     4>6** 	"x8K/L/L 	"N!!!!!	" 	" 	" 	"r   c                    t                      rddlm}m} | j        j        sj| j        j        t          j	        k    rPt          j        | j                  }t          j        d|dt          j                    z             d| j        _        t                      rt          | j        ||f          rd S | j        j        r| j        y| j        | j        _        | j                            | j        |           | j                                         | j        sd| _        nd| _        | j        | j        _        d| _        n| j                            |           | j        j        t          j	        k    rd| j        _        d S d S )Nr   )AdaLomoLomosumg      ?)scaleTF)r   
lomo_optimr]   r^   r+   is_xla_gradients_syncedr*   r4   r   r5   r6   _fetch_gradientsr(   
all_reducexrt_world_sizer   rO   r)   r2   r/   updater.   r-   r0   )r9   closurer]   r^   	gradientss        r   r/   zAcceleratedOptimizer.step   s    	100000000 #;	?&7?;NNN+DN;;IM%#8I8K8K2KLLLL:>D7 	$.4/:: - 	-{&&*&I#  999""$$$3 .(,D%%(-D%&*&J#/4,,##G,,,!2o6III:?D777 JIr   c                 T    | j         j        D ]}fd|d         D             |d<   d S )Nc                 <    g | ]}                     ||          S r   )get)r   pparameters_maps     r   
<listcomp>z;AcceleratedOptimizer._switch_parameters.<locals>.<listcomp>   s)    $]$]$]!^%7%71%=%=$]$]$]r   paramsrA   )r9   rm   rJ   s    ` r   _switch_parametersz'AcceleratedOptimizer._switch_parameters   sL    >6 	^ 	^K$]$]$]$]{S[G\$]$]$]K!!	^ 	^r   c                     | j         S )z.Whether or not the optimizer step was skipped.)r-   r=   s    r   step_was_skippedz%AcceleratedOptimizer.step_was_skipped   s       r   c                 V    g dfd| j                                         D             S )N)r.   r0   r2   c                 $    i | ]\  }}|v	||S r   r   )r   r   r   _ignored_keyss      r   r   z5AcceleratedOptimizer.__getstate__.<locals>.<dictcomp>   s)    QQQA!=:P:P1:P:P:Pr   )__dict__r   )r9   ru   s    @r   __getstate__z!AcceleratedOptimizer.__getstate__   s?    
 
 

 RQQQ!4!4!6!6QQQQr   c                     | j                             |           | j        9d| _        | j        j        | _        t          | | j        j                  | _        d S d S r'   )	rv   rf   r)   r.   r(   r/   r0   r1   r2   r?   s     r   __setstate__z!AcceleratedOptimizer.__setstate__   sZ    U###;"+0D(37>3FD02FtT^M`2a2aD/// #"r   )TNr   )__name__
__module____qualname____doc__r:   propertyr#   setterrB   rF   rI   r8   r3   rR   rW   r[   r/   rp   rr   rw   ry   r   r   r   r%   r%   %   s         7 7 7 7, $ $ X$ \% % \% + + X+ 3 3 3 ' ' X' _+ + _+4 4 43 3 3
+ + +
+ 
+ 
+ 
+# # #" " "$@ $@ $@ $@L^ ^ ^ ! ! X!R R Rb b b b br   r%   accelerated_optimizerc                       fd}|S )Nc                  "    d_          | i |S )NT)r.   )argskwargsr   methods     r   patched_stepz*patch_optimizer_step.<locals>.patched_step   s!    8<5vt&v&&&r   r   )r   r   r   s   `` r   r1   r1      s*    ' ' ' ' ' ' r   )rP   r    r#   r   r   utilsr   r   r   r	   torch_xla.core.xla_modelcore	xla_modelr6   r   optim	Optimizerr%   r1   r   r   r   <module>r      s      2 2 2 2 2 2 2 2 Y Y Y Y Y Y Y Y Y Y Y Y  *)))))))))  `b `b `b `b `b5;0 `b `b `bF0D      r   