
    gH                         d dl mZ d dlZd dlmc mZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d d	lmZmZ d
 ZddZddZd Zd Zd ZddZ	 ddZ d Z!d Z"d Z#e$dk    r e"             dS dS )    )deepcopyN)AdamW)LambdaLR)
DataLoader)AcceleratorDataLoaderConfigurationGradientAccumulationPlugin)GradientState)RegressionDatasetRegressionModel)DistributedTypeset_seedc           	         t          |                                 |                                          D ]\  }}|j        s|s>t          j        |j        |j        fi |du sJ d| d|j         d|j         d            Mt          j        |j        |j        fi |du sJ d| d|j         d|j         d            d S )	NF7Gradients in sync when they should not be at iteration z:
model_a grad (z) == model_b grad ()T7Gradients not in sync when they should be at iteration z) != model_b grad ()zip
parametersrequires_gradtorchallclosegrad)model_amodel_bdid_step	iterationkwargsparam
grad_params          c/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/test_utils/scripts/test_sync.pycheck_model_parametersr!      sW    !3!3!5!5w7I7I7K7KLL W Wz" 	 		W uz:?EEfEENNN W  W  Wejeo  W  W  EO  ET  W  W  W ONNN
 uz:?EEfEEMMM W  W  Wejeo  W  W  EO  ET  W  W  W NMMMW W    Tc                    |                                    | |          }t          j        ||                    |j                            }|s ||j        z  }|                                 d S |                    |           d S N)trainFmse_losstodevicegradient_accumulation_stepsbackward)modelinputtargetacceleratordo_backwardoutputlosss          r    
step_modelr3   -   s{    	KKMMMU5\\F:ffii6677D #77T"""""r"   Fc                 :   t          d           t                      }t          |          }t          d          }t	          |d          }|                    | j                   |rjt          |                                d          }t          |                                d          }t          |d 	          }t          |d
 	          }|r| 
                    ||||          \  }}}}n| 
                    ||          \  }}|r	|||||||fS |||fS )z3Returns everything needed to perform basic training*   P   length   
batch_sizeMbP?)paramslrc                     | dz  S Ng? epochs    r    <lambda>z$get_training_setup.<locals>.<lambda>C   s
    eTk r"   )	lr_lambdac                     | dz  S r@   rA   rB   s    r    rD   z$get_training_setup.<locals>.<lambda>D   s
    eTk r"   )r   r   r   r   r   r(   r)   r   r   r   prepare)	r/   schedr,   	ddp_modeldset
dataloaderoptddp_opt	ddp_scheds	            r    get_training_setuprO   8   s@   RLLLEIB'''DDR000J	HH[    K5++--$777y3355$???(A(ABBBW0I0IJJJ	 K4?4G4G	SZ\egq4r4r1	7Izz + 3 3Iz J J	: NsE:y'9MM)Z''r"   c                    t          |           \  }}}t          t          |                                                    \  }}t	          d          D ]}|                     ||f          \  }}|                    | j                  |                    | j                  }}t          ||||            |dz  dk    r@| 	                    |          5  t          ||||            d d d            n# 1 swxY w Y   nt          ||||            t          ||d|           t          |                                |                                          D ]G\  }	}
|	j        st          j        |	j        |
j                  sJ d|	j         d|
j         d            Ht          j        d|z              |t          j        t'          |                             }d S )	N      r   T7Gradients not in sync when they should be:
Model grad () != DDP grad (r   9  )rO   nextitervaluesrangegatherr(   r)   r3   no_syncr!   r   r   r   r   r   r   manual_seedrandpermlenr/   r,   rI   rK   	ddp_input
ddp_targetr   r-   r.   r   	ddp_params              r    test_noop_syncrc   O   sN   #5k#B#B E9j j!1!12299;;Iz1XX > >	#**Iz+BCCv!344fii@R6S6Sv5%555q=A$$Y// J J9i[IIIJ J J J J J J J J J J J J J J y)ZEEE 	uiyAAA #E$4$4$6$6	8L8L8N8N O O 	w 	wE9& >
IN  w wv%*vvenesvvvw w w w
 	$*+++enS^^<<=		5> >   C44C8	;C8	c                 0   t          |           \  }}}t          t          |                                                    \  }}t	          d          D ]}|                     ||f          \  }}|                    | j                  |                    | j                  }}t          ||||            |dz  dk    r@| 	                    |          5  t          ||||            d d d            n# 1 swxY w Y   nt          ||||            t          |                                |                                          D ]\  }	}
|	j        s|dz  dk    r=t          j        |	j        |
j                  du sJ d|	j         d|
j         d            St          j        |	j        |
j                  du sJ d	|	j         d
|
j         d            t          j        d|z              |t          j        t%          |                             }d S )NrQ   rR   r   F7Gradients in sync when they should not be:
Model grad () == DDP grad (r   TrS   rT   rU   )rO   rV   rW   rX   rY   rZ   r(   r)   r3   r[   r   r   r   r   r   r   r\   r]   r^   r_   s              r    test_distributed_syncrh   q   s   #5k#B#B E9j j!1!12299;;Iz1XX  >  >	#**Iz+BCCv!344fii@R6S6Sv5%555q=A$$Y// J J9i[IIIJ J J J J J J J J J J J J J J y)ZEEE !$E$4$4$6$6	8L8L8N8N O O 	{ 	{E9& 1}!! N5:y~>>%GGGzejzziriwzzz HGGG
 N5:y~>>$FFFzejzziriwzzz GFFF 	$*+++enS^^<<=		A >  >rd   c                 d   t          |           \  }}}g }d}t          |          D ]}t          t          |                                                    \  }}|                     ||f          \  }	}
|	                    | j                  |
                    | j                  }
}	t          ||	|
|            | 	                    |          5   ||          }t          j        ||                    |j                            }|                    |           d d d            n# 1 swxY w Y   t          |          D ]h}||         }||dz
  k     r|                     |           t          |                                |                                          D ]I\  }}|j        st#          j        |j        |j                  du sJ d|j         d|j         d            J|                     |          5  |                     |           d d d            n# 1 swxY w Y   t          |                                |                                          D ]I\  }}|j        st#          j        |j        |j                  du sJ d|j         d	|j         d            Jjd S )
NrQ      Frf   rg   r   TrS   rT   )rO   rY   rV   rW   rX   rZ   r(   r)   r3   r[   r&   r'   appendr+   r   r   r   r   r   r   trigger_sync_in_backward)r/   r,   rI   rK   lossesnum_iterationsr   r`   ra   r-   r.   
ddp_outputr2   r   rb   s                  r    "test_distributed_sync_multiple_fwdrp      sk   #5k#B#B E9jFN>**    	 $T*%5%5 6 6 = = ? ?	: $**Iz+BCCv!344fii@R6S6Sv 	5%555   ++ 	  	 "9--J:j*--
8I*J*JKKDMM$	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  >** { {	i ~)))  &&& %((8(8(:(:I<P<P<R<R$S$S { { y*  N5:y~>>%GGGzejzziriwzzz HGGG{ 55i@@ + +$$T***+ + + + + + + + + + + + + + + %((8(8(:(:I<P<P<R<R$S$S { { y*  N5:y~>>$FFFzejzziriwzzz GFFF{-{ {s%   AD**D.	1D.	H%%H)	,H)	c           
         t          d|          }t          | |          }t          ||          }t          |          \  }}}t	          |          D ]\  }	}
|
                                \  }}|                    ||f          \  }}|                    |j                  |                    |j                  }}t          ||||d           |
                    |          5  t          ||||           d d d            n# 1 swxY w Y   t          |                                |                                          D ]\  }}|j        s|	dz   dz  dk    s|	t          |          dz
  k    s|r@t          j        |j        |j                  du sJ d	|	 d
|j         d|j         d            qt          j        |j        |j                  du sJ d|	 d
|j         d|j         d            t          j        d|	z              |t          j        t          |                             }t)          j                     d S )NrR   	num_stepssync_each_batchsplit_batchesdispatch_batchesdataloader_configgradient_accumulation_pluginFrj   r   Tr   z:
Model grad (rT   r   r   rg   rU   )r	   r   r   rO   	enumeraterX   rZ   r(   r)   r3   
accumulater   r   r   r^   r   r   r   r\   r]   r
   _reset_state)rv   rw   rt   rz   ry   r/   r,   rI   rK   r   batchr`   ra   r-   r.   r   rb   s                    r    test_gradient_accumulationr      s   #=[j#k#k#k /m^nooo+%A  K
 $6k#B#B E9j%j11 > >	5 %	:#**Iz+BCCv!344fii@R6S6Sv5%e<<<##I.. 	F 	Fy)ZEEE	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F !$E$4$4$6$6	8L8L8N8N O O 	T 	TE9& Q!#q((i3z??Q;N.N.NSb.N N5:y~>>$FFF TY  T  Tglgq  T  T  CL  CQ  T  T  T GFFF
 N5:y~>>%GGG TY  T  Tglgq  T  T  CL  CQ  T  T  T HGGG 	$*+++enS^^<<=		     s   )DD	D	c                    t          d|          }t          | |          }t          ||          }t          |d          \  }}}}	}
}}t	          |	          D ]d\  }}|                                \  }}|                    ||f          \  }}|                    |j                  |                    |j                  }}|	                                 |
	                                 t          ||||d           |                                 |dz   dz  dk    s|dz   t          |	          k    rB| r|                                 n+t          |j                  D ]}|                                 |                    |
          5  t          |
|||           |                                 |                                 d d d            n# 1 swxY w Y   |j        d         d	         |j        d         d	         k    s3J d
|j        d         d	          d|j        d         d	          d            |dz   dz  dk    p|dz   t          |	          k    }|j        dk    rt#          ||
|p||d           |r|                                 |                                 t'          j        d|z              ft+          j                     d S )NrR   rr   ru   rx   TFrj   r   r>   z:Learning rates found in each optimizer did not align
opt: z

DDP opt: 
r<   )rtolrU   )r	   r   r   rO   r{   rX   rZ   r(   r)   r%   r3   stepr^   rY   num_processesr|   param_groupsr!   	zero_gradr   r\   r
   r}   )rv   rw   rt   rz   ry   r/   r,   rL   rH   rK   rI   rM   rN   r   r~   r`   ra   r-   r.   _r   s                        r    1test_gradient_accumulation_with_opt_and_schedulerr      si    $>[j#k#k#k /m^nooo+%A  K
 DVVacgChCh@E3z9gy%j11 +, +,	5 %	:#**Iz+BCCv!344fii@R6S6Sv5%e<<<


]a1$$9q=S__*L*L !

{899 ! !AJJLLLL ##I.. 	 	y)ZEEELLNNNNN	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 Q%)=a)@)FFFF QIYZ[I\]aIb  Q  Qov  pD  EF  pG  HL  pM  Q  Q  Q GFF!mq(Q.WY]s:4V$q(("+O     	MMOOO 	$*++++     s   ;GG	G	c                     t                      } t          d          }t          |d          }t          d          }t          |d          }|                     ||          \  }}| j        j        J t          |          D ]\  }}t          | j        j                  t          |          k    sJ |t          |          dz
  k     r| j        j	        rJ |dk    rtt          |          D ]d\  }}t          | j        j                  t          |          k    sJ |t          |          dz
  k     r| j        j	        rJ V| j        j	        sJ e| j        j	        sJ | j        j        J d S )Nr6   r7   r9   r:   `   rj   )
r   r   r   rG   gradient_stateactive_dataloaderr{   idr^   end_of_dataloader)r/   
first_dsetfirst_dataloadersecond_dsetsecond_dataloaderr   r   	batch_nums           r    test_dataloader_breakr   2  s   --K""---J!*<<<#2...K";2>>>*5*=*=>NPa*b*b''%7???!"233 @ @	1+,>??2FVCWCWWWWWs+,,q000"1CCCCA~~$-.?$@$@ L LLIqk8JKKrRcOdOddddd 3'8#9#9A#===#.#=#OOOOO*9KKKKK-?????%7?????r"   c                     t                      } | j        }|j        dk    rt          d           |j        t
          j        k    rt                       |j        t
          j        k    r)|j        dk    rt          d           t          |            |j        t
          j
        t
          j        t
          j        t
          j        t
          j        fv rR|j        dk    rt          d           t          |            |j        dk    rt          d           t!          |            |j        t
          j
        t
          j        t
          j        t
          j        fv rEdD ]B}dD ]=}dD ]8}|j        dk    rt          dd| d	| d
| d           t#          |||           9>C|j        dk    rt          dd           t%                       |j        t
          j
        t
          j        t
          j        t
          j        fv rLdD ]I}dD ]D}dD ]?}|s|s|s	|j        dk    rt          dd| d	| d
| d           t%          |||           @EJ|                                 d S )Nr   zA**Test `accumulate` gradient accumulation with dataloader break**z'**Test NOOP `no_sync` context manager**z.**Test Distributed `no_sync` context manager**zE**Test Distributed `no_sync` context manager with multiple forwards**)TFz+**Test `accumulate` gradient accumulation, z`split_batches=z` and `dispatch_batches=z` and `sync_each_batch=z`**zH**Test `accumulate` gradient accumulation with optimizer and scheduler, zJ`split_batches=False`, `dispatch_batches=False`, `sync_each_batch=False`**)r   statelocal_process_indexprintdistributed_typer   XLAr   NOrc   	MULTI_GPU	MULTI_NPU	MULTI_MLU
MULTI_MUSA	MULTI_CPUrh   rp   r   r   destroy_process_group)r/   r   split_batchrw   rt   s        r    mainr   K  sL   --KE A%%QRRR!444!333$));<<<{###!!!"!"   $))BCCCk***$))YZZZ*;777!!!"	"   ) 	_ 	_K$1 _ _ '4 _ _O0A55I Qk  Q  QSc  Q  Q  }L  Q  Q  Q   /{<Lo^^^^__  A%%VX	
 	
 	
 6777!!!"	"   ) 
	v 
	vK$1 	v 	v '4 v vO& !/? ! ! 0A55f Qk  Q  QSc  Q  Q  }L  Q  Q  Q   FkScetuuuuv	v 
!!!!!r"   c                 "    t                       d S r$   )r   )indexs    r    _mp_fnr     s    FFFFFr"   __main__)T)F)FFF)%copyr   r   torch.nn.functionalnn
functionalr&   torch.optimr   torch.optim.lr_schedulerr   torch.utils.datar   accelerate.acceleratorr   r   r	   accelerate.stater
   accelerate.test_utilsr   r   accelerate.utilsr   r   r!   r3   rO   rc   rh   rp   r   r   r   r   r   __name__rA   r"   r    <module>r      s                         - - - - - - ' ' ' ' ' ' c c c c c c c c c c * * * * * * D D D D D D D D 6 6 6 6 6 6 6 6W W W # # # #( ( ( (.> > >D%> %> %>P3{ 3{ 3{l&! &! &! &!T BG7! 7! 7! 7!t@ @ @2@" @" @"F  
 zDFFFFF r"   