gddlZddlZddlZddlZddlmZddlmZddlZ ddl Z ddl m Z m Z ddlmZddlmZmZddlmZddlmZmZdd lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%e#rdd lm&Z'ndd lm'Z'd%d Z(dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4d%dZ5d%dZ6dZ7dZ8dZ9dZ:d Z;d!Ze?d$kr e>dSdS)&N)deepcopy)Path) DataLoaderDataset) Accelerator)SeedableRandomSamplerprepare_data_loader)AcceleratorState)RegressionDatasetare_the_same_tensors) DataLoaderConfigurationDistributedTypegatheris_bf16_availableis_datasets_availableis_ipex_availableis_mlu_availableis_musa_availableis_npu_availableis_pytest_availableis_xpu_availableset_seedsynchronize_rng_states)RegressionModel4XPU)RegressionModelFc|r1t||t|}t|||St||d|S)zBCreates a dataloader that can also use the `SeedableRandomSampler`) generator data_source num_samples) batch_sizesamplerT)r shuffler)rlenr) train_setrr use_seedable_samplerr!s e/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/test_utils/scripts/test_script.pygenerate_baseline_dataloaderr'8s_ _(!I    ) GLLLL) DT]^^^^c4td|jdS)NzPrinting from the main process print process_indexstates r& print_mainr/G" AE,? A ABBBBBr(c4td|jdS)Nz%Printing from the local main process )r+local_process_indexr-s r&print_local_mainr3Ks" M%2K M MNNNNNr(c4td|jdS)NPrinting from the last process r*r-s r& print_lastr6Or0r(c:td|d|jdS)NPrinting from process : r*)r. process_idxs r&print_onr;Ss* G; G G%2E G GHHHHHr(cD t}|j}td}|5|jrRt jdt|d5}|ddddn #1swxYwYn=t|d5}|ddddn #1swxYwYdddn #1swxYwY| |jrt|5}d | }dddn #1swxYwY | ds Jd|dkr| ds Jd|d|jdz ks+Jd |d dzd |jn##t$r|wxYw|jr(|r|| t%j}t)j|5|t.|jdddn #1swxYwY|}|jr|d ks J|d n7|dks J|d|d|dt)j|5|t<|jdddn #1swxYwY|jr-|dksJn,|dksJ|d|dt)j|5| tB|jdddn #1swxYwY|j"r=|d|jjdz ksJn,|dksJ|d|dtG|D]}t)j|5|$tJ||j|dddn #1swxYwY|j&|kr8|d|d|j&ksJn,|dksJ|d|ddS)Nzcheck_main_process_first.txt皙?za+zCurrently in the main process zNow on another process zMain process was not firstzOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""rz&Printing from the local main process 0r5)r,r8r9)'r num_processesrmain_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin readlines startswithendswithcountAssertionErrorunlinkexistsioStringIO contextlibredirect_stdouton_main_processr/r.getvaluerstriptruncateseekon_local_main_processr3is_local_main_processon_last_processr6is_last_processrange on_processr;r,) acceleratorr@pathftextresultr:s r&process_execution_checkrdWs--K-M . / /D  ' ' ) )44  & 4 JsOOOdD!! ;Q9::: ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;dD!! 4Q2333 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 444444444444444!!###"  $ZZ *1771;;==))D * * * * * * * * * * * * * * * ??#DEE c cGc c c cq  }}%?@@^^B^^^^ 566+:SVW:WWWWvTZZ0H%I%IA%Mvv[f[tvvXWWW    KKMMM  "t{{}} !!### A  #A & &CC/ ##J// 0ABBBCCCCCCCCCCCCCCC ZZ\\ " "F">;;;;=l=l=l;;;;zz||""$$***v,=,=,=***JJqMMMFF1III  #A & &OO; ))*:;;K$>??mmAmmmmm  ?#< < <w'''#EI$;$;$=$=>>ll@llll!!IK=I>>>>  3 3 5 5 6 6jj8jjjj  A%% -.....&%r(cxt}d|jz}tt|d}t ||j|j|jd}g}|D]$}|t|%tj |}t|j|t|tj |tjd|s Jdtt|d}t ||j|j|jdd}g}|D]$}|t|%tj |}tj |tjd|s Jd|jdkrtd tt|dd }t ||j|j|jd}g}|D]$}|t|%tj |}||t't|ks Jd tt|dd }t ||j|j|jdd}g}|D]$}|t|%tj |}||t't|ks Jd |jdkrtd dSdS) N r T) put_on_devicer%Wrong non-shuffled dataloader result.)rv split_batchesz Non-shuffled dataloader passing.r r"!Wrong shuffled dataloader result.zShuffled dataloader passing.)r r@rr]r devicer,appendrrhcatr+typeequalcpuarangelongtolistsortlistr2r.lengthdlrcbatchs r&dl_preparation_checkrs   E %% %F E&MMa 0 0 0B Ru/BEDWgk l l lB F%% fUmm$$$$ Yv  F % vtBxx000 ;vzz||U\!V%<%<%A%A%C%C D DmmFmmmm E&MMa 0 0 0B         BF%% fUmm$$$$ Yv  F ;vzz||U\!V%<%<%A%A%C%C D DmmFmmmm a 0111 E&MMa > > >B Ru/BEDWgk l l lB F%% fUmm$$$$ Yv   % % ' 'F KKMMM T%--(( ( ( (*M ( ( ( E&MMa > > >B         BF%% fUmm$$$$ Yv   % % ' 'F KKMMM T%--(( ( ( (*M ( ( (  A%% ,-----&%r(c :t}d|jz}tt|d}t ||j|j|jdd}g}|D]$}|t|%tj |}tj | tj d|s Jdtt|d}t ||j|j|jddd}g}|D]$}|t|%tj |}tj | tj d|s Jd|jdkrtd tt|dd }t ||j|j|jdd}g}|D]$}|t|%tj |}||t%t|ks Jd tt|dd }t ||j|j|jddd}g}|D]$}|t|%tj |}||t%t|ks Jd |jdkrtd dSdS) NrsrtruT)rvdispatch_batchesrrw)rvrxrz(Non-shuffled central dataloader passing.ryrzz$Shuffled central dataloader passing.)r r@rr]r r{r,r|rrhr}rrrrr+rrrr2rs r&central_dl_preparation_checkrs   E %% %F E&MMa 0 0 0B  EL%-u/BRVim   BF%% fUmm$$$$ Yv  F ;vzz||U\!V%<%<%A%A%C%C D DmmFmmmm E&MMa 0 0 0B        BF%% fUmm$$$$ Yv  F ;vzz||U\!V%<%<%A%A%C%C D DmmFmmmm a 8999 E&MMa > > >B  EL%-u/BRVim   BF%% fUmm$$$$ Yv   % % ' 'F KKMMM T%--(( ( ( (*M ( ( ( E&MMa > > >B        BF%% fUmm$$$$ Yv   % % ' 'F KKMMM T%--(( ( ( (*M ( ( (  A%% 455555&%r(ct}Gddt}Gdd}|td|jz}|t |d}t ||}t ||j|j|j}t|j d r&t|j j |s Jd dSt|j |s Jd dS) Nc eZdZdZdZdZdS)+custom_sampler_check..CustomDatasetc||_dSNdata)selfrs r&__init__z4custom_sampler_check..CustomDataset.__init__<s DIIIr(c*t|jSr)r#rrs r&__len__z3custom_sampler_check..CustomDataset.__len__?sty>> !r(c|j|Srr)rindexs r& __getitem__z7custom_sampler_check..CustomDataset.__getitem__Bs9U# #r(N)__name__ __module__ __qualname__rrrr(r& CustomDatasetr;sA    " " " $ $ $ $ $r(rc0eZdZd dededefdZdZdZdS) 0custom_sampler_check..CustomBatchSamplerTdataset_lengthr r"cT||_tj||_||_dSr)r npr data_indexr")rrr r"s r&rz9custom_sampler_check..CustomBatchSampler.__init__Fs%(DO i77DO"DLLLr(c3Kt|}|jr%tj|j}n|j}tj||}|Ed{VdSr)r#r"rrandom permutationr array_split)r num_batchesroutputs r&__iter__z9custom_sampler_check..CustomBatchSampler.__iter__Ksjd))K| ( --do>>^E;77F         r(c^tjt|j|jz Sr)mathceilr#rr rs r&rz8custom_sampler_check..CustomBatchSampler.__len__Ts#9S11DOCDD Dr(N)T)rrrintboolrrrrr(r&CustomBatchSamplerrEse # #3 #C #$ # # # #     E E E E Er(rrsrtru) batch_samplerrz>Custom sampler was changed after calling `prepare_data_loader`) r rr]r@r#rr r{r,hasattrr isinstance)r.rrdatasetr!rs r&custom_sampler_checkr8sq   E$$$$$$$$EEEEEEEE$mE"u':":;;<   L L K L L L L L  0   L L K L L L L Lr(cztdtdd}t|dd}td}t |}||}g}t d D]"}|D]}||d #tj |}td| d g}t d D]"}|D]}||d #tj |}tj ||s Jd dS) N* rseedTryr%dataloader_configxrz;Did not obtain the same items with the same seed and epoch.) rr rr rpreparer]r|rhr} set_epochallclose)r$train_dlconfigr_original_items_r new_itemss r&check_seedable_samplerrfs^ RLLL!"555I)4@@@H $$ ? ? ?F777K""8,,HN 1XX.. . .E  ! !%* - - - - .Y~..N RLLL qI 1XX)) ) )E   U3Z ( ( ( ( ) )$$I >.) 4 4ss6ssssssr(chtdtd}t|}|jdks Jdt t t ddd}t|d }|jjj }t|ts Jd dS) NrTrrr?z)This test requires more than one process.rry) dataloaderr%z:Sampler in BatchSamplerShard is not SeedableRandomSampler.) rr rr@rrr]r rr!rr)rr_rprepared_data_loadertarget_samplers r&-check_seedable_sampler_in_batch_sampler_shardrs RLLL $$ ? ? ?F777K  $q ( ( (*U ( ( (DrOO4HHHJ.! *7EMN -  DDCDDDDDr(c8td|dt|d}t||||}t }t j|d}tdD]}|D]z} | || d} t j j | | d} | |{||fS)Nrrr=lrrry)r manual_seedr r'rrhoptimSGD parametersr] zero_gradnn functionalmse_lossbackwardstep) rr rr%r$rmodel optimizerepochrrlosss r& mock_trainingrs  RLLL "!b999I+Iy*NbccH   E  0 0 2 2s;;Iq  E OO   U5:&&F8&//c CCD MMOOO NN       e r(ct}tj}d}|dz|jz}t |||jz||\}}t |js Jdt |js Jdt}t||||}t} tj | d} ||| | \}} } td|dt#dD]} |D]{} | | | d} tjj| | d }||| ||| } tj|j| js Jd tj|j| js Jd |d t9d | }t|}t||||jz|}t} tj | d} ||| | \}} } td|dt#dD]} |D]{} | | | d} tjj| | d }||| ||| } tj|j| js Jd tj|j| js Jd |dtjs+t?stAstCrt7dtj"t9|}td|}t||||}t} tj | d} ||| | \}} } td|dt#dD]} |D]{} | | | d} tjj| | d }||| ||| } tj|j| js Jd tj|j| js Jd tjrt7dtj"td}tj#dd} || } || d }tj$ddg%tj&|j'}||} tQrt7dtj"t9|}td|}t||||}t} tj | d} ||| | \}} } td|dt#dD]} |D]{} | | | d} tjj| | d }||| ||| } tj|j| js Jd tj|j| js Jd tSrt7dtj"t9|}tdd |}t||||}t} tj | d} ||| | \}} } td|dt#dD]} |D]{} | | | d} tjj| | d }||| ||| } tj|j| js Jd tj|j| js Jd tUrt7dtj"t9|}tdd|}t||||}t} tj | d} ||| | \}} } td|dt#dD]} |D]{} | | | d} tjj| | d }||| ||| } tj|j| js Jd tj|j| js Jd dSdS)!Nrtz0Did not obtain the same model on both processes.r=rrrrrz=Did not obtain the same model on CPU or distributed training.zVTraining yielded the same results on one CPU or distributed setup with no batch split.T)rxr%rzSTraining yielded the same results on one CPU or distributes setup with batch split.zFP16 training check.rfp16)mixed_precisionrzKeep fp32 wrapper check.)rr)keep_fp32_wrapperr?)dtyper{zBF16 training check.bf16zipex BF16 training check.)rrrzxpu BF16 training check.Fz=Did not obtain the same model on XPU or distributed training.)+r rhror@rr abrr'rrrrrrrr]rrrrrr unwrap_modelrrr+r ri is_availablerrr _reset_stateLinearTensortofloat16r{rrr)r%r.rr rr$ old_modelr_rrrrrrrrmodel_with_fp32_wrapper input_tensors r&training_checkrs8   E!!IJ !^e1 1F(e>Q1QS\^rssIy  , ,``.````  , ,``.````--K+Iy*NbccH   E  0 0 2 2s;;I!,!4!4Xui!P!PHeY RLLL " 1XX  E OO   U5:&&F8&//c CCD   & & & NN        $ $U + + / / 1 1E >)+uw / /pp1pppp >)+uw / /pp1ppppnooo/dYmnnn0ABBBK+9j5+>>@TH   E  0 0 2 2s;;I!,!4!4Xui!P!PHeY RLLL " 1XX  E OO   U5:&&F8&//c CCD   & & & NN        $ $U + + / / 1 1E >)+uw / /pp1pppp >)+uw / /pp1ppppklll z  u$4$6$6u:J:L:LuPaPcPcu $%%%%'''3I]^^^!&L]^^^ / 9jRfgg!!KOOE$4$4$6$63O?? %0%8%85)%T%T"% b!!!q ! !A! ! !!!!uSz**x*33FE#JGG$$T***     !((//3355~ik5733tt5tttt~ik5733tt5tttt z  7 ()))%'''!&999 1%%##E**"-":":5TX":"Y"Y|QF++..U];K].^^ ((66u $%%%%'''3I]^^^!&L]^^^ / 9jRfgg!!KOOE$4$4$6$63O?? %0%8%85)%T%T"% b!!!q ! !A! ! !!!!uSz**x*33FE#JGG$$T***     !((//3355~ik5733tt5tttt~ik5733tt5ttttu )***%'''3I]^^^!&dVghhh / 9jRfgg!!KOOE$4$4$6$63O?? %0%8%85)%T%T"% b!!!q ! !A! ! !!!!uSz**x*33FE#JGG$$T***     !((//3355~ik5733tt5tttt~ik5733tt5ttttu ()))%'''3I]^^^!&eWhiii / 9jRfgg!!KOOE$4$4$6$63O?? %0%8%85)%T%T"% b!!!q ! !A! ! !!!!uSz**x*33FE#JGG$$T***     !((//3355~ik5733tt5tttt~ik5733tt5tttt-uu, utr(ct}|dtd|jzD}||d5}t |dks"Jd|jdt | dddn #1swxYwY|dtd|jzdz D}||d5}|jr6t |dks"Jd |jdt |n5t |dks"Jd |jdt |dddn #1swxYwY|d td|jzdz D}||d 5}|jdkr6t |dks"Jd |jdt |n5t |dks"Jd|jdt |dddn #1swxYwY|dS)Nc.g|]}t|S)kdict.0vs r& z8test_split_between_processes_dataset..Us &Y&Y&YQtayyy&Y&Y&Yr(rF apply_padding4Each process did not have two items. Process index: ; Length: c.g|]}t|Srrrs r&rz8test_split_between_processes_dataset..[ &]&]&]Qtayyy&]&]&]r(r?z;Last process did not receive a single item. Process index: zLOne of the intermediate processes did not receive two items. Process index: c.g|]}t|Srrrs r&rz8test_split_between_processes_dataset..fr r(Tz=Single process did not receive a single item. Process index: ) r from_listr]r@split_between_processesr#r,r\rG)datasets_Datasetr.rresultss r&$test_split_between_processes_datasetrSs   E  % %&Y&Y%EDW@W:X:X&Y&Y&Y Z ZD  & &t5 & A ApW LLA    o%BU o oadelamam o o    ppppppppppppppp  % %&]&]%EDW@WZ[@[:\:\&]&]&] ^ ^D  & &t5 & A ALW   LG !!!zUM`zzlopwlxlxzz"!!!G !!!L^c^qLL~ABI~J~JLL"!!LLLLLLLLLLLLLLL  % %&]&]%EDW@WZ[@[:\:\&]&]&] ^ ^D  & &t4 & @ @tG  ! # #G !!!|uOb||nqrynznz||"!!!G !!!seFYssehipeqeqss"!!ttttttttttttttt s86BB!$B!6A3E55E9<E9A7IIIct}ttdd|jz}||5}t |dks"Jd|jdt | dddn #1swxYwYttdd|jzdz }||d5}|jr^tj t ||jz }t ||ks"Jd |jdt |dddn #1swxYwY| dS) Nrrrrrr?Trz;Last process did not get the extra item(s). Process index: ) r rr]r@r r#r,r\rrrG)r.rrnum_samples_per_devices r&!test_split_between_processes_listrts    E aU0011 2 2D  & &t , ,p LLA    o%BU o oadelamam o o    ppppppppppppppp a!e11Q677 8 8D  & &t4 & @ @{G   {%)Ys4yy5;N/N%O%O "G 6666zUM`zzlopwlxlxzz766 {{{{{{{{{{{{{{{ s% 6B  BBA&EE E ct}gd}gd}tjgd}|jdvrz|||d}t |}||5}|jdkr%|d|ddd|jzksJno|jdkr|d|dd dksJnG|jd kr<|d|dd dks"Jd |dd d |d|jdkr%|d|ddd|jzksJnO|jdkr|d|dd dksJn'|jd kr|d|dd dksJ|jdkr^tj|d|ddd|jzs,Jd|ddd|jzd|dn|jdkrNtj|d|dd ds$Jd|dd dd|dnX|jd krMtj|d|dd ds$Jd|dd dd|ddddn #1swxYwY|dS)N)r?rrrrt)rrcderagh)r?rr)rrrrrrtrrrz Expected: , Actual: rrz7Did not obtain expected values on process 0, expected `z `, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `) r rhtensorr@rr r,rrG)r.rrrr data_copyrs r&(test_split_between_processes_nested_dictr s    E   A000A ---..A i''QQ''TNN  * *4 0 0 yG"a''s|y~6PU=P8P6P'QQQQQQ$))s|y~abb'999999$))s|y~bcc'::::#bhmTUFVBWBW W W W W W Ws3BBBc6t}|j}|jdkrtdt ||jtjkrtj }n|j }||j kr|j dkrtdt|j dkrtdt|j dkrtdt!|j dkrtdt#|j dkrtdt%|j dkrtdt'rdd lm}t-|ntd |jdkrtd t/|jdkrtd t1|jtjkr*t5t7t9|j d krt;|jtjkrdS|jdkrtdt?dt?d|jdkrtdtAtCr(|jdkrtdtE|#dS)Nrz**Initialization**z **Test process execution**z+ **Test split between processes as a list**z+ **Test split between processes as a dict**z- **Test split between processes as a tensor**z( **Test split between processes evenly**z7 **Test split between processes as a datasets.Dataset**)rz6Skipped because Hugging Face datasets is not availablez1 **Test random number generator synchronization**z **DataLoader integration test**r?z **Training integration test**FrTz **Breakpoint trigger test**z **Test reinstantiated state**)$rr.r2r+rfrGrlrrmrhri device_countr@r,rdrr r"r'rdatasetsrrrqrXLArrrr DEEPSPEEDrr+rr5destroy_process_group)r_r.num_processes_per_noder s r&mainr=s(--K  E  A%% "###  !:::!&!8!8!:!:!&!4!444  ! # # 0 1 1 1!!!  ! # # @ A A A)+++  ! # # @ A A A0222  ! # # B C C C+---  ! # # = > > >+---  ! # # L M M M " " L < < < < < < 01A B B B B J K K K  A%% BCCC  A%% 1222 !444$&&&    Q5777 !:::  A%% /000....----  A%% -...NNN$  $ ) ) 3 4 4 4!### !!!!!r(__main__)F)@rRrPrrCcopyrpathlibrnumpyrrhtorch.utils.datarr accelerateraccelerate.data_loaderrr accelerate.stater accelerate.test_utilsr r accelerate.utilsr rrrrrrrrrrrrrrr'r/r3r6r;rdrfrqrrrrrrrrrr r"r'r+r5r=rrr(r&rHsa"  00000000""""""MMMMMMMM------IIIIIIII$6LLLLLLL555555 _ _ _ _CCCOOOCCCIIIFFFR///$:.:.:.z>6>6>6B+L+L+L\ttt4DDD&$kukukuku\B&###L   $000" X X X M"M"M"` zDFFFFFr(