g[ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZmZddlmZddlZddlZddlmZmZdd lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4d Z5e5\Z6Z7Z8d e9fd Z:e:e7d Z;dTdZ<edZ?dZ@dZAdZBdZCdZDdZEdZFdZGdZHdZIdZJd ZKd!ZLd"ZMd#ZNd$ZOd%ZPd&ZQd'ZRd(ZSd)ZTd*ZUd+ZVd,ZWd-ZXd.ZYdUd/ZZd0Z[d1Z\d2Z]d3Z^d4Z_d5Z`d6Zad7Zbd8Zcede2e)go e Zed9Zfd:ZgGd;dejhZjGd?d@ejhZkdAZlGdBdCZmdDZndVd emfdEZodWdHe9d emfdIZpGdJdKeqZrdXdLeesfdMZtdNesd efdOZue dYdPeqdQesd evfdRZwdSZxdS)ZN)contextmanager)partial)Path)ListUnion)mock)AcceleratorState PartialState)gatheris_bnb_availableis_clearml_availableis_comet_ml_availableis_cuda_availableis_datasets_availableis_deepspeed_availableis_dvclive_availableis_import_timer_availableis_mlu_availableis_mps_availableis_musa_availableis_npu_availableis_pandas_availableis_pippy_availableis_schedulefree_availableis_tensorboard_availableis_timm_availableis_torch_versionis_torch_xla_available*is_torchdata_stateful_dataloader_availableis_torchvision_availableis_transformer_engine_availableis_transformers_availableis_triton_availableis_wandb_availableis_xpu_available str_to_boolcftr0dtjtjjfSt r0dtjtjjfSt drddtjjfSt rdddfStr0dtj tj jfStr0d tj tj jfStr0d tjtjjfStr0d tjtjjfSd dd fS)Nxlacudaz2.0) min_versionmpscdSNrr0Y/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/test_utils/testing.pyzget_backend..Kr1mlumusanpuxpucpucdSr/r0r0r1r2r3zget_backend..Ur4r1)rtorchr* device_countmemory_allocatedrrr,current_allocated_memoryrr5rr6rr7r&r8r0r1r2 get_backendr?Csg#ej--//1LLL   #uz..00%*2MMM e , , , #a;;;    #a""    #ei,,.. 0JJJ   #uz..00%*2MMM   #ei,,.. 0JJJ   #ei,,.. 0JJJa""r1returnc ddg}|D]R\}}t|tr|r|d|5||d|d|S|S)z Wraps around `kwargs` to help simplify launching from `subprocess`. Example: ```python # returns ['accelerate', 'launch', '--num_processes=2', '--device_count=2'] get_launch_command(num_processes=2, device_count=2) ``` acceleratelaunchz--N=)items isinstanceboolappend)kwargscommandkvs r2get_launch_commandrM[sX&G ))1 a   )1 ) NN888 $ $ $ $ ] NN<<S T TU^ _ __r1cPtjtdkd|S)z Decorator marking a test that requires a multi-device setup. These tests are skipped on a machine without multiple devices. r-z,test requires multiple hardware accelerators)r\rar<r^s r2require_multi_devicer0s) a8 |a/1_ ` `aj k kkr1c~tjtjdkd|S)z Decorator marking a test that requires a multi-GPU setup. These tests are skipped on a machine without multiple GPUs. r-ztest requires multiple GPUsrr^s r2require_multi_gpur8s7 ]8 uz66881<>[ \ \]f g ggr1c~tjtjdkd|S)z Decorator marking a test that requires a multi-XPU setup. These tests are skipped on a machine without multiple XPUs. r-ztest requires multiple XPUsrr^s r2require_multi_xpur@s7 \8 uy5577!;=Z [ [\e f ffr1cVtjtd|S)z| Decorator marking a test that requires DeepSpeed installed. These tests are skipped when DeepSpeed isn't installed ztest requires DeepSpeed)r\rarr^s r2require_deepspeedrHs+ T8 5779R S ST] ^ ^^r1cZtjtddd|S)zr Decorator marking a test that requires FSDP installed. These tests are skipped when FSDP isn't installed >=z1.12.0z%test requires torch version >= 1.12.0)r\rarr^s r2 require_fsdprOs/ j8 /h??Ah i ijs t ttr1c|tt|Stjt d|d||S)z Decorator marking that a test requires a particular torch version to be tested. These tests are skipped when an installed torch version is less than the required one. N)versionrztest requires torch version >= )rrequire_torch_min_versionr\rar)r_rs r2rrVsQ 0'BBBB l8 /g>>@kbi@k@k l lmv w wwr1cVtjtd|S)z Decorator marking a test that requires tensorboard installed. These tests are skipped when tensorboard isn't installed ztest requires Tensorboard)r\rarr^s r2require_tensorboardr`s+ X8 799;V W WXa b bbr1cVtjtd|S)zt Decorator marking a test that requires wandb installed. These tests are skipped when wandb isn't installed ztest requires wandb)r\rar%r^s r2 require_wandbrh* L8 1335J K KI V VVr1cVtjtd|S)zz Decorator marking a test that requires comet_ml installed. These tests are skipped when comet_ml isn't installed ztest requires comet_ml)r\rarr^s r2require_comet_mlros+ R8 4668P Q QR[ \ \\r1cVtjtd|S)zx Decorator marking a test that requires clearml installed. These tests are skipped when clearml isn't installed ztest requires clearml)r\rarr^s r2require_clearmlrv+ P8 3557N O OPY Z ZZr1cVtjtd|S)zx Decorator marking a test that requires dvclive installed. These tests are skipped when dvclive isn't installed ztest requires dvclive)r\rarr^s r2require_dvcliver}rr1cVtjtd|S)zv Decorator marking a test that requires pandas installed. These tests are skipped when pandas isn't installed ztest requires pandas)r\rarr^s r2require_pandasrs* N8 2446L M Mi X XXr1cVtjtd|S)zt Decorator marking a test that requires pippy installed. These tests are skipped when pippy isn't installed ztest requires pippy)r\rarr^s r2 require_pippyrrr1cVtjtd|S)z Decorator marking a test that requires tuna interpreter installed. These tests are skipped when tuna isn't installed ztest requires tuna interpreter)r\rarr^s r2require_import_timerrs+ ^8 8::<\ ] ]^g h hhr1cVtjtd|S)z Decorator marking a test that requires transformers engine installed. These tests are skipped when transformers engine isn't installed z!test requires transformers engine)r\rar"r^s r2require_transformer_enginers+ g8 >@@Be f fgp q qqr1cHtjtd|S)z Decorator marking that a test requires at least one tracking library installed. These tests are skipped when none are installed zYtest requires at least one tracker to be available and for `comet_ml` to not be installed)r\ra_atleast_one_tracker_availabler^s r2require_trackersrs0  8 &c    r1cVtjtd|S)z Decorator marking a test that requires torchdata.stateful_dataloader. These tests are skipped when torchdata with stateful_dataloader module isn't installed. z+test requires torchdata.stateful_dataloader)r\rar r^s r2%require_torchdata_stateful_dataloaderrs4 8 2446c    r1cHeZdZdZdZedZedZdZdS)TempDirTestCasea A TestCase class that keeps a single `tempfile.TemporaryDirectory` open for the duration of the class, wipes its data at the start of a test, and then destroyes it at the end of the TestCase. Useful for when a class or API requires a single constant folder throughout it's use, such as Weights and Biases The temporary directory location will be stored in `self.tmpdir` TcPttj|_dS)zECreates a `tempfile.TemporaryDirectory` and stores it in `cls.tmpdir`N)rtempfilemkdtemptmpdirclss r2 setUpClasszTempDirTestCase.setUpClasss (*,,-- r1ctj|jrt j|jdSdS)z1Remove `cls.tmpdir` after test suite has finishedN)rQpathexistsrshutilrmtreers r2 tearDownClasszTempDirTestCase.tearDownClasss> 7>>#* % % & M#* % % % % % & &r1c|jrn|jdD]U}|r|+|rt j|TdSdS)z>(KKMMMM[[]](M$'''  ( ( ( (r1N) __name__ __module__ __qualname____doc__r classmethodrrrr0r1r2rrshN..[.&&[& (((((r1rc"eZdZdZfdZxZS)AccelerateTestCasez A TestCase class that will reset the accelerator state at the end of every test. Every test that checks or utilizes the `AcceleratorState` class should inherit from this to avoid silent failures due to state being shared between tests. cttjt jdSN)supertearDownr _reset_stater )r __class__s r2rzAccelerateTestCase.tearDowns= %'''!#####r1)rrrrr __classcell__)rs@r2rrsB $$$$$$$$$r1rcNeZdZdZdeejeejffdZdS)MockingTestCasea A TestCase class designed to dynamically add various mockers that should be used in every test, mimicking the behavior of a class-wide mock when defining one normally will not do. Useful when a mock requires specific information available only initialized after `TestCase.setUpClass`, such as setting an environment variable with that information. The `add_mocks` function should be ran at the end of a `TestCase`'s `setUp` function, after a call to `super().setUp()` such as: ```python def setUp(self): super().setUp() mocks = mock.patch.dict(os.environ, {"SOME_ENV_VAR", "SOME_VALUE"}) self.add_mocks(mocks) ``` mocksct|ttfr|n|g|_|jD]0}|||j1dS)aQ Add custom mocks for tests that should be repeated on each test. Should be called during `MockingTestCase.setUp`, after `super().setUp()`. Args: mocks (`mock.Mock` or list of `mock.Mock`): Mocks that should be added to the `TestCase` after `TestCase.setUpClass` has been run N)rFtuplelistrstart addCleanupstop)rrms r2 add_mockszMockingTestCase.add_mockssc) >>KUUUG  $ $A GGIII OOAF # # # # $ $r1N) rrrrrrMockrrr0r1r2rrsK" $uTYTY%?@ $ $ $ $ $ $r1rcrt}|d|j}t |}|d}t |jdD] }tj |||sdS!dS)NrFT) r clonetodevicer r9rangeshaper;equal)tensorstatetensorsis r2are_the_same_tensorsrs   E D\   ! ! $ $U\ 2 2FVnn  ""G AY]]__F 7=# $ ${71:v.. 55  4r1ceZdZdZdS) _RunOutputc0||_||_||_dSr) returncodestdoutstderr)rrrrs r2__init__z_RunOutput.__init__s$  r1N)rrrrr0r1r2rrs#r1rc^K |d{V}|r ||ndS+r)readline)streamcallbacklines r2 _read_streamr sM__&&&&&&&&   HTNNNN E r1c 2 K|r#tdd|tj|dg|ddR|tjjtjj|dd{V}ggd fd tjtjt|j  fdtjt|j  fd g| d{Vt|d{VS) Nz Running:  rr-)stdinrrenvc|d}||st|||dSdS)Nutf-8)file)decoderstriprHprint)rsinkpipelabelquiets r2teez_stream_subprocess..teeAs_{{7##**,, D * %D ) ) ) ) ) ) * *r1c6|tjdS)Nzstdout:r)sysr)loutrs r2r3z$_stream_subprocess..JQSZ_hAiAiAir1c6|tjdS)Nzstderr:r)rr)rerrrs r2r3z$_stream_subprocess..Krr1)timeout)r ) rjoinasynciocreate_subprocess_exec subprocessPIPEwait create_taskrrrr) cmdr r r!rechopr rrs ` @@@r2_stream_subprocessr,)s , mSXXc]]+++, A QRR!&!&        A" C C****** ,   QX7i7i7i7i7i j j k k   QX7i7i7i7i7i j j k k      AFFHHnnnnnnc3 / //r1Tr)c t|D],\}}t|trt|||<-t j}|t||||||} d|} | j dkr7d| j } td| d| j d| | S)N)r r r!rr*r r 'z' failed with returncode z, The combined stderr from workers follows: ) enumeraterFrstrr#get_event_looprun_until_completer,r"rr RuntimeError) r)r r r!rr*rcloopresultcmd_strrs r2execute_subprocess_asyncr:Rs#1 a   VVCF  ! # #D  $ $3CugUY]^^^FhhsmmG 16=)) C C C&2C C C:@ C C   Mr1ceZdZdS)SubprocessCallExceptionN)rrrr0r1r2r<r<gsDr1r<rJct|D],\}}t|trt|||<-|tj} tj|tj |}|r't|dr| d}|SdS#tj $rE}tdd|d|j |d}~wwxYw)z Runs `command` with `subprocess.check_output` and will potentially return the `stdout`. Will also properly capture if an error occured while running `command` N)rr rrz Command `r z$` failed with the following error: )r1rFrr2rQrRcopyr% check_outputSTDOUThasattrrCalledProcessErrorr<r"output)rJ return_stdoutr rr6rCes r2 run_commandrFks '""  1 a   QGAJ {joo (9JPSTTT  vx(( 0w//M    (% d)) d dQRQYQ`Q`QbQb d d   sA B**C>9AC99C> componentsclttjtj}|j|S)z Get a path within the `accelerate` package's directory. Args: *components: Components of the path to join after the package directory. Returns: `Path`: The path to the requested file or directory. )rinspectgetfilerBparentjoinpath)rGaccelerate_package_dirs r2path_in_accelerate_packagerNs0"'/*"="=>>E * ! *J 77r1exception_classmsgc#2Kd} dVd}nu#t$rh}t||sJd|dt||/|t|vsJd|dt|dYd}~nd}~wwxYw|rt d|d dS) z Context manager to assert that the right `Exception` class was raised. If `msg` is provided, will check that the message is contained in the raised exception. FNTzExpected exception of type z but got zExpected message 'z' to be in exception but got 'r0z but ran without issue.) ExceptionrFtyper2AssertionError)rOrPwas_ranrEs r2assert_exceptionrVsGd  ddd!_--pp/p_/p/pgklmgngn/p/pppp ?#a&&==="cs"c"cZ]^_Z`Z`"c"c"c===dec?cccdddees A?AA::A?ctj}tj} |t_||i|n#t$r}|d}~wwxYw |t_n#|t_wxYw|S)z` Takes in a `func` with `args` and `kwargs` and returns the captured stdout as a string N)ioStringIOrrrRgetvalue)funcargsrIcaptured_outputoriginal_stdoutrEs r2capture_call_outputr_skmmOjO%$  df  % _ $$$$  # # % %%s&6A AAAAA&)F)NN)NNNFF)NNr-FT)FNr)yr#rIrXrQrr%rrr\ contextlibr functoolsrpathlibrtypingrrrr;rBrr r utilsr r rrrrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r?rfr<memory_allocated_funcrrMDEFAULT_LAUNCH_COMMANDrYrbr]rcrgrjrlrorqrsrvrxrzr|r~rrrrrrrrrrrrrrrrrrrrrrrrranyrrrTestCaserrrrrrr,r:rRr<r2rFrNrGrVr_r0r1r2ris  %%%%%% 22222222@###*5@KMM1 l1D&,+,Y\]]]     &%j%@@@888 KKK]]]XXXwwwUUU\\\TTTVVVTTTrrrqqqaaaoooeeeqqqhhhjjjqqqaaa```lllhhhggg___uuuxxxxcccWWW]]][[[[[[YYYWWWiiirrrC    7799:;;[DYDYD[D[@[    (((((h'(((B $ $ $ $ $* $ $ $$$$$$h'$$$B&0&0bl&0&0&0&0R$fp*     i   c. 8C 8D 8 8 8 8eeieceTeeee$ & & & & &r1