gddlmZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z ddlZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(e"r ddl)m*cm+Z,ed rddl-Z-e d rddl.Z.e!d rddl/Z/ej0e1Z2dd Z3dZ4Gddej5Z6e"se7ne6Z8GddZ9GddZ:GddZ;dS)) annotationsN)contextmanager)partial)AnyCallableOptional)DistributedType DynamoBackendGradientAccumulationPlugincheck_cuda_p2p_ib_supportcheck_fp8_capabilitydeepspeed_requiredget_ccl_versionget_cpu_distributed_informationget_int_from_envis_ccl_availableis_datasets_availableis_deepspeed_availableis_fp8_availableis_ipex_availableis_mlu_availableis_mps_availableis_musa_availableis_npu_availableis_torch_xla_availableis_xpu_availableparse_choice_from_envparse_flag_from_envset_numa_affinity)SageMakerDistributedTypeF) check_devicereturnboolc"tjikS)z Checks if the `AcceleratorState` has been initialized from `Accelerator`. Same as `AcceleratorState.initialized`, but works as a module method. )AcceleratorState _shared_stateL/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/state.pyis_initializedr+Fs  )R //r)cdSNr()argskwargss r* do_nothingr0Os 4r)c*eZdZdZd d dZd dZdZdS) ThreadLocalSharedDicta Descriptor that holds a dict shared between instances of a class in the same thread. Note: Descriptors have slightly different semantics than just a dict field on its own. `PartialState(...)._shared_state` and `PartialState._shared_state` (instance vs class) give the same value: the underlying _storage dict. Likewise, `PartialState(...)._shared_state = {...}` overrides the _storage dict inside the descriptor as you would expect. However, `PartialState._shared_state = {}` actually replaces the descriptor object with a dict instead Thus, you should modify the _storage dict in-place (e.g. `_shared_state.clear()`). See Python documentation for an explanation of descriptors: https://docs.python.org/3/howto/descriptor.html This is required for using PyTorch/XLA with PJRT in multithreaded mode (required for TPU v2 and v3). See https://github.com/pytorch/xla/blob/r2.0/docs/pjrt.md#multithreading-on-tpu-v2v3 F thread_localr$ci|_dSr-_storage)selfr3s r*__init__zThreadLocalSharedDict.__init__ds  r)Nc|jSr-r5)r7objobjtypes r*__get__zThreadLocalSharedDict.__get__gs }r)c||_dSr-r5)r7r:values r*__set__zThreadLocalSharedDict.__set__js  r)F)r3r$r-)__name__ __module__ __qualname____doc__r8r<r?r(r)r*r2r2SsZ r)r2ceZdZdZeZgdZd.d/dZd0d Ze d Z e d1d Z e d Z e d1d Ze d1dZe d1dZdZd2dZed.d3dZedZedZd4d5dZd4d5dZd5dZd6d7d!Zd6d8d#Zd$Ze d9d&Z d:d;d)Zd*Zd4d+Z d install it using `pip3 install deepspeed` or build it from sourcer)comm) dist_backendauto_mpi_discoveryTRANK WORLD_SIZELOCAL_WORLD_SIZE MASTER_PORT29500 MASTER_ADDRmpizTried to launch on distributed with multinode, but `MASTER_ADDR` env was not set, please try exporting rank 0's hostname as `MASTER_ADDR`rank world_sizeOMP_NUM_THREADS)logicalr z4OMP_NUM_THREADS/MKL_NUM_THREADS unset, we set it at z to improve oob performance. check_is_tpuACCELERATE_CPU_AFFINITYcudaNCCL_P2P_DISABLENCCL_IB_DISABLEzUsing RTX 4000 series doesn't support faster communication broadband via P2P or IB. Please set `NCCL_P2P_DISABLE="1"` and `NCCL_IB_DISABLE="1" or use `accelerate launch` which will do this automatically. FORK_LAUNCHEDr()=r'__dict__ initializedrGrIosenvirongettorchrKrrJpopr!NO_prepare_backend ValueErrorrLintr ImportError deepspeedr^r+init_distributedr MULTI_XPU MULTI_CPU distributedinit_process_grouprstrrhri local_ranklocal_world_sizerpsutil cpu_countset_num_threadswarningswarnrOrPrN set_devicexmset_replicationget_xla_supported_devicesxrt_world_size get_ordinalrget_local_ordinalget_world_sizeget_rank DEEPSPEEDr typer NotImplementedErrorrM) r7rQr/ env_deviceuse_sagemaker_dpdist_informationoriginal_backendrIrL use_deepspeeddistrnum_cpu_threads_per_processs r*r8zPartialState.__init__sY* y DIDL(A4HHJ6@6L%,z222RVDK,-DEEDJ%zz*=tDD # 'JNN#=wGG6Qo 'NOOSkSnn! &zz)T:: (,(=(=cCSUe(f(f %G%+;K0K0K !s:J!s!sjq!s!sttt"DL$4D !!M ]4<500rz~~lB7788B>>z~~&@'JJfTT577"-!B##;:::::#2244q1D1pt|`eppioppp(, -o6OQ`Qj5kkk % 1 @ @ B Bl)<\\T\\U[\\\$)BOD](^^^#B#D#D %()9)>%?%? 6"+./?/J+K+K <(+./?/J+K+K <(145E5V1W1W -.z~~mT::807BJ}- }d;;(9=M=XXX --$R"2!6v'7'B|$)_-FFF(*;)>,PR3S3S/T/TD,,%*%6%E%E%G%G"%*%6%?%?%A%A"=M=UC |R88999[k[v( OO    B(7(A%##>> # Assuming two GPU processes >>> import time >>> from accelerate.state import PartialState >>> state = PartialState() >>> if state.is_main_process: ... time.sleep(2) >>> else: ... print("I'm waiting for the main process to finish its sleep...") >>> state.wait_for_everyone() >>> # Should print on every process at the same time >>> print("Everyone is here") ``` z"accelerate.utils.wait_for_everyoneN)rLr MULTI_GPU MULTI_MLU MULTI_MUSA MULTI_NPUrrrFSDPrxrbarrierXLAr rendezvousrs r*wait_for_everyonezPartialState.wait_for_everyoneWs,   %  %  &  %  %  %  %   %    % % ' ' ' ' '  "o&9 9 9 M> ? ? ? ? ?: 9r)is_mainc#nK|s|dV|r|dSdSr-)r)r7rs r* _goes_firstzPartialState._goes_first{sS %  " " $ $ $   %  " " $ $ $ $ $ % %r)inputs"list | tuple | dict | torch.Tensor apply_paddingc#2Kjdkr|VdSt|t|trvt|t |dt fd|Dstdtj\}j ztj |z}|zj |krdndz}fd|||VdS)a Splits `input` between `self.num_processes` quickly and can be then used on that process. Useful when doing distributed inference, such as with different prompts. Note that when using a `dict`, all keys need to have the same number of elements. Args: inputs (`list`, `tuple`, `torch.Tensor`, `dict` of `list`/`tuple`/`torch.Tensor`, or `datasets.Dataset`): The input to split between processes. apply_padding (`bool`, `optional`, defaults to `False`): Whether to apply padding by repeating the last element of the input so that all processes have the same number of elements. Useful when trying to perform actions such as `gather()` on the outputs or passing in less inputs than there are processes. If so, just remember to drop the padded elements afterwards. Example: ```python # Assume there are two processes from accelerate import PartialState state = PartialState() with state.split_between_processes(["A", "B", "C"]) as inputs: print(inputs) # Process 0 ["A", "B"] # Process 1 ["C"] with state.split_between_processes(["A", "B", "C"], apply_padding=True) as inputs: print(inputs) # Process 0 ["A", "B"] # Process 1 ["C", "C"] ``` r Nrc3>K|]}t|kVdSr-)len).0vlengths r* z7PartialState.split_between_processes..s.AAAs1vv'AAAAAAr)z6All values in the dictionary must have the same lengthct|tttjfr|t |kr |dd}n |||} rit|tjr-ddlm}m}|| j }|||d}n"||dg dzt |z zz }|St|tr/| D]} ||||||<|Strddl m}t||r|t |krt |dz }|t |krt |}tt||} r| |dz g dzt | z zz } || S|S)Nr\r)pad_across_processessend_to_device) pad_indexr )Dataset) isinstancelisttuplerxTensorraccelerate.utilsrrrKdictkeysrdatasetsrrangeselect)r start_index end_indexresultrrtensorized_resultkeyr result_idcs _split_valuesrnum_samples_per_processr7s r*rz;PartialState.split_between_processes.._split_valuess&4 "=>> #f++--#BCC[FF#K $9:F ]!&%,77]YYYYYYYY-;N64;,O,O)!5!56GSYZ\S]!^!^!^6":,2IA2MPSTZP[P[2["\\ FD)) !;;==UUC"/-s [)"T"TF3KK (** :000000!&'22:&#f++55*-f++/K$s6{{22(+F I&*5i+H+H&I&I (n'IM?>UXY>Y\_`k\l\l>l+mmK%}}[999 r)) rOrrrrrallvaluesr|divmodrPmin) r7rr num_extrasrrrrrs ` ` @@@r*split_between_processesz$PartialState.split_between_processessSN   " "LLL FV fd # # [V[[]] 3 3A 6788FAAAAAAAAA [ !YZZZ.4VT=O.P.P+(+BBSI[]gEhEhh "99$BTWaBaBaQQghi ! ! ! ! ! ! ! ! FmFK;;;;;;;r)c#JK||jEd{VdS)a1 Lets the main process go first inside a with block. The other processes will enter the with block after the main process exits. Example: ```python >>> from accelerate import Accelerator >>> accelerator = Accelerator() >>> with accelerator.main_process_first(): ... # This will be printed first by process 0 then in a seemingly ... # random order by the other processes. ... print(f"This will be printed by process {accelerator.process_index}") ``` N)rrrs r*main_process_firstzPartialState.main_process_firsts7&##D$899999999999r)c#JK||jEd{VdS)a9 Lets the local main process go inside a with block. The other processes will enter the with block after the main process exits. Example: ```python >>> from accelerate.state import PartialState >>> state = PartialState() >>> with state.local_main_process_first(): ... # This will be printed first by local process 0 then in a seemingly ... # random order by the other processes. ... print(f"This will be printed by process {state.local_process_index}") ``` N)rrrs r*local_main_process_firstz%PartialState.local_main_process_firsts7&##D$>???????????r)NfunctionCallable[..., Any]c\|jstd|js|js|StS)a Decorator that only runs the decorated function on the main process. Args: function (`Callable`): The function to decorate. Example: ```python >>> from accelerate.state import PartialState >>> state = PartialState() >>> @state.on_main_process ... def print_something(): ... print("This will be printed by process 0 only.") >>> print_something() "This will be printed by process 0 only" ``` zUThe `PartialState` or `Accelerator` must be initialized before calling this function.)rtr|rrr0r7rs r*on_main_processzPartialState.on_main_processs?0 vtuu u   t'; Or)c0|js|js|StS)a Decorator that only runs the decorated function on the local main process. Args: function (`Callable`): The function to decorate. Example: ```python # Assume we have 2 servers with 4 processes each. from accelerate.state import PartialState state = PartialState() @state.on_local_main_process def print_something(): print("This will be printed by process 0 only on each server.") print_something() # On server 1: "This will be printed by process 0 only" # On server 2: "This will be printed by process 0 only" ``` )rrr0rs r*on_local_main_processz"PartialState.on_local_main_process%s$6  % T-A Or)c0|js|js|StS)a Decorator that only runs the decorated function on the last process. Args: function (`Callable`): The function to decorate. Example: ```python # Assume we have 4 processes. from accelerate.state import PartialState state = PartialState() @state.on_last_process def print_something(): print(f"Printed on process {state.process_index}") print_something() "Printed on process 3" ``` )rrr0rs r*on_last_processzPartialState.on_last_processDs$0   t'; Or)rPr}ch|t|j|S|j|ks|js|StS)a Decorator that only runs the decorated function on the process with the given index. Args: function (`Callable`, `optional`): The function to decorate. process_index (`int`, `optional`): The index of the process on which to run the function. Example: ```python # Assume we have 4 processes. from accelerate.state import PartialState state = PartialState() @state.on_process(process_index=2) def print_something(): print(f"Printed on process {state.process_index}") print_something() "Printed on process 2" ``` N)rP)r on_processrPrr0)r7rrPs r*rzPartialState.on_process`s@6  4?-HHH H  - / /9M /Or)rNch|t|j|S|j|ks|js|StS)aO Decorator that only runs the decorated function on the process with the given index on the current node. Args: function (`Callable`, *optional*): The function to decorate. local_process_index (`int`, *optional*): The index of the local process on which to run the function. Example: ```python # Assume we have 2 servers with 4 processes each. from accelerate import Accelerator accelerator = Accelerator() @accelerator.on_local_process(local_process_index=2) def print_something(): print(f"Printed on process {accelerator.local_process_index}") print_something() # On server 1: "Printed on process 2" # On server 2: "Printed on process 2" ``` N)rN)ron_local_processrNrr0)r7rrNs r*rzPartialState.on_local_processsC<  40FYZZZ Z  $(; ; ;TEY ;Or)c0|jrt|i|dSdSr-)rprintr7r.r/s r*rzPartialState.prints0  % # 4 "6 " " " " " # #r) torch.devicectr#dtjd<tjdSt rtjdSt rtjdStrtjdStj rtjdStrtjdStjd S) a[ Returns the default device which is: - MPS if `torch.backends.mps.is_available()` and `torch.backends.mps.is_built()` both return True. - CUDA if `torch.cuda.is_available()` - MLU if `is_mlu_available()` - MUSA if `is_musa_available()` - NPU if `is_npu_available()` - CPU otherwise 1PYTORCH_ENABLE_MPS_FALLBACKmpsmlumusanpuroxpurQ) rrurvrxrKrrrro is_availablerrs r*default_devicezPartialState.default_devices    '8;BJ4 5<&& &    '<&& &  '<'' '   '<&& & Z $ $ & & '<'' '    '<&& &<&& &r)rItuple[str, DistributedType]cd}|rddl}d}tj}ntrd}tj}nt t jdddkr|strd}tj }nhtrd}tj }nKtrd }tj}n.tjr|d }tj}|t t jdddkst%gd d d kr|st'r tj}n tj}|d vrQt-rCt%dgddks|tjkrt/dkrddl}nddl}d}n'|dvr!tjrd}nd}| tj}||fS)zhPrepares any imports needed before initializing the distributed backend and sets `self.backend` properlyNrsmddprZr[r\cnclmcclhcclnccl)PMI_SIZEOMPI_COMM_WORLD_SIZEMV2_COMM_WORLD_SIZErbr )NcclCCL_WORKER_COUNTz1.12r)Nrgrggloo),smdistributed.dataparallel.torch.torch_smddpr rrrr}rurvrwrrrrrrrxrorrrrrrroneccl_bindings_for_pytorch torch_cclris_mpi_availablerz)r7rQ sagemaker_dprIrL smdistributedrrs r*r{zPartialState._prepare_backends,   = ? ? ? ?G.8   # % % =G.2    b11 2 2b 8 8 8!! = #2#<  "$$ = #2#=  "## = #2#<  ((** =?$G#2#<  #  |R00 1 1R 7 7 i i iklmmpqqq =+-- =#2#<  #2#< =(($&&)%'9&:A>>BBFVZiZsFsFs"$$..66666$$$$M))e.?.P.P.R.R)   #.1 (((r)c|jdS|jtjkr)|jrt jdn|j|_dSt|jdd dd }|dvrtd|jd |d |d krtj |_dS|d krd }tt |}|j|z}t j|||_||jdS)zZ Sets the device in `self.device` to the current distributed environment. NrQ.r\MULTI_r)rQgpurrrrrZzCan't set device for z (z6), verify we should be calling `_set_device()` for it!rZr ro)rKrLr rzrGrxr rsplitreplacelowerr|r xla_devicegetattrrN device_countr)r7rK device_module device_indexs r*rzPartialState.set_devices> ; " F  O$6 6 615S%,u---@SDK FT*++11#66r:BB8RPPVVXX K K K(=  U??-//DKKK#E622M3m6P6P6R6RRL,v|<8B"""""H###'''X':EI9)9)9)9)9)v2220;;;; Q Q Q Q Q Qr)rFceZdZdZeZejgdzZ d%d&d Ze d'd Z d Z d(dZ e dZ ed)d*dZd+dZe dZe dZe d'dZe d'dZe d'dZdZed)d,dZedZedZe dZed-d!Zed+d-d"Zd#Zd-d$Z dS).r&a Singleton class that has information about the current training environment. **Available attributes:** - **device** (`torch.device`) -- The device to use. - **distributed_type** ([`~accelerate.state.DistributedType`]) -- The type of distributed environment currently in use. - **initialized** (`bool`) -- Whether or not the `AcceleratorState` has been initialized from `Accelerator`. - **local_process_index** (`int`) -- The index of the current process on the current server. - **mixed_precision** (`str`) -- Whether or not the current script will use mixed precision, and if so the type of mixed precision being performed. (Choose from 'no','fp16','bf16 or 'fp8'). - **num_processes** (`int`) -- The number of processes currently launched in parallel. - **process_index** (`int`) -- The index of the current process. - **is_last_process** (`bool`) -- Whether or not the current process is the last one. - **is_main_process** (`bool`) -- Whether or not the current process is the main one. - **is_local_main_process** (`bool`) -- Whether or not the current process is the main one on the local node. - **debug** (`bool`) -- Whether or not the current script is being run in debug mode. )deepspeed_pluginuse_ipex fsdp_pluginmegatron_lm_plugin dynamo_pluginNFmixed_precisionrrQr$_from_acceleratorc |j|_tdrd}tjikr t|fi||jtj||||jsd|_d|_|tddn| }|dkrgtstdtsrr#rr|rloggerwarningrxroget_device_capabilityrArLr rrHrrrurvrwrdowncast_bfloatrrrrrrset_mixed_precisionr?rr@rrzrrIr rKrbackendsmatmul allow_tf32r) r7rBrQrAr=r?r@rCr/s r*r8zAcceleratorState.__init__NsT* 3 4 4 C  % + +  ' ' ' ' ' \7888 555N S%)D " DM#*&&BDIII$**,,  %''')) -$j.//-NNb HhHhHjHjbbb '-O!.D $  T -1,A_E^,^,^DDdsD !$(;;;@Vdh@i@i@i;"f,,z~~&@AA558VV >2:=a&& #67/3,,58VV >2:=a&& #67/4, :GDDNNWZN)9&(7(A%%&))*)) +:>>"7AAVKK{Of,;,@D),44#778MNNN'2D$:>>">HHFRRW[Wl#-uXX-<,GD)&::4;PQQQ.@D+&?+DoF_apas*ttt$&&*$78MW[$\$\$\DMM$)DM"*m.>>>)T11K$..8<#*5"*m.>>>)T11K$..8<#*5=A=RL &'9 : : :]N SN Sr)r#c,|jtjkSr-)r'rFrs r*rtzAcceleratorState.initializeds!\%???r)ctd|jdz}|jtjkr|d|jjdz }|S)Nz Mixed precision type: rz ds_config: )rFrrBrLr rr=deepspeed_config)r7reprs r*rzAcceleratorState.__repr__s]~~&&((+^dFZ+^+^+^^  O$= = = L$"7"HLLL LD r)c |jrd}|r3|jjdkr#t|d|I||jkr@|jtjkr-t|d|ddSdSdSdS)zeChecks if a modification is trying to be made and the `AcceleratorState` has already been initializedzAcceleratorState has already been initialized and cannot be changed, restart your runtime completely and pass `{flag}` to `Accelerator()`.rQzcpu=True)flagNzmixed_precision='r/) rtrKrr|formatrHrLr r)r7rBrQerrs r*rTz#AcceleratorState._check_initializeds   Z_C >t{'500 !rsV#""""" %%%%%%********** 2877777*)))))))))'''%((('''  8 $ $0000IO80/11 LTT7L z Qz Qz Qz Qz Qz Qz Qz Qz{U{U{U{U{U{U{U{U| {,{,{,{,{,{,{,{,{,{,r)