
    g]s                        d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlZddlmZ ddlmZmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZ d
dlm Z m!Z! g fdZ"d Z#de j$        de
e%e%f         fdZ&de j$        deee%         e
e%e%f         f         fdZ'de j$        de
e%e%f         fdZ(de j$        deee%         e
e%e%f         f         fdZ)	 dde j$        de
e%e%f         de*dee j$        e
e%e%f         f         fdZ+dee%         de
e%e%f         fdZ,dede j$        dee j$        e
e%e	f         f         fdZ-d Z. G d d          Z/dS )     Nliteral_evalwhich)AnyDictListTuple   )SageMakerConfig)	DynamoBackendPrecisionTypeis_fp8_availableis_ipex_availableis_mlu_availableis_musa_availableis_npu_availableis_torch_xla_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)is_port_in_usemerge_dicts   )DistributedTypeSageMakerDistributedTypec                     |                     |          \  }}t          |                                           D ]9\  }}|t          |                                          v rt	          |||           :|S )z4
    Filters out all `accelerate` specific args
    )parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalues          S/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/utils/launch.py_filter_argsr*   *   su     )),77KHa4jj&&(( * *
U$x..%%''''Hc5)))O    c                      d dD             } t          |           dk    rt          d          | d         }t          j        |dg          }d|v r|ddd	d
fS |ddddfS )a  
    Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
    are: OpenMPI, Intel MPI, or MVAPICH.

    Returns: Program name and arg names for hostfile, num processes, and processes per node
    c                 0    g | ]}t          |          |S  r   ).0xs     r)   
<listcomp>z$_get_mpirun_args.<locals>.<listcomp>=   s#    ===aE!HH====r+   )mpirunmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPI
--hostfilez-nz
--npernodez	--bind-toz-fz-ppn )lenOSError
subprocesscheck_output)mpi_appsmpi_appmpirun_versions      r)   _get_mpirun_argsr=   5   s     >=0===H
8}}tuuu qkG,g{-CDDNn$$dL+EE dFB..r+   r"   current_envc                     d}t          |           D ]_}|                    d          rHt          | |          }|6t          t          | |                    || |                                 <   `|S )z.
    Setup the FP8 environment variables.
    ACCELERATE_fp8_)r   
startswithgetattrstrupper)r"   r>   prefixargr(   s        r)   setup_fp8_envrH   M   s     FDzz P P>>&!! 	PD#&&E 8;GD#<N<N8O8Ov4syy{{445r+   returnc                 	   g }| j         r| j        rt          d          | j        t	                      \  }}}}}t          | dd          }t          | dd          }| j        }	t          | dd          }
|
r|	rt          |
|	z            nd}|||| j        ||gz  }|
r||t          |
          gz  }|r|||gz  }| j         s;|                    t          j
                   | j        r|                    d           |                    | j                   |                    | j                   t          j                                        }t          | j        p| j                  |d	<   | j        rd
|d<   | j        dk    ru| j        nt+                      r| j        |d<   nUt-                      r| j        |d<   n<t/                      r| j        |d<   n#t1                      r| j        |d<   n
| j        |d<   | j        dk    r;| j        |d<   t          | j                  |d<   | j        t          |          |d<   n>| j        dk    r3| j        | j        nd|d<   | j        t          | j                  nd|d<   	 t9          | j                                                  }nL# t          $ r? t          d| j                                         dt9          j                     d          w xY wt          |          |d<   | j                                        dk    r-tA                      stC          d          tE          | |          }	 tG          | j$        %                                          }nL# t          $ r? t          d| j$        %                                 dtG          j                     d          w xY w|j&        |d<   | j'        |d <   t          | j(                  |d!<   t          | j)                  |d"<   t          | j*                  |d#<   tW                      rRt          | j,                                                  |d$<   t          | j-                                                  |d%<   | j.        rd|d&<   ||fS )'zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    0--module and --no_python cannot be used togetherN
mpirun_cclzbind-tosocketnum_processes1z-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKMLU_VISIBLE_DEVICESMUSA_VISIBLE_DEVICESASCEND_RT_VISIBLE_DEVICESCUDA_VISIBLE_DEVICESr   MASTER_ADDRMASTER_PORTCCL_WORKER_COUNTz	127.0.0.129500Unknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONfp8iFP8 is not available on this machine. Please ensure that either Transformer Engine or MSAMP is installed.Unknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMICOMP_NUM_THREADSACCELERATE_USE_IPEXACCELERATE_USE_XPUACCELERATE_CPU_AFFINITY)/	no_pythonmodule
ValueErrormpirun_hostfiler=   rC   num_machinesrD   appendsys
executabletraining_scriptextendtraining_script_argsosenvironcopycpuuse_cpudebuggpu_idsr   r   r   r   main_process_ipmain_process_portrN   r   mixed_precisionlowerlistr   RuntimeErrorrH   r   dynamo_backendrE   r(   dynamo_modedynamo_use_fullgraphdynamo_use_dynamicnum_cpu_threads_per_processr   ipexuse_xpuenable_cpu_affinity)r"   cmdmpi_app_namehostfile_argnum_proc_argproc_per_node_argbind_to_argrL   bind_torp   rN   nproc_per_noder>   r   r   s                  r)   prepare_simple_launcher_cmd_envr   Z   s    C~ M$+ MKLLL'ScSeSePlL2C[T<66
$	844(ot<<?LfQ]f]l:;;;cf 
 	
  	6L#m"4"455C 	*K))C> 

3>"""; 	JJtJJt#$$$JJt()))*//##K(+DH,D(E(EK$%z 6/5+,|u!9 		?.2lK*++ 	?15K-..   	?26,K.// 	?7;|K34426,K./1%)%9M"%()?%@%@M"+.1*ooK*+		a		=A=Q=]T%9%9cnM"DHDZDfS)?%@%@%@lsM"
'(<(B(B(D(DEE 
 
 
sT-A-G-G-I-Iss\i\n\p\psss
 
 	


 14O0D0DK,-!!##u,,!! 	{   $D+66
&t':'@'@'B'BCC 
 
 
lt':'@'@'B'BllUbUgUiUilll
 
 	

 0>/CK+,,0,<K()589R5S5SK1236t7N3O3OK/0%()I%J%JK!" F-0^^-A-A-C-C)*,/,=,=,C,C,E,E() 514-.s   >&J% %A	K.&M5 5A	N>c                 b   | j         }| j        }| j        }| j        }|dk    rt	          ||z            | _        t	          |          | _        t          | j                  | _	        t          | dd          r)t	          |          | _        t	          |          | _        n7| d| | _        n*t	          |          | _        |t	          |          | _        |d}|dk    pt          | j                  dk    }|r"t          |          rt          d| d	          | j        r| j        rt%          d
          | j        rd| _        n| j        rd| _        t&          j                                        }| j        rd|d<   t          | dd          }|dk    r\| j        Ut1                      r||d<   nAt3                      r||d<   n-t5                      r||d<   nt7                      r||d<   n||d<   | j                                        }	 t=          |          }n5# t$          $ r( t%          d| dt=          j                     d          w xY wt	          |          |d<   | j                                        dk    r-tA                      stC          d          tE          | |          }	 tG          | j$        %                                          }	nL# t$          $ r? t%          d| j$        %                                 dtG          j                     d          w xY w|	j&        |d<   | j'        |d<   t	          | j(                  |d<   t	          | j)                  |d<   | j*        rd|d <   | j+        r| j,        st%          d!          t	          | j-                  |d"<   t	          | j.                                                  |d#<   t	          | j/                  |d$<   | j0        t	          | j0                  |d%<   | j1        t	          | j1                  |d&<   | j2        t	          | j2                  |d'<   | j3        t	          | j3                  |d(<   t	          | j4                                                  |d)<   t	          | j5                                                  |d*<   t	          | j+                                                  |d+<   t	          | j,                                                  |d,<   t	          | j6                                                  |d-<   | j7        rd.}
d|d/<   t	          | j8                  ||
d0z   <   t	          | j9                  ||
d1z   <   t	          | j:                  ||
d2z   <   | j;        t	          | j;                  ||
d3z   <   | j<        t	          | j<                  ||
d4z   <   | j=        t	          | j=                  ||
d5z   <   | j>        t	          | j>                  ||
d6z   <   t	          | j?                  |d7<   | j@        rd8|d9<   |S ):z_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    r   same_networkF:N<s  r   3Tried to launch distributed communication on port `$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rK   TrQ   rR   r}   rS   rT   rU   rV   rW   rX   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`FSDP_SHARDING_STRATEGYFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESFSDP_ACTIVATION_CHECKPOINTINGMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREE	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERrh   rO   rk   )ArN   rp   r~   r   rD   r   nnodesintmachine_rank	node_rankrC   master_addrmaster_portrdzv_endpointr   ConnectionErrorrm   rl   rn   rw   rx   ry   r|   r}   r   r   r   r   r   r   r   r   r   r   rH   r   r   rE   r(   r   r   r   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_statesfsdp_sharding_strategyfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetchfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsfsdp_activation_checkpointinguse_megatron_lmmegatron_lm_tp_degreemegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizerr   r   )r"   rN   rp   r~   r   need_port_checkr>   r}   r   r   rF   s              r)   prepare_multi_gpu_envr      s    &M$L*O.a!-<"?@@,''T.//4// 	J"?33D"#455D$3!I!I6G!I!ID!-00("#455D ! #a'F3t/@+A+AQ+FO 
>*;<< 
{BS { { {
 
 	
 { t~ KLLL	 	 *//##Kz 6/5+,dIu--G%DL4 		:.5K*++ 	:18K-..   	:29K.// 	:7>K34429K./*0022Ou'88 u u us/ss\i\n\p\pssstttu 14O0D0DK,-!!##u,,!! 	{   $D+66
&t':'@'@'B'BCC 
 
 
lt':'@'@'B'BllUbUgUiUilll
 
 	

 0>/CK+,,0,<K()589R5S5SK1236t7N3O3OK/0} g-3)*. 	xt7S 	xvwww03D4O0P0P,--01I-J-J-P-P-R-R)*-01I-J-J)*%136t7Q3R3RK/02>:=d>e:f:fK67&2478S4T4TK01$025d6O2P2PK.//243M/N/N/T/T/V/V+,.1$2K.L.L.R.R.T.T*+8;D<_8`8`8f8f8h8h4514T5Q1R1R1X1X1Z1Z-.7:4;]7^7^7d7d7f7f34 p4:01,/0J,K,KF[(),/0J,K,KF[()478Z4[4[F001-98;D<^8_8_K!4450<;>t?d;e;eK!7781=<?@f<g<gK!8895A@CDDn@o@oK!<<=%()I%J%JK!" 514-.s   H 2I1&K A	L!c                 &   | j         }| j        }| j        }| j        }d}| j        t
          d         | _        |dk    r| j        t
          d         k    rddg}|                    dt          | j                  dt          | j                  g           | j	        *|                    dt          | j	                  g           nb| j
        *|                    d	t          | j
                  g           n1|                    d
t          | j         | j        z            g           |r$|                    dt          |          g           |                    dt          |          g           | j        r| j        rt          d          | j        r|                    d           n| j        r|                    d           |                    | j                   |                    | j                   n|dk    r| j        t
          d         k    rt          ||z            | _        t          |          | _        t'          | j                  | _        t-          | dd          r)t          |          | _        t          |          | _        n7| d| | _        n*t          |          | _        |t          |          | _        |d}|dk    pt'          | j                  dk    }|r"t5          |          rt7          d| d          | j        r| j        rt          d          | j        rd| _        n| j        rd| _        t8          j                                        }| j        rd|d<   t-          | dd          }|dk    r\| j         UtC                      r||d<   nAtE                      r||d<   n-tG                      r||d<   ntI                      r||d<   n||d<   	 tK          | j&        '                                          }	nL# t          $ r? t          d | j&        '                                 d!tK          j(                     d"          w xY wtS          d#t8          j*        +                    d"                    |d#<   t          |	          |d$<   | j&        '                                d%k    r-tY                      st[          d&          t]          | |          }t          | j/                  '                                |d'<   d|d(<   | j0        t          | j0                  |d)<   | j1        t          | j1                  |d*<   | j2        )t          | j2                  '                                |d+<   | j3        )t          | j3                  '                                |d,<   | j4        )t          | j4                  '                                |d-<   | j5        )t          | j5                  '                                |d.<   | j6        )t          | j6                  '                                |d/<   | j7        t          | j7                  |d0<   | j8        rd1|d2<   | j9        t          | j9                  |d3<   ||fS )4zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    Nr   r   	deepspeedz--no_local_rankr4   z
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrK   z--modulez--no_pythonr   Fr   r   r   r   TrQ   rR   r}   rS   rT   rU   rV   rW   rX   r]   r^   r_   
PYTHONPATHr`   ra   rb   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILErO   rk   (ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES):rN   rp   r~   r   deepspeed_multinode_launcherr   ru   rD   deepspeed_hostfiledeepspeed_exclusion_filterdeepspeed_inclusion_filterrm   rl   rn   rq   rt   rv   r   r   r   r   r   rC   r   r   r   r   r   rw   rx   ry   r|   r}   r   r   r   r   r   r   r   r   env_var_path_addpathabspathr   r   rH   'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_filer   deepspeed_moe_layer_cls_names)
r"   rN   rp   r~   r   r   r   r>   r}   r   s
             r)   prepare_deepspeed_cmd_envr   4  s    &M$L*O.
C (0,I!,L)aD=A^_`Aaaa-.

L#d&=">">cRVRsNtNtuvvv*6JJ788    ,8JJ788    JJc$*<@Q*Q&R&RSTTT 	@JJ_)=)=>???

OS):%;%;<===; 	&4> 	&OPPP[ 	&JJz""""^ 	&JJ}%%%

4'(((

4,----			d?C`abCccc!-<"?@@,''T.//4// 	J"?33D"#455D$3!I!I6G!I!ID!-00("#455D ! #a'F3t/@+A+AQ+FO 
>*;<< 
{BS { { {
 
 	
 { t~ KLLL	 	 *//##Kz 6/5+,dIu--G%DL4 		:.5K*++ 	:18K-..   	:29K.// 	:7>K34429K./
'(<(B(B(D(DEE 
 
 
sT-A-G-G-I-Iss\i\n\p\psss
 
 	


 !1rws?S?S T TK03O0D0DK,-!!##u,,!! 	{   $D+6614T5a1b1b1h1h1j1jK-..4K*+"9<T_9M9M56'3@CDDd@e@e<=)69$:P6Q6Q6W6W6Y6Y23$0GJ4KhGiGiGoGoGqGqCD ,CFtG`CaCaCgCgCiCi?@'9<T=Q9R9R9X9X9Z9Z56".EHIdEeEeEkEkEmEmAB!-:=d>X:Y:Y67 514-.)5BEdFhBiBi>?s   .&P A	QFpodc                     | j         dk    r"t          d          r| j        rd|d<   nd|d<   | j        rd|d<   |r| j        | _        | j        | _        | |fS )	zY
    Prepares and returns an environment with the correct TPU environment variables.
    bf16T)check_is_tpurO   XLA_DOWNCAST_BF16XLA_USE_BF16rQ   rR   )r   r   downcast_bf16r|   tpu_vmvmtpu_nametpu)r"   r>   r   s      r)   prepare_tpur     s     v%%*@d*S*S*S% 	./2K+,,*-K'z 6/5+,
 !+=r+   nargsc                 T   t          |           dk     ri S d }t          j                    }|                    |           \  }}t	          |          D ]\  }}|                    d          rd }|dz   t          |          k     r.||dz                                d          rt          d          nt          d          ||                    ||           |                    ||           d |                    |           j	        
                                D             S )	Nr   c                 z    	 t          |           } | dz  | k    rt          |           S | S # t          $ r | cY S w xY w)Nr   )floatr   rn   )ss    r)   _infer_typez+_convert_nargs_to_dict.<locals>._infer_type  sS    	aAAv{{1vvH 	 	 	HHH	s   &+ + ::)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)actionc                 @    i | ]\  }}||d v rt          |          n|S ))TrueFalser   )r/   r'   r(   s      r)   
<dictcomp>z*_convert_nargs_to_dict.<locals>.<dictcomp>  sF       C 	U.?%?%?l5!!!U  r+   )r6   argparseArgumentParserr   	enumeraterB   rn   add_argument
parse_args__dict__r   )r   r  r#   r&   unknownindexargumentr  s           r)   _convert_nargs_to_dictr    sh   
5zzA~~	   $&&F((//JAw$W-- = =x{++ 	=Fqy3w<<''519%00== $ G   ! C   ~##H;#????##HV#<<<  ++E22;AACC   r+   sagemaker_configc                 N	   t          d           | j        t          j        d<   | j        | j        t          j        d<   nF|j        0|j        )|j        t          j        d<   |j        t          j        d<   nt          d          t          j        	                    |j
                  }|sd}t          j                            |j
                  }|                    d          st          d	| d
          t          d           t          |j                  }	 t!          |j                                                  }nL# t          $ r? t          d|j                                         dt!          j                     d          w xY w	 t)          |j                                                  }nL# t          $ r? t          d|j                                         dt)          j                     d          w xY wdt/          |          |j        |j        t/          |j                  t/          |j                  | j        j        d}|j                                        dk    r-t;                      st=          d          t?          ||          }d }| j        t@          j!        k    rddddiii}d }	| j"        t          d| j"         d           i }	tG          | j"                  5 }
tI          |
          D ]D\  }}|dk    r|%                    d          }|d         &                                |	|d         <   E	 d d d            n# 1 swxY w Y   t          d|	            d }| j'        t          d| j'         d           g }tG          | j'                  5 }
tI          |
          D ]Y\  }}|dk    r|%                    d          }|d         |d         &                                d}|(                    |           Z	 d d d            n# 1 swxY w Y   t          d|            t          d            | j)        ||| j*        | j+        | j,        | j-        | j.        | j/        | j0        d!||||d"}| j1        te          | j1        |          }||	fS )#Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profiler_   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr]   r^   rc   rQ   )ACCELERATE_USE_SAGEMAKERr`   rd   re   rf   rg   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEra   rb   smdistributeddataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configdistributionhyperparametersenvironmentmetric_definitions)3printregionrw   rx   profileaws_access_key_idaws_secret_access_keyr7   r   dirnamert   basenameendswithrn   r  rv   r   r   r   r   r   r   rE   rD   r(   r   r   r   distributed_typer   r   rH   r   DATA_PARALLELsagemaker_inputs_fileopenr  splitstripsagemaker_metrics_filerq   r"  iam_role_namer&  r'  r(  r)  rp   ec2_instance_typeadditional_argsr   )r  r"   r$  r#  r.  r   r   r/  r-  sagemaker_inputsfileilinelsagemaker_metricsmetric_dicts                   r)   prepare_sagemager_args_inputsrJ    s    

4555'7'>BJ#$ +$4$<
=!!			+0J0V*.*@
&'.2.H
*++uvvv !566J 
'""4#788K&& dbT_bbbccc	
3444,T-FGGO
'(<(B(B(D(DEE 
 
 
sT-A-G-G-I-Iss\i\n\p\psss
 
 	



&t':'@'@'B'BCC 
 
 
lt':'@'@'B'BllUbUgUiUilll
 
 	

 %+&)/&:&:%3%9"&"2+.t/H+I+I),T-D)E)E1A1R1X K !!##u,,!! 	{   $D+66L(,D,RRR'.9d:K)LM -9\/?/U\\\]]]"899 	6T$T?? 6 6466JJt$$)*1 1&&	6	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	<*:<<=== .:^0@0W^^^___"9:: 		6d$T?? 6 6466JJt$$aDqTZZ\\  "((55556		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 	>+<>>??? 

%/"  . 0 E+;&1)7*7); %$*"/ D$ '3+;TBB!!!s@   &E A	F&F9 9A	H4AMMM*A*P!!P%(P%c                     d t           j                            | d                              d          D             }|                    t          |                     d                    |          S )z
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    c                 8    g | ]}t          |          d k    |S )r   )r6   )r/   ps     r)   r1   z$env_var_path_add.<locals>.<listcomp>r  s$    RRR1s1vvPQzzQzzzr+   r5   r   )rw   rx   getr=  rq   rD   join)env_var_namepath_to_addpathss      r)   r   r   m  s`    
 SR
|R88>>sCCRRRE	LL[!!"""88E??r+   c                        e Zd ZdZddZd ZdS )PrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    NOFc                 J    || _         t          |          | _        || _        d S )N)launcherr   r9  r|   )selfrW  r9  r|   s       r)   __init__zPrepareForLaunch.__init__  s%      /0@ A A


r+   c                 @   | j         rt          t          j                            d                    }t          j                            d          }t
          j                            d|t
          j                            ||          |           n| j	        t          j        t          j        t          j        t          j        t          j        t          j        fv rt#          |          t          j        d<   t          t          j                            dd                    }t          t          j                            dd	                    }t#          ||z  |z             t          j        d
<   t#          d          t          j        d<    | j        |  d S )N
WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROCr   	NODE_RANKr   RANKFORK_LAUNCHED)r|   r   rw   rx   rN  torchdistributedinit_process_group	FileStorer9  r   	MULTI_GPU	MULTI_MLU
MULTI_MUSA	MULTI_NPU	MULTI_XPU	MULTI_CPUrD   rW  )rX  r  r"   r`  rdv_filenprocr   s          r)   __call__zPrepareForLaunch.__call__  sN   : 	@RZ^^L99::Jz~~&ABBH00'11(JGG%	 1     "%%&%%%'
 
 
 (+5zzBJ|$
w2233EBJNN;::;;I!$UY%6%>!?!?BJv&)!ff
?#tr+   N)rU  F)__name__
__module____qualname____doc__rY  rr  r.   r+   r)   rT  rT  w  sA        
 
   
    r+   rT  )F)0r	  rw   r8   rr   astr   shutilr   typingr   r   r	   r
   rf  commands.config.config_argsr   utilsr   r   r   r   r   r   r   r   r   utils.constantsr   utils.otherr   r   dataclassesr   r   r*   r=   	NamespacerD   rH   r   r   r   boolr   r  rJ  r   rT  r.   r+   r)   <module>r     st    				     



             ) ) ) ) ) ) ) ) ) ) ) )  9 9 9 9 9 9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 < ; ; ; ; ; 5 5 5 5 5 5 5 5 B B B B B B B B -/    / / /0
* 
c3h 
 
 
 
Z(*< ZtCyRVWZ\_W_R`G`Aa Z Z Z Zzz 2 ztCH~ z z z zzCH$6 C5cDQTVYQYNAZ;[ C C C CN HM 

+/S>@D
8tCH~-.   ('$s) 'S#X ' ' ' 'Tr"%r"-5-?r"
8tCH~-.r" r" r" r"j  + + + + + + + + + +r+   