gu4ddlZddlZddlZddlZddlZddlZddlmZddlZddl Z ddl m Z m Z ddl mZddlmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/e!r*dd l0m1Z1dd l2m3Z3d Z4ej5e4d e3gej6e7Z8dddddddZ9dZ:Gddej;ZdZ?dZ@dZAdZBdefd ZCd!ZDd"ZEd#ZFe7d$kr eFdSdS)&N)Path)default_config_fileload_config_from_file)SageMakerConfig)DYNAMO_BACKENDS)CustomArgumentParser)get_int_from_env)ComputeEnvironmentDistributedTypePrepareForLaunch _filter_argscheck_cuda_p2p_ib_supportconvert_dict_to_env_variablesis_bf16_availableis_deepspeed_availableis_mlu_availableis_musa_availableis_npu_availableis_rich_availableis_sagemaker_availableis_torch_versionis_torch_xla_availableis_xpu_availablepatch_environmentprepare_deepspeed_cmd_envprepare_multi_gpu_envprepare_sagemager_args_inputsprepare_simple_launcher_cmd_env prepare_tpu str_to_bool)DEEPSPEED_MULTINODE_LAUNCHERSTORCH_DYNAMO_MODES) get_console) RichHandlerz %(message)sz[%X])formatdatefmthandlersDistributed GPUsTPUDeepSpeed ArgumentsFSDP ArgumentsMegatron-LM Arguments FP8 Arguments) multi_gputpu use_deepspeeduse_fsdpuse_megatron_lm fp8_backendcxd|vrd}|dr|ddddSdS)zIFinds all cases of - after the first two characters and changes them to _r3 --fp8_backendz--N-_) startswithreplace)options V/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/commands/launch.py clean_optionr=RsP  ,abbz!!#s+++,,cFeZdZdZfdZdejffd ZfdZxZ S)CustomHelpFormattera  This is a custom help formatter that will hide all arguments that are not used in the command line when the help is called. This is useful for the case where the user is using a specific platform and only wants to see the arguments for that platform. cLtj|i|gd|_dS)N)Hardware Selection ArgumentsResource Selection ArgumentsTraining Paradigm Argumentszpositional argumentszoptional arguments)super__init__titles)selfargskwargs __class__s r<rFzCustomHelpFormatter.__init__as6$)&)))    r>actioncdtjdvr+dtjddvrtjdd}ntjdd}t|dkr-tt t |}d|D}d|D}|jj|j|zvrtj |_ n|jjdkrVt|j t|rtj |_ nu|j d z|_ ne|jjd krUt|j t|rtj |_ n|j d z|_ d |j D|_ t|dS) N acceleraterlaunchr6cHg|]}|tv| S)options_to_groupkeys).0args r< z4CustomHelpFormatter.add_argument..ss/TTTcSc(g|]}t|SrR)rS)rUos r<rWz4CustomHelpFormatter.add_argument..tsGGG1+A.GGGr>rBz (currently selected)rDc*g|]}d|ddv|S)r7r6NrR)rUss r<rWz4CustomHelpFormatter.add_argument..s/ V V VqSPQRSRTRTPUEUEUEUEUEUr>)sysargvlenlistmapr= containertitlerGargparseSUPPRESShelpsetoption_strings isdisjointrE add_argument)rHrLrIused_platforms used_titlesrKs r<riz CustomHelpFormatter.add_argumentks 38A; & &8sx|+C+C8ABBct|jjdkrg|j_d|j_t dS)Nr6)r^_current_sectionitemsheadingrE end_section)rHrKs r<rqzCustomHelpFormatter.end_sectionsN t$* + +a / /*,D ! ',.D ! ) r>) __name__ __module__ __qualname____doc__rFrcActionrirq __classcell__)rKs@r<r@r@Zs      %8?%%%%%%4r>r@cDd}| |d|ddt}ntd|ddt}|dddd |d dd |dddd |dd}|dddd|dddd|dddd|dddd|dd}|dt gd d!"|d#t dd$%|d&t dd'%|d(t dd)%|d*ddd+|d,t d-gd.tDzd/"|d0t d1td23|d4ddd5|d6ddd7|d8d9}|d:ddd;|dddd?|d@dddA|dBdC}|dDddE |dFdddG|dHt ddI%|dJt ddK%|dLt ddM%|dNdOdPt dQR|dSt ddT%|dUt d1dV%|dWt dXdY%|dZt d[d\%|d]t d^d_%|d`tdadb%|dcdddde |dfddg |dhdi}|djddkdlm|dndodkdpm|dqddr |dst dtduv|dwt dtdxv|dyt ddz%|d{dd| |d}d~}|ddt dR|ddt dR|ddt dR|ddt dR|ddt dR|ddt dR|ddt dR|ddtdR|ddt dR|ddt dR|ddt dR|ddt dR|ddt dR|ddt dR|ddt dR|dd} | ddt dR| dt dd%| dt dd%| dt dd%| ddt dR| ddt dR| ddt dR| ddt dR| ddt dR| ddt dR| ddt dR| ddt dR|dd} | dt dd%| dt dd%| dt ddì%| ddt dŬR| ddt dǬR| ddt dɬR| ddtd̬R|ddΦ} | dt ddgdҬ"| ddddԬ| dt d^d֬%| dt ddج%| dt dddgdܬ3| dt dd߬%| dt dddgd3| dddd%| dt dddgd3|dd} | dt dd%| dt dd%|ddd |dt d|dd} | dt dd%| dt dd%|dtj d|| t|S(NzLaunch a python script in a distributed scenario. Arguments can be passed in with either hyphens (`--num-processes=2`) or underscores (`--num_processes=2`)rOF) descriptionadd_help allow_abbrevformatter_classzAccelerate launch commandz-hz--helprez Show this help message and exit.)rLrez --config_filezFThe config file to use for the default values in the launching script.)defaultrez--quietz-q store_truezSilence subprocess errors from the launch stack trace and only show the relevant tracebacks. (Only applicable to DeepSpeed and single-process configurations)rBz0Arguments for selecting the hardware to be used.z--cpuz0Whether or not to force the training on the CPU.)r}rLrez --multi_gpuz=Whether or not this should launch a distributed GPU training.--tpuz1Whether or not this should launch a TPU training.z--ipexzLWhether or not this should launch a Intel PyTorch Extension (IPEX) training.rCz@Arguments for fine-tuning how available hardware should be used.--mixed_precision)nofp16bf16fp8zWhether or not to use mixed precision training. Choose between FP16 and BF16 (bfloat16) training. BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.)typechoicesre--num_processesz9The total number of processes to be launched in parallel.)rr}re--num_machinesz3The total number of machines used in this training.z--num_cpu_threads_per_processzLThe number of CPU threads per process. Can be tuned for optimal performance.z--enable_cpu_affinityziWhether or not CPU affinity and balancing should be enabled. Currently only supported on NVIDIA hardware.--dynamo_backendrc6g|]}|SrR)lower)rUbs r<rWz)launch_command_parser..s ===!''))===r>zkChoose a backend to optimize your training with dynamo, see more at https://github.com/pytorch/torchdynamo.z --dynamo_moder}z4Choose a mode to optimize your training with dynamo.)rr}rrez--dynamo_use_fullgraphz[Whether to use full graph mode for dynamo or it is ok to break model into several subgraphsz--dynamo_use_dynamicz(Whether to enable dynamic shape tracing.rDz;Arguments for selecting which training paradigm to be used.z--use_deepspeedzWhether to use deepspeed.z --use_fsdpzWhether to use fsdp.z--use_megatron_lmzWhether to use Megatron-LM.z --use_xpuzDWhether to use IPEX plugin to speed up training on XPU specifically.r(z.Arguments related to distributed GPU training.z --gpu_idszWWhat GPUs (by id) should be used for training on this machine as a comma-seperated listz--same_networkzQWhether all machines used for multinode training exist on the same local network.z--machine_rankz9The rank of the machine on which this script is launched.z--main_process_ipz(The IP address of the machine of rank 0.z--main_process_portz:The port to use to communicate with the machine of rank 0.z-tz--tee0z4Tee std streams into a log file and also to console.)r}rrez --log_dirzBase directory to use for log files when using torchrun/torch.distributed.run as launcher. Use with --tee to redirect std streams info log files.z--rolez"User-defined role for the workers.z--rdzv_backendstaticzFThe rendezvous method to use, such as 'static' (the default) or 'c10d'z --rdzv_confrmzJAdditional rendezvous configuration (=,=,...).z--max_restartsrz7Maximum number of worker group restarts before failing.z--monitor_intervalg?z6Interval, in seconds, to monitor the state of workers.z-mz--modulezwChange each process to interpret the launch script as a Python module, executing with the same behavior as 'python -m'.z --no_pythonz|Skip prepending the training script with 'python' - just execute it directly. Useful when the script is not a Python script.r)zArguments related to TPU.z --tpu_clustertpu_use_clusterz*Whether to use a GCP TPU pod for training.)rLdestre--no_tpu_cluster store_falsez?Should not be passed explicitly, this is for internal use only.z--tpu_use_sudozGWhether to use `sudo` when running the TPU training script in each pod.z--vmappendzkList of single Compute VM instance names. If not provided we assume usage of instance groups. For TPU pods.)rrLrez--envzOList of environment variables to set on the Compute VM instances. For TPU pods.--main_training_functionzTThe name of the main function to be executed in your script (only for TPU training).z--downcast_bf16zWhether when using bf16 precision on TPUs if both float and double tensors are cast to bfloat16 or if double tensors remain as float32.r*zArguments related to DeepSpeed.z--deepspeed_config_filezDeepSpeed config file.z --zero_stagez{DeepSpeed's ZeRO optimization stage (useful only when `use_deepspeed` flag is passed). If unspecified, will default to `2`.z--offload_optimizer_devicezDecides where (none|cpu|nvme) to offload optimizer states (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'.z--offload_param_devicezDecides where (none|cpu|nvme) to offload parameters (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'.z--offload_optimizer_nvme_pathzDecides Nvme Path to offload optimizer states (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'.z--offload_param_nvme_pathzDecides Nvme Path to offload parameters (useful only when `use_deepspeed` flag is passed). If unspecified, will default to 'none'.z--gradient_accumulation_stepszNo of gradient_accumulation_steps used in your training script (useful only when `use_deepspeed` flag is passed). If unspecified, will default to `1`.z--gradient_clippingzgradient clipping value used in your training script (useful only when `use_deepspeed` flag is passed). If unspecified, will default to `1.0`.z--zero3_init_flagzDecides Whether (true|false) to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will default to `true`.z--zero3_save_16bit_modelzDecides Whether (true|false) to save 16-bit model weights when using ZeRO Stage-3. Only applicable with DeepSpeed ZeRO Stage-3. If unspecified, will default to `false`.z--deepspeed_hostfilez@DeepSpeed hostfile for configuring multi-node compute resources.z--deepspeed_exclusion_filterz>DeepSpeed exclusion filter string when using mutli-node setup.z--deepspeed_inclusion_filterz>DeepSpeed inclusion filter string when using mutli-node setup.z--deepspeed_multinode_launcherzMDeepSpeed multi-node launcher to use. If unspecified, will default to `pdsh`.z--deepspeed_moe_layer_cls_nameszcomma-separated list of transformer MoE layer class names (case-sensitive) to wrap ,e.g, `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention,JetMoEBlock` ... (useful only when `use_deepspeed` flag is passed).r+z3Arguments related to Fully Shared Data Parallelism.z--fsdp_offload_paramsfalsezvDecides Whether (true|false) to offload parameters and gradients to CPU. (useful only when `use_fsdp` flag is passed).z--fsdp_min_num_paramsgחAzlFSDP's minimum number of parameters for Default Auto Wrapping. (useful only when `use_fsdp` flag is passed).z--fsdp_sharding_strategy FULL_SHARDzGFSDP's Sharding Strategy. (useful only when `use_fsdp` flag is passed).z--fsdp_auto_wrap_policyzFFSDP's auto wrap policy. (useful only when `use_fsdp` flag is passed).z$--fsdp_transformer_layer_cls_to_wrapzTransformer layer class name (case-sensitive) to wrap ,e.g, `BertLayer`, `GPTJBlock`, `T5Block` .... (useful only when `use_fsdp` flag is passed).z--fsdp_backward_prefetchzNFSDP's backward prefetch policy. (useful only when `use_fsdp` flag is passed).z--fsdp_state_dict_typezEFSDP's state dict type. (useful only when `use_fsdp` flag is passed).z--fsdp_forward_prefetchzIf True, then FSDP explicitly prefetches the next upcoming all-gather while executing in the forward pass (useful only when `use_fsdp` flag is passed).z--fsdp_use_orig_paramstruezIf True, allows non-uniform `requires_grad` during init, which means support for interspersed frozen and trainable paramteres. (useful only when `use_fsdp` flag is passed).z --fsdp_cpu_ram_efficient_loadinguIf True, only the first process loads the pretrained model checkoint while all other processes have empty weights. Only applicable for 🤗 Transformers. When using this, `--fsdp_sync_module_states` needs to True. (useful only when `use_fsdp` flag is passed).z--fsdp_sync_module_stateszIf True, each individually wrapped FSDP unit will broadcast module parameters from rank 0. (useful only when `use_fsdp` flag is passed).z--fsdp_activation_checkpointingzDecides Whether (true|false) intermediate activations are freed during the forward pass, and a checkpoint is left as a placeholder. (useful only when `use_fsdp` flag is passed).r,z!Arguments related to Megatron-LM.z--megatron_lm_tp_degreerPzbMegatron-LM's Tensor Parallelism (TP) degree. (useful only when `use_megatron_lm` flag is passed).z--megatron_lm_pp_degreezdMegatron-LM's Pipeline Parallelism (PP) degree. (useful only when `use_megatron_lm` flag is passed).z--megatron_lm_num_micro_batchesznMegatron-LM's number of micro batches when PP degree > 1. (useful only when `use_megatron_lm` flag is passed).z"--megatron_lm_sequence_parallelismzDecides Whether (true|false) to enable Sequence Parallelism when TP degree > 1. (useful only when `use_megatron_lm` flag is passed).z#--megatron_lm_recompute_activationszDecides Whether (true|false) to enable Selective Activation Recomputation. (useful only when `use_megatron_lm` flag is passed).z'--megatron_lm_use_distributed_optimizerzDecides Whether (true|false) to use distributed optimizer which shards optimizer state and gradients across Data Pralellel (DP) ranks. (useful only when `use_megatron_lm` flag is passed).z--megatron_lm_gradient_clippingg?zMegatron-LM's gradient clipping value based on global L2 Norm (0 to disable). (useful only when `use_megatron_lm` flag is passed).r-zDArguments related to FP8 training (requires `--mixed_precision=fp8`)r5temsampzIChoose a backend to train with FP8 (te: TransformerEngine, msamp: MS-AMP)z--fp8_use_autocast_during_evalzWhether to use FP8 autocast during eval mode (useful only when `--fp8_backend=te` is passed). Generally better metrics are found when this is not passed.z --fp8_marginz[The margin to use for the gradient scaling (useful only when `--fp8_backend=te` is passed).z--fp8_intervalzsThe interval to use for how often the scaling factor is recomputed (useful only when `--fp8_backend=te` is passed).z --fp8_formatE4M3HYBRIDzUThe format to use for the FP8 recipe (useful only when `--fp8_backend=te` is passed).z--fp8_amax_history_leniztThe length of the history to use for the scaling factor computation (useful only when `--fp8_backend=te` is passed).z--fp8_amax_compute_algo most_recentmaxziThe algorithm to use for the scaling factor computation. (useful only when `--fp8_backend=te` is passed).z--fp8_override_linear_precisionclttt|dS)N,)tupler`r split)xs r<z'launch_command_parser..s"uSaggcll;;<<r>)FFFzWhether or not to execute `fprop`, `dgrad`, and `wgrad` GEMMS in higher precision. Should be passed in a comma-seperated string of booleans (useful only when `--fp8_backend=te` is passed).z--fp8_opt_levelO2O1z{What level of 8-bit collective communication should be used with MS-AMP (useful only when `--fp8_backend=msamp` is passed).z AWS ArgumentszArguments related to AWS.z--aws_access_key_idzFThe AWS_ACCESS_KEY_ID used to launch the Amazon SageMaker training jobz--aws_secret_access_keyzKThe AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job.z--debugzLWhether to print out the torch.distributed stack trace when something fails.training_scriptznThe full path to the script to be launched in parallel, followed by all the arguments for the training script.)rrez MPI Argumentsz)Arguments related to mpirun for Multi-CPUz--mpirun_hostfilezLocation for a hostfile for using Accelerate to launch a multi-CPU training job with mpirun. This will get passed to the MPI --hostfile or -f parameter, depending on which MPI program is installed.z --mpirun_cclzcThe number of oneCCL worker threads when using Accelerate to launch multi-CPU training with mpirun.training_script_argsz!Arguments of the training script.)nargsre)func) add_parserr@rriadd_argument_groupstrintrr"floatrc REMAINDER set_defaultslaunch_command) subparsersryparser hardware_args resource_args paradigm_argsdistributed_argstpu_argsdeepspeed_args fsdp_argsmegatron_lm_argsfp8_argsaws_args mpirun_argss r<launch_command_parserrs?pK&& +Ecv'  & '#/     hv<^___  U   m --&(ZM|:l L  |:m [ --&(jM ---[ T8ssD7l'  [   x  ==_====2  " C   j   7 --%'dM (   #   *   S 001CEuvv!! f" !! ` " !!sD7r"!!#t:d"!!  I " !!   C "!!  E "!!  1 "!!  U " !!  Y " !!  F " !!  E "   G   L((0KLLH   9     N    V    P    ^   "  c   W../DFghhN!  %    /  $ 2    2  ' 2  # 2  ' /   1   _  " `    O   &  M   &  M   (  \   ) >  ))*:ct|\}}tj||}||jdkr8|jstj|j|tjddSdS)Nenvr returncodecmdrP) r subprocessPopenwaitrquietCalledProcessErrorr\exit)rIr current_envprocesss r<simple_launcherrs6t<<Cs 444G LLNNNQz /7;MSVWWW W HQKKKKK r>c~ddlmcm}t|}t s6d}d}d|vrd|d<d}d|vrd|d<d}|rt |t|dd}t|| d |j d |j g}td i|5 ||nb#t$rUtrC|rAt}|d |t$gd nYnwxYwddddS#1swxYwYdS)Nr{Using RTX 4000 series which doesn't support faster communication speedups. Ensuring P2P and IB communications are disabled.FNCCL_P2P_DISABLE1TNCCL_IB_DISABLEdebug--training_script--training_script_argsE [bold red]Using --debug, `torch.distributed` Stack Trace:[/bold red]suppress show_localsrR)torch.distributed.run distributedrunrrloggerwarninggetattrr get_args_parserrrr Exceptionrr#printprint_exception__file__)rI distrib_runrmessagewarnrconsoles r<multi_gpu_launcherrs/////////'--K $ & & $P [ 0 0.1K* +D K / /-0K) *D  $ NN7 # # # D'5 ) )E  ##%% d24LdNgh  D  ) )[ ) )    OOD ! ! ! !    "" u %-- fggg''('OOOOPO                    s7+D2-CD2AD"D2!D""D22D69D6cddlmcm}tst dddlm}t|\}}ts6d}d}d|vrd|d<d}d |vrd|d <d}|rt ||j d kr|j td krt|d 5}t|}t!|d kr||dddn #1swxYwYt%j|| } | | jdkr8|jst%j| j| t1jd dSdSt5|dd} t7||d|jd|jg}t?di|5 ||nb#t@$rUtCrC| rAtE} | #d| $tJgdnYnwxYwddddS#1swxYwYdS)NrzSDeepSpeed is not installed => run `pip3 install deepspeed` or build it from source.)DEEPSPEED_ENVIRONMENT_NAMErFrrTrrParrrrrrrrR)&rrrr ImportErrordeepspeed.launcher.runnerrrrrr num_machinesdeepspeed_multinode_launcherr!openrr^ writelinesrrrrrrr\rrr rrrrrrr#rrr) rIrrrrrrfvalid_env_itemsrrrs r<deepspeed_launcherr#sB///////// ! # #IopppHHHHHH066C $ & & $P [ 0 0.1K* +D K / /-0K) *D  $ NN7 # # # 1!BFcdeFf!f!f ,c 2 2 .a;KHHO?##a'' _--- . . . . . . . . . . . . . . . "3K888    " ":  3w?QWZ[[[[ # " gu--   ' ' ) ) $"68PRVRk l    - - - -   %%%%   $&&5)mmGMM"jkkk++hZU+SSSSTS                    sI48C88C<?C<IGIAH74I6H77II I cddlmcm}|jrt dt |i\}}|jr|j}n^t|j}tj t|j |j}t!j|}t%||jst d|jd|jg|jzt_t/||j}t1di|5|t5|d|jddddS#1swxYwYdS)Nrz,--no_python cannot be used with TPU launcherz2Your training script should have a function named zE, or you should pass a different value to `--main_training_function`.rR)rInprocs))torch_xla.distributed.xla_multiprocessingrxla_multiprocessing no_python ValueErrorrmodulerrr\pathrrparentresolvestem importlib import_modulehasattrmain_training_functionrrr]rrspawnr num_processes)rIxmprmod_name script_pathmod main_functions r< tpu_launcherr Xs;;;;;;;;; ~IGHHH#D"--D+ {$'4/00  K.668899:::#  !( + +C 33 4 4  =A\ = = =    ~ 99CHC!<==M  ) )[ ) )WW "=114CU VVVWWWWWWWWWWWWWWWWWWs+EEEcddlm}i}t||d\}}t|dd}|j}|j}t ||d|jddd g}|j rd g}ng}|d dd d ddddddt|j dt|j |g|zz }||_ d}t|D]:} | dr#t|| } | dkr | || d| dz };|dkrt!d|d|D|_|jd ||dS#t*$rUt-rE|rCt/} | d| t4gdYdSwxYw)Nr)xla_distTrFrz --positionalrmz--restart-tpuvm-pod-serversudozaccelerate-launchrrrrrrrrdocker_z="z" zgDocker containers are not supported for TPU pod launcher currently, please remove the following flags: c"g|] \}}|d| S)=rR)rUkvs r<rWz$tpu_pod_launcher..s&???41aqJJ1JJ???r>zACCELERATE_IN_TPU_POD=1zF [bold red]Using --debug, `torch_xla.xla_dist` Stack Trace:[/bold red]r)torch_xla.distributedrrrrrr rtpu_name tpu_use_sudorrr positionalvarsr9rrorrresolve_and_executerrr#rrr) rIrrrrrnew_argsnew_cmd bad_flagsrVvaluers r<tpu_pod_launcherr wsh......K#D+t<<D+ D'5 ) )E*O4 h&&((7DM>SUWs*tH (     D " D '(( G""HIH~~22 >>) $ $ 2Hc**E{{u011u1111 B CxA C C   @?;+<+<+>+>???HL L1222$$X.....     5 !mmG MMc d d d  # #hZU # K K K K K K  sE00AG Gsagemaker_configctstd|js|jrt dddlm}t||\}}|di|}||td|j dS)NzsPlease install sagemaker to be able to launch training on Amazon SageMaker with `pip install accelerate[sagemaker]`z`SageMaker requires a python training script file and cannot be used with --module or --no_pythonr) HuggingFace)inputsz!You can find your model data at: rR) rrrrrsagemaker.huggingfacer#rfitr model_data)r!rIr#sagemaker_inputshuggingface_estimators r<sagemaker_launcherr*s ! # #  B    { dn  n   211111:;KTRRD 'K//$//%5666 P.C.N P PQQQQQr>c :t|j|j|j|j|jgdkrt d|jr!|j|jdkrt dd}g}d}|j-tj tr|jst|j}|js|js|js|js|js|js|jt"jk|_|jt"jt"jt"jt"jt"jfvrdnd|_|jt"jk|_|jt"jk|_|jt"jk|_|jr|jnd|_|j|j |j|_nd|_|jr|j |j|_t;|jddkr,|jdkr!|jr|jdkrt d |jt@j!krM|j"#D]2\}}tI|tJr|j&D]}tO|||j&||j(D])}|}d |vrd |z}tO|||j(|*|j)D]}tO|||j)||j*D]}tO|||j*||j+D]}tO|||j+||j,D]}tO|||j,| |d vr"t[||dtO|||4|j.s |j.|_.|j/s|j/d |_/n|j/|_/d}n|j0s%|j1r/tdj34rtkdd}ntmd}|j/dkr(|s&|jrtodst d|j8d |_8|jdkrt dn|j|j1r2tsr$tdj3:|_ntwr$tdj<:|_nt{r$tdj>:|_nUtr$tdj@:|_n#tdjA:|_|Bd|jd|j.d|_.|js|jdkr|j1r0tsr"tdj3:dkstwr"tdj<:dkst{r"tdj>:dksRtr"tdj@:dks"tdjA:dkr|Bdd|_|j|Bdd|_|j/|Bdd |_/t|ds |j|_0|j8|Bdd |_8|j.rt.d|dup|duo|jt@jEk} | r|jFtdgd|_F|j0r|jdkrtdgddkrtgdtt|j|jz d} ttjKd | z } | dkr%| |_F|Bd!|jFd"t|r9d#} | d$M|z } | d%z } tN| |||fS)&NrPzcYou can only use one of `--cpu`, `--multi_gpu`, `--tpu`, `--use_deepspeed`, `--use_fsdp` at a time.r6z:You need to use at least 2 processes to use `--multi_gpu`.FTallrzLess than two GPU ids were configured and tried to run on on multiple GPUs. Please ensure at least two are specified for `--gpu_ids`, or use `--gpu_ids='all'`.fsdpfsdp_)compute_environmentmixed_precisiondistributed_typerz>=z1.10r) check_is_tpuzEbf16 mixed precision requires PyTorch >= 1.10 and a supported device.zFYou need to manually pass in `--num_processes` using this config yaml.z* `--num_processes` was set to a value of ``zx More than one GPU was found, enabling multi-GPU training. If this was unintended please pass in `--num_processes=1`.z+ `--num_machines` was set to a value of `1`z1 `--mixed_precision` was set to a value of `'no'`use_cpuz0 `--dynamo_backend` was set to a value of `'no'`zRRunning script in debug mode, expect distributed operations to be slightly slower.OMP_NUM_THREADSr)MPI_LOCALNRANKSOMPI_COMM_WORLD_LOCAL_SIZEMV2_COMM_WORLD_LOCAL_SIZE)logicalz- `--num_cpu_threads_per_process` was set to `z9` to improve out-of-box performance when training on CPUsz[The following values were not passed to `accelerate launch` and had defaults used instead:  zh To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.)Osumr.cpur/r0r1rr config_fileosrisfilerrrr2r1r DEEPSPEED MULTI_GPU MULTI_NPU MULTI_MLU MULTI_MUSA MULTI_XPUXLAFSDP MEGATRON_LMgpu_idsrr^rr/r LOCAL_MACHINE__dict__ro isinstancedictdeepspeed_configsetattr fsdp_configmegatron_lm_config dynamo_config ipex_config mpirun_configrrr0r5use_xputorchxpu is_availablerrrdynamo_backendr device_countrmlurmusarnpucudarrrAMAZON_SAGEMAKERnum_cpu_threads_per_processr rrpsutil cpu_countanyjoinr) rIdefaultswarnedmp_from_config_flagnameattrr arg_to_set native_ampis_aws_env_disabled local_sizethreads_per_processrs r<_validate_launch_commandrps4  DNDHdh0BDM RSSVWWW q    ~W4-9@RUV@V@VUVVVH F #rw~~6I'J'J#SWS[#()9:: SH S( S&  S M  S (  S"*!:o>W!WD ,#-#-#-#.#-  N 0O4GGDH$59MMDM#+#<@[#[D ?Cx#R8#;#;UD < +'/ $ > 6d/7 ( 5D  t|!!#&& ' '! + +1F1FDN1F_c_ptu_u_uf   '+=+K K K&/5577 . . ddD))%6GGa)B1)EFFFF%1KK%& !33)0:)=Jj(2Fq2IJJJJ%8IIa)DQ)GHHHH%3DDa)?)BCCCC%1BBa)=a)@AAAA%3DDa)?)BCCCC ^^^dD119D$---z (!DJ# j'/'+$$'/'?$&*##| 5  51G1G1I1I 5-dF;; .t44 $.."//&<$&O&O&O/!!hiii   &"&D    # #eff f $   %| ? 0 2 2 ?%*Y%;%;%=%=""!## ?%*Y%;%;%=%="""$$ ?%*Z%<%<%>%>""!## ?%*Y%;%;%=%=""%*Z%<%<%>%>" MM]HZ]]] ^ ^ ^ : DJ ""Q&&'"2"4"4'9>9O9O9Q9QTU9U9U$&&:V+09+A+A+C+Ca+G+G%'',H,1J,C,C,E,E,I,I$&&-J+09+A+A+C+Ca+G+GJ++--11 MMQ   "DN   $ MMH I I I !D    ' MMN O O O#'D tY'' $8DL   & MMM N N N"&D  zk ijjj"d*d!=ASAd!d t?G+;=N>??CCJ#&f&6u&E&E&E &R"S"S "Q&&3F0 aTEeaaa 6{{ p499V$$$ w  w . ..r>ct|\}}}|jr|js{|r&t|jng|_|r|jdd|j|_t|dS|j r|jst|dS|j r|jst|dS|j r|jst|dS|jr0|js)|jrt!|dSt#|dS|'|jt&jkrt+||dSt-|dS)Nr0r)rpr0r=r_rOrT'deepspeed_fields_from_accelerate_configrrerr1rr2r.r/rr r r/r r`r*r)rIrfrhs r<rrys*B4*H*H'D(' $(ai7qtHBTBe"e"e8T*****r>cht}|}t|dSN)r parse_argsr)rrIs r<mainrvs2 " $ $F     D4r>__main__rt)Grcrloggingr?rr\pathlibrrbrWaccelerate.commands.configrr&accelerate.commands.config.config_argsr'accelerate.commands.config.config_utilsraccelerate.commands.utilsraccelerate.stater accelerate.utilsr r r r rrrrrrrrrrrrrrrrrrr accelerate.utils.constantsr!r"richr# rich.loggingr$FORMAT basicConfig getLoggerrrrrSr= HelpFormatterr@rrrrr r r*rprrvrRr>r<rs"   QQQQQQQQBBBBBBCCCCCC::::::------2YXXXXXXXQ      (((((( FGvvPPPP  8 $ $$ * ." ,,,/////(0///de e e e P      F222jWWW>888vRRRRR(p/p/p/f4  zDFFFFFr>