g(ddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z ddl mZmZddlZddlmZejeZdefdZd efd Zd Zdd ZddZded eefdZdZdZ dZ!dZ"dZ#eGddZ$d e$fdZ%d dedee&d dfdZ'e d dedee&d dfdZ(dS)!N) dataclassfield) lru_cache)which)ListOptional)parse current_envcHgd}g}|D]\tfd|DrBtdkr/tdkr|ddctddd|S)a Verifies that all keys and values in `current_env` do not contain illegal keys or values, and returns a list of strings as the result. Example: ```python >>> from accelerate.utils.environment import verify_env >>> env = {"ACCELERATE_DEBUG_MODE": "1", "BAD_ENV_NAME": ">> valid_env_items = verify_env(env) >>> print(valid_env_items) ["ACCELERATE_DEBUG_MODE=1 ", "OTHER_ENV=2 "] ``` ); <> c3&K|] }|zvV dSN).0charkeyvalues X/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/utils/environment.py z0convert_dict_to_env_variables..3s,EETtC%K(EEEEEE=r zWARNING: Skipping z7 as it contains forbidden characters or missing values.)itemsalllenappendloggerwarning)r forbidden_charsvalid_env_itemsrrs @@rconvert_dict_to_env_variablesr%!s100OO!''))vv U EEEEE_EEE E E v#c((VW--\_`e\f\fjk\k\k  " "c#5#5E#5#5#5 6 6 6 6 NNtttettt u u u u rreturncf|}|dvrdS|dvrdStd|)z Converts a string representation of truth to `True` (1) or `False` (0). True values are `y`, `yes`, `t`, `true`, `on`, and `1`; False value are `n`, `no`, `f`, `false`, `off`, and `0`; )yyesttrueon1r)nnoffalseoff0rzinvalid truth value )lower ValueError)rs r str_to_boolr6:sJ KKMME 444q 7 7 7q777888rc~|D]9}ttj|d}|dkr|cS:|S)zQReturns the first positive env value found in the `env_keys` list or the default.r)intosenvironget)env_keysdefaultevals rget_int_from_envrAIsJ "*..B''(( !88JJJ  NrFctj|t|}t |dkS)zJReturns truthy value for `key` from the env if available else the default.r)r:r;r<strr6rr>rs rparse_flag_from_envrERs0 JNN3G - -E u   ""rr/c`tj|t|}|Sr)r:r;r<rCrDs rparse_choice_from_envrGXs" JNN3G - -E Lr library_namescd|DS)zk Checks if any of `library_names` are imported in the environment. Will return any names that are. cRg|]$}|tjv"|%Sr)sysmoduleskeys)rlib_names r z-are_libraries_initialized..as0 U U Uh#+BRBRBTBT6T6TH6T6T6Trr)rHs rare_libraries_initializedrP]s V U] U U UUrctjdkr'td}|tjdd}nd}|S)zC Returns the right nvidia-smi command based on the system. Windowsz nvidia-smiN systemdrivez6\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe)platformsystemrr:r;)commands r _nvidia_smirWdsKI%% %% ?M2nnnG Nrctjtddgd}|}|t j}t|}d|D}||fS)z Gets GPU count and names using `nvidia-smi` instead of torch to not initialize CUDA. Largely based on the `gputil` library. z--query-gpu=count,name--format=csv,noheaderTuniversal_newlineschg|]/}|dd0S),r)splitstrip)rgpus rrOz get_gpu_info..s3;;;s3"((**;;;r) subprocess check_outputrWr_r^r:linesepr)outputgpus gpu_count gpu_namess r get_gpu_inforhssy $ 02IJ_cF\\^^F << # #DD I;;d;;;I i rctjtddgd}|}|t jdS)z^ Returns the driver version In the case of multiple GPUs, will return the first. z--query-gpu=driver_versionrYTrZr)rarbrWr_r^r:rc)rds rget_driver_versionrjsV  $ 46MNcgF\\^^F << # #A &&rc t\}}dh|dkrOtfd|Dr4d}t}t|t|krdSdSn#t$rYnwxYwdS)z Checks if the devices being used have issues with P2P and IB communications, namely any consumer GPU hardware after the 3090. Noteably uses `nvidia-smi` instead of torch to not initialize CUDA. zRTX 40rc3*K|] }D]}||vV dSrr)r device_nameunsupported_deviceunsupported_devicess rrz,check_cuda_p2p_ib_support..sS*='#k1rz 550.40.07FT)rhanyrjr Exception) device_names device_countacceptable_driver_versioncurrent_driver_versionros @rcheck_cuda_p2p_ib_supportrvs %1^^" l'j !  #/  -8));)=)=&/0059R3S3SSS 5t      4sA%A-- A:9A:cJtj}|dkS)zs Checks if all the current GPUs available support FP8. Notably must initialize `torch.cuda` to check. ) )torchcudaget_device_capability)cuda_device_capacitys rcheck_fp8_capabilityr~s# !:;;== 6 ))rceZdZUdZedddiZeed<edddiZeed <eddd iZ eed <eddd iZ eed <dS)CPUInformationag Stores information about the CPU in a distributed environment. It contains the following attributes: - rank: The rank of the current process. - world_size: The total number of processes in the world. - local_rank: The rank of the current process on the local node. - local_world_size: The total number of processes on the local node. rhelpz The rank of the current process.)r>metadatarankrz+The total number of processes in the world. world_sizez2The rank of the current process on the local node. local_rankz0The total number of processes on the local node.local_world_sizeN) __name__ __module__ __qualname____doc__rrr9__annotations__rrrrrrrrsa63U*VWWWD#WWWeA9f0ghhhJhhheA9m0noooJooo!E!v?q6rssscsssssrrci}tgdd|d<tgdd|d<tgdd|d<tgd d|d <td i|S) z Returns various information about the environment in relation to CPU distributed training as a `CPUInformation` dataclass. )RANKPMI_RANKOMPI_COMM_WORLD_RANKMV2_COMM_WORLD_RANKrr) WORLD_SIZEPMI_SIZEOMPI_COMM_WORLD_SIZEMV2_COMM_WORLD_SIZErr) LOCAL_RANKMPI_LOCALRANKIDOMPI_COMM_WORLD_LOCAL_RANKMV2_COMM_WORLD_LOCAL_RANKr)LOCAL_WORLD_SIZEMPI_LOCALNRANKSOMPI_COMM_WORLD_LOCAL_SIZEMV2_COMM_WORLD_LOCAL_SIZErr)rAr) informations rget_cpu_distributed_informationrs K*+n+n+npqrrK 0QQQST!!K !1dddfg!!K '7jjj ''K"#  ( (K ( ((rlocal_process_indexverbosec|tdd}tjr,ddlm}|st dddl}|tj tj dz }| |}d}|||D] }|d |} d |D}|d t!|D} tjd| |rHtjd} t&d t+| d |d| dSdSdS)a Overrides whatever NUMA affinity is set for the current process. This is very taxing and requires recalculating the affinity to set, ideally you should use `utils.environment.set_numa_affinity` instead. Args: local_process_index (int): The index of the current process on the current server. verbose (bool, *optional*): Whether to log out the assignment of each CPU. If `ACCELERATE_DEBUG_MODE` is enabled, will default to True. NACCELERATE_DEBUG_MODEFr)is_pynvml_availablez_To set CPU affinity on CUDA GPUs the `pynvml` package must be available. (`pip install pynvml`)@064bc,g|]}t|Sr)r9)rxs rrOz*override_numa_affinity..s999AQ999rc$g|] \}}|dk |S)rr)rir?s rrOz*override_numa_affinity..s!LLLAQ!VV1VVVrz Assigning z cpu cores to process z: )rErzr{ is_availableaccelerate.utilsr ImportErrorpynvmlnvmlInitmathceilr: cpu_countnvmlDeviceGetHandleByIndexnvmlDeviceGetCpuAffinityreverse enumeratesched_setaffinitysched_getaffinityr!infor) rrrnvml num_elementshandleaffinity_stringj affinity_listaffinity_to_set cpu_coress roverride_numa_affinityrs%&=uEE z  o888888""$$ q   y"!455 001DEE..v|DD ; ;A!"::::OO99999 LL=)A)ALLL Q000  o,Q//I KKmS^^mmK^mmbkmm n n n n n/oo* o orc(t||dS)a2 Assigns the current process to a specific NUMA node. Ideally most efficient when having at least 2 cpus per node. This result is cached between calls. If you want to override it, please use `accelerate.utils.environment.override_numa_afifnity`. Args: local_process_index (int): The index of the current process on the current server. verbose (bool, *optional*): Whether to print the new cpu cores assignment for each process. If `ACCELERATE_DEBUG_MODE` is enabled, will default to True. rrN)rrs rset_numa_affinityrs/BGTTTTTTr)F)r/r))loggingrr:rTrarK dataclassesrr functoolsrshutilrtypingrrrzpackaging.versionr getLoggerrr!dictr%r9r6rArErGrCrPrWrhrjrvr~rrboolrrrrrrs  ((((((((!!!!!!!! ######  8 $ $t2 9# 9 9 9 9####  VcVd3iVVVV      $ ' ' '8***  t t t t t t t  t)))))($o$o$ohtn$oX\$o$o$o$oN UU3U$USWUUU UUUr