g/ddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlZddlmZddlmZdd lmZdd lmZdd lmZd d lmZd dlmZd dlmZm Z m!Z!d dl"m#Z#d dl$m%Z%d dl&m'Z'ee(Z)e!r ddl*m+cm,Z-dZ.d*de/de/fdZ0dZ1de2fdZ3d+de/de/fdZ4edZ5ed Z6d!Z7d"Z8d,d#e9d$e/fd%Z:d&Z;d'ZdS)-N)contextmanager)partialreduce) MethodType) OrderedDict)Version) save_file)write_basic_config) get_logger) PartialState)FSDP_PYTORCH_VERSION)DistributedType)is_deepspeed_availableis_torch_distributed_availableis_torch_xla_available)id_tensor_storage) convert_model)is_torch_versionctddsttdsdSt|tjjjS)zD Check whether the module was compiled with torch.compile() =)FullyShardedDataParallelct|dr|j}n|}|D]\}}t||| |S)Nr)rrnamed_childrensetattr)runwrapped_modulenamechild_recursive_unwraps r r.z6extract_model_from_parallel.._recursive_unwrap^s|vx(( *#4#4V]#C#C  #) />>@@ J J e($0A0A%0H0HIIII# #r"_original_forwardN __wrapped__ _converted_to_transformer_engineF)to_transformer_engine)rnnparallelDistributedDataParallel DataParallelr! _orig_modr deepspeedr&rrr2torch.distributed.fsdp.fully_sharded_data_parallelr'rrforward__dict__poprr0rgetattrr) modelr#r$options is_compiledcompiled_modelr&FSDPr:original_forwardr.s @r extract_model_from_parallelrD8s x 8%(:OPG$U++K &------O%%2338V8X8XggggggD7 UG $ $  UG $ $) $ $ $ $ $"!%((  >- >--.A4HH  ''=11 !-...'=11 'w66EM 5 %u = = = =#(  Lr"cFtdS)a  Introduces a blocking point in the script, making sure all processes have reached this point before continuing. Make sure all processes will reach this instruction otherwise one of your processes will hang forever. N)r wait_for_everyoner"r rFrFs NN$$&&&&&r" state_dictctjt}D]B\}}t |t s(|t ||Cd|D}t}| D]=}fd|D}| |dd|ddD]}|=>t|dkrt d|ddDS) z Cleans the state dictionary from a model and removes tensor aliasing if present. Args: state_dict (`dict`): The state dictionary from a model c@i|]\}}t|dk||S)r)len).0ptrnamess r z4clean_state_dict_for_safetensors..s)OOO*#uE Q3r"cg|]}|v| SrGrG)rLr,rHs r z4clean_state_dict_for_safetensors..s#DDD1C1Ct1C1C1Cr"rNrzRemoved shared tensor zk while saving. This should be OK, but check by verifying that you don't receive any warning while reloadingcvi|]6\}}|t|tjr|n|7SrG)rrTensor contiguous)rLkvs r rOz4clean_state_dict_for_safetensors..s<iiidaQR!z!U\'B'BIQ\\^^^iiir") collections defaultdictlistitemsrstrrappendsetvaluesupdaterKloggerwarning)rHptrsr,tensor shared_ptrs warn_namesrN found_namess` r clean_state_dict_for_safetensorsrgst  "4 ( (D"((**99 f&#&& 9 "6** + 2 24 8 8 8PO OOOKJ##%% ! ! EDDDDDD +abb/***O ! !D4   ! : ]Z ] ] ]   jiV`VfVfVhVhiiiJ r"save_on_each_nodesafe_serializationctjtjkrt j|}|r=t tddi}t|trt|}n tj }tj r|s|||dStjr|r|||dSdSdS)a Save the data to disk. Use in place of `torch.save()`. Args: obj: The data to save f: The file (or file-like object) to use to save the data save_on_each_node (`bool`, *optional*, defaults to `False`): Whether to only save on the global main process safe_serialization (`bool`, *optional*, defaults to `False`): Whether to save `obj` using `safetensors` or the traditional PyTorch way (that uses `pickle`). formatpt)metadataN)r distributed_typerXLAxm_maybe_convert_to_cpursafe_save_filerrrgrsaveis_main_processis_local_main_process)objfrhri save_funcs r rsrss$~~&/*===&s++Nh5EFFF c; ' ' 82377CJ ~~%.? #q  -2C #qr"c#Ktj}tj dVtjtj|dS#tjtj|wxYw)a A context manager that will temporarily clear environment variables. When this context exits, the previous environment variables will be back. Example: ```python >>> import os >>> from accelerate.utils import clear_environment >>> os.environ["FOO"] = "bar" >>> with clear_environment(): ... print(os.environ) ... os.environ["FOO"] = "new_bar" ... print(os.environ["FOO"]) {} new_bar >>> print(os.environ["FOO"]) bar ``` N)osenvironcopyclearr_)_old_os_environs r clear_environmentrs2joo''OJ+   /*****  /****s B?Cc+HKi}|D]X\}}|}|tjvrtj|||<t |tj|<Y dV|D]P}|}||vr||tj|<0tj|dQdS#|D]P}|}||vr||tj|<0tj|dQwxYw)a A context manager that will add each keyword argument passed to `os.environ` and remove them when exiting. Will convert the values in `kwargs` to strings and upper-case all the keys. Example: ```python >>> import os >>> from accelerate.utils import patch_environment >>> with patch_environment(FOO="bar"): ... print(os.environ["FOO"]) # prints "bar" >>> print(os.environ["FOO"]) # raises KeyError ``` N)rZupperrzr{r[r<)kwargs existing_varskeyvalues r patch_environmentrs)$Mllnn%% Uiikk "*  !#CM# e** 3 *  * *C))++Cm##"/"4 3 sD))))  * *6 * *C))++Cm##"/"4 3 sD))))  *s 3C AD!ct|ds!t|dst|d|}t|dr|jSt|dr|jSt |S)z( Gets a pretty name from `obj`. __qualname____name__ __class__)rr=rrr[)rvs r get_pretty_namersx 3 ' '-Z0H0H-c;,,sN## sJ| s88Or"c|D]F\}}t|tr'||i}t ||A|||<G|S)z Recursively merges two dictionaries. Args: source (`dict`): The dictionary to merge into `destination`. destination (`dict`): The dictionary to merge `source` into. )rZrdict setdefault merge_dicts)source destinationrrnodes r rr'slllnn%% U eT " " %))#r22D t $ $ $ $$K   r"portreturnc|d}tjtjtj5}|d|fdkcdddS#1swxYwYdS)z Checks if a port is in use on `localhost`. Useful for checking if multiple `accelerate launch` commands have been run and need to see if the port is already in use. Ni  D [F X0$,??NAwKw'+.... sw s s\g s s s  sd33333 /.r"attrcVd}t||g|dzS)z Recursive `getattr`. Args: obj: A class instance holding the attribute. attr (`str`): The attribute that is to be retrieved, e.g. 'attribute1.attribute2'. c"t||SN)r=)rvrs r _getattrz#recursive_getattr.._getattrkssD!!!r".)rr)rvrrs r recursive_getattrr`s4""" (SEDJJsOO3 4 44r")TF)FFr)?rWrzrrr contextlibr functoolsrrtypesrtypingrrpackaging.versionrsafetensors.torchr rrcommands.config.defaultr loggingr stater constantsr dataclassesrimportsrrrmodelingrtransformer_enginerversionsrrr`torch_xla.core.xla_modelcore xla_modelrpr!boolrDrFrrgrsrrrrintrrrr[rrGr"r rs  %%%%%%%%%%%%%% %%%%%%999999888888 ++++++((((((cccccccccc''''''------&&&&&& H  *)))))))))HHHEE$ERVEEEEP ' ' '     FDdD+++D!*!*!*H   $6666666"""444$5555555r"