gDR ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl mZmZmZmZmZmZddlZddlmZddlmZddlmZmZddlmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,m-Z-m.Z.dd l/m0Z0m/Z/dd l1m2Z2m3Z3e$drddl4Z4e!drddl5Z5e#drddl6Z6ddl7m8Z8ddl9m:Z;dZ<ej=e>Z?dZ@dZAdeeBeCffdZDdejEfdZFdejGdeejHeBeBffdZI d_dejJdeCdeeBeCejHfdeejGdeeeCejEfd eejKd!eeeBeejHejGfffd"ZL d`dejJd$eMd%eMd&eMfd'ZNdadejJd%eMfd(ZOGd)d*ePZQd+ejJfd,ZRd-ZSd.ZT dbdejjJd0eeejjJd1eCd2eMfd3ZUdcdejjJd2eMfd4ZVd+ejjJfd5ZWd6ZXdeeCejHfdejEfd7ZY ddd+ejJdeeeCejHfd8eeeCeeCejHffd9eMfd:ZZ ded+ejJdeeeCejHfd8eeeCeeCejHfffd;Z[deeCfd?Z\dfd@eeeeBeCfeeBeCfffdAZ]dgdBeeCeeBeCejHffdCeCfdDZ^dEZ_dFZ` dhd+ejJd@eeeeBeCfeeBeCffd>eeeCdeeeCejEfd8eeeCeeCejHffdGeMf dHZad+ejjJfdIZb did+ejJd@eeeeBeCfeeBeCffd>eeeCdeeeCejEfd8eeeCeeCejEffdJeMdKeMdLeMfdMZcd+ejJdBeeCeeBeCejHfffdNZddfdOZed+ejJfdPZf djdejJdCeCdReeCeeCejgffdSeeBeCejHffdTZh dkd+ejJdUeeCejifdBeeeCeeBeCejHffdVeeeCejifdeeeCejEfdWeMdLeMdXeeCdYeMdZeMfd[Zjdld\eMd]efd^ZkdS)mN) OrderedDict defaultdict)DictListOptionalSetTupleUnion)AcceleratorState)SAFE_WEIGHTS_NAME WEIGHTS_NAME)AutocastKwargs CustomDtypeDistributedType)is_mlu_availableis_mps_availableis_musa_availableis_npu_availableis_peft_availableis_torch_xla_availableis_xpu_available)clear_device_cacheget_xpu_available_memory)load_offloaded_weightoffload_weightsave_offload_index)is_tqdm_availabletqdm)compare_versionsis_torch_versionF) check_device) safe_open) load_filezpytorch_model.bin.index.jsoncddlm}trddlm}tot |||S)Nr )extract_model_from_parallelr) PeftModel)otherr'rpeftr( isinstance)modelr'r(s U/var/www/html/ai-engine/env/lib/python3.11/site-packages/accelerate/utils/modeling.py is_peft_modelr.Cs\222222#""""""    \:.I.I%.P.PR[#\#\\c|j|jkrdS|jdkr|jtjdd}|jdkr|jtjdd}||kS)ar Utility method to check if two `torch` devices are similar. When dealing with CUDA devices, torch throws `False` for `torch.device("cuda") == torch.device("cuda:0")` whereas they should be the same Args: first_device (`torch.device`): First device to check second_device (`torch.device`): Second device to check FcudaNrindex)typer3torchdevice) first_device second_devices r-check_device_samer9Ls}M...uF""|'9'A|F!444 V## (;(C V1555 = ((r/sizecd}d|d} t|tr|}n0|dr)tt |dddz}n|dr)tt |ddd z}n|d r)tt |ddd z}n@|d rDtt |dd dz}|dr|dzn|}n|drDtt |dd dz}|dr|dzn|}nj|drCtt |dd dz}|dr|dzn|}n#t $rt |wxYw|dkrt ||S)a( Converts a size expressed as a string with digits an unit (like `"5MB"`) to an integer (in bytes). Args: size (`int` or `str`): The size to convert. Will be directly returned if an `int`. Example: ```py >>> convert_file_size_to_int("1MiB") 1048576 ``` z`size` z] is not in a valid format. Use an integer for bytes, or a string with an unit (like '5.0GB').GIBNi@MIBiKIBiGBiʚ;bMBi@BKBir)r+intupperendswithfloat ValueError)r:mem_sizeerr_msgint_sizes r-convert_file_size_to_intrOgsmHu$uuu " dC  IHH ZZ\\ " "5 ) ) I5crc++u566HH ZZ\\ " "5 ) ) I5crc++u566HH ZZ\\ " "5 ) ) I5crc++u566HH ZZ\\ " "4 ( ( I5crc++u566H(, c(:(:Hx1}}HH ZZ\\ " "4 ( ( I5crc++u566H(, c(:(:Hx1}}HH ZZ\\ " "4 ( ( I5crc++u566H(, c(:(:Hx1}}H """!!!"!||!!! Os I II.dtypec|tjkrdS|tjkrdS|tjkrdS|tjkrdSt ddr|tjkrdStj dt|}|td |d t| d }|d zS) z Returns the size (in bytes) occupied by one parameter of type `dtype`. Example: ```py >>> dtype_byte_size(torch.float32) 4 ``` g?g?g?r z>=z2.1.0z [^\d](\d+)$Nz`dtype` is not a valid dtype: .rrD)r5boolrINT2INT4FP8r" float8_e4m3fnresearchstrrKrGgroups)rP bit_searchbit_sizes r-dtype_byte_sizer^s  u +" " "u +" " "u +/ ! !q $ ( (Ue6I-I-Iq>3u::66JB%BBBCCC:$$&&q)**H q=r/tensorreturnctjdtjdtjdtjdtjdtjdtjdtjdtj dtj di } | }| }n#t$r | }|||jz}n4#t$$r'd}|||jz}YnwxYwYnwxYw|j||fS)a Unique identifier to a tensor storage. Multiple different tensors can share the same underlying storage. For example, "meta" tensors all share the same storage, and thus their identifier will all be equal. This identifier is guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id. rDr r r)r5int64float32int32bfloat16float16int16uint8int8rSfloat64untyped_storagedata_ptrnbytes Exceptionstorager:rPNotImplementedErrornelementr6)r__SIZE storage_ptr storage_sizes r-id_tensor_storagervsR  Q q Q  q Q Q A A q E C,,..7799 --//6688  C C C C ..**3355K!>>++0022U6<5HHLL" C C CK!??,,uV\/BBLLL  C C =+| 33s8P/P/PF { + +VZEU I  ?ek ) )i.@L.P.PO5;OO[OOenetOOO  =HHY_--EEU[!!,,-VWW $HHUOOE aEaE   !V++ V$$)V33"&OOO"( F fc " " )!! )(!## )("$$ ))))!## )( CKK  (8(:(: ^^^__ _ =! V,,I V V8L8L/M%M%M9?++667`aa4 ) U 3 3I r6?i YbYp6q6q6qF&{3 u| , , ;((II U6:::I  *(F < E+4FOK ( (  &7 V8L8LfN`alNmNt&u&u V/ <==I' 4=F!%NNN%55)/U]:Z:Z ) U] ; ;IU??y'9\'I'I ) ) e e9CZ e^d e e h hij k k n not u uI#,C\&=Q=Q=VZ`=`=`u|F3399fj v}eT::;|?W{.6;3E3Jf3T3T!'\!:!:,!'\!:!: )\99FM=$??G ,--77>C\&=Q=Q=VZ`=`=`u|F3399fj v}mTBBE|G_$*M$6$6|$D$DFMCaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEF #    O 3 3 /)*<*<*>*>? ? ?8A **,,-f555   ' NN   / / /%..*:*:; ; ;4=(()&111  ' ' / / ; ;s7V$`''`+.`+Tinclude_buffersrecurseremove_non_persistentc#K||Ed{V|rGt}|rt||}||D]}|\}}||vr|VdSdS)a? A helper function that gathers all the tensors (parameters + buffers) of a given module. If `include_buffers=True` it's the same as doing `module.named_parameters(recurse=recurse) + module.named_buffers(recurse=recurse)`. Args: module (`torch.nn.Module`): The module we want the tensors on. include_buffer (`bool`, *optional*, defaults to `True`): Whether or not to include the buffers in the result. recurse (`bool`, *optional`, defaults to `False`): Whether or not to go look in every submodule or just return the direct parameters and buffers. remove_non_persistent (`bool`, *optional*, defaults to `False`): Whether or not to remove the non persistent buffer from the buffers. Useful only when include_buffers = True rN)named_parameterssetget_non_persistent_buffers named_buffers)rwrrrnon_persistent_buffers named_buffername_s r-named_module_tensorsrs$&&w&777777777#!$  Y%?PW%X%X%X ""000AA # #L"GD!111""""## # #r/c`|j}|r$|D]\}}||jz}|S)aV Gather all non persistent buffers of a given modules into a set Args: module (`nn.Module`): The module we want the non persistent buffers on. recurse (`bool`, *optional*, defaults to `False`): Whether or not to go look in every submodule or just return the direct non persistent buffers. )_non_persistent_buffers_set named_modules)rwrnon_persistent_buffers_setrms r-rrsN"(!CH((** H HDAq &!*G G & & %%r/c(eZdZdZfdZdZxZS)FindTiedParametersResultz This is a subclass of a list to handle backward compatibility for Transformers. Do not rely on the fact this is not a list or on the `values` method as in the future this will be removed. c:tj|i|dSN)super__init__)selfargsrrs r-rz!FindTiedParametersResult.__init__s%$)&)))))r/cjtjdttd|DgS)NzhThe 'values' method of FindTiedParametersResult is deprecated and will be removed in Accelerate v1.3.0. c"g|] }|dd S)r N).0xs r- z3FindTiedParametersResult.values..s (((aAabbE(((r/)warningswarn FutureWarningsum)rs r-valueszFindTiedParametersResult.valuess= v    ((4((("---r/)r __module__ __qualname____doc__rr __classcell__)rs@r-rrsQ *****.......r/rr,cd}d}d}ddtj|jDvrt|do)t |jddo|}t|do+t |jddot |jdd}td|D}t|||gS) z Check if there is any indication in the given model that some weights should be tied. Args: model (`torch.nn.Module`): The model to inspect Returns: bool: True if the model needs to have tied weights FPreTrainedModelcg|] }|j Sr)r)rcs r-rz3check_tied_parameters_in_config..sQQQAQZQQQr/configtie_word_embeddingsis_encoder_decodertie_encoder_decoderc36K|]}t|dVdS) _tie_weightsN)hasattr)rrws r- z2check_tied_parameters_in_config..s,\\&gfn==\\\\\\r/) inspectgetmrorrrrget_output_embeddingsanymodules)r,has_tied_word_embeddinghas_tied_encoder_decoderhas_tied_modules r-check_tied_parameters_in_configrs$$OQQ1P1PQQQQQ E8 $ $ . &;UCC .++-- E8 $ $ D &:EBB D &;UCC ! \\EMMOO\\\\\ ')A?S T TTr/c||vr||Sd|ddd}||krtd|dt||S)NrRr<z-The `device_map` does not contain the module )joinrrK_get_param_device)r device_map parent_params r-rrss %  88EKK,,SbS122LuQQQQRRR z:::r/c|D]l}i}|D]}t||||<tt|dkrtd|dmdS)a9 Check if tied parameters are on the same device Args: tied_params (`List[List[str]]`): A list of lists of parameter names being all tied together. device_map (`Dict[str, Union[int, str, torch.device]]`): A map that specifies where each submodule should go. r z*Tied parameters are on different devices: zC. Please modify your custom device map or set `device_map='auto'`. N)rlenrrloggerr) tied_paramsr tie_paramtie_param_devicesrs r-$check_tied_parameters_on_same_devicer s!  L LE'8 'K'K e $ $ s$++--.. / /! 3 3 KKT=NTTT    r/memoprefixremove_duplicatec#K|t}||vrc|r||||fV|jD].\}}|||rdndz|z}t ||||Ed{V-dSdS)a7 Return an iterator over all modules in the network, yielding both the name of the module as well as the module itself. Copied from PyTorch `torch.nn.Module.named_modules` for compatability with torch < 2.0 versions with `remove_duplicate` option added. Args: memo (set of `torch.nn.Module`, *optional*): A memo to store the set of modules already added to the result prefix (`str`, *optional*): A prefix that will be added to the name of the module remove_duplicate (`bool`, *optional*): Whether to remove the duplicated module instances in the result or not Yields: (str, Module): Tuple of name and module Note: Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. NrRr)radd_modulesitems_get_named_modules)rwrrrr sub_modulesubmodule_prefixs r-rr#s2 |uu T   HHV   fn & 5 5 7 7 ` ` D*!%)>B?$F )*d The signature accepts keyword arguments, but they are for the recursive part of this function and you should ignore them. Args: model (`torch.nn.Module`): The model to inspect. Returns: List[List[str]]: A list of lists of parameter names being all tied together. Example: ```py >>> from collections import OrderedDict >>> import torch.nn as nn >>> model = nn.Sequential(OrderedDict([("linear1", nn.Linear(4, 4)), ("linear2", nn.Linear(4, 4))])) >>> model.linear2.weight = model.linear1.weight >>> find_tied_parameters(model) [['linear1.weight', 'linear2.weight']] ``` ci|]\}}|| Srrrrrs r- z(find_tied_parameters..|spppKD%D%pppr/F)rci|]\}}|| Srrr s r-rz(find_tied_parameters..s+%%%#ee%%%r/Tc ng|]2\}}t|gtt|z3Sr)sortedlistr)rrtieds r-rz(find_tied_parameters..s8$v$v$vLFTXVVHtCII,F%G%G$v$v$vr/)r rkeysrappendr) r,rall_named_parametersno_duplicate_named_parameterstied_param_namestied_param_groupstied_param_name tied_param param_namers r-find_tied_parametersr]s@>qp;PQVin;o;o;oppp%%'[>`>`>b>b:c:cc+FF)/: !>!D!D!F!F F F J ""%66646%j1!*-44_EEE  F $$v$v\m\s\s\u\u$v$v$v w wwr/c|D]}d}|D]o}|}|d}|ddD]}t||}t||d}|!|jtjdkr|}np|P|D]M}|}|d}|ddD]}t||}t ||d|NdS)aX Reties tied parameters in a given model if the link was broken (for instance when adding hooks). Args: model (`torch.nn.Module`): The model in which to retie parameters. tied_params (`List[List[str]]`): A mapping parameter name to tied parameter name as obtained by `find_tied_parameters`. NrRr<r})rrr6r5setattr) r,r tied_group param_to_tierrwrrrs r-retie_parametersr"s"::  $  JF%%c**F 0 0 //FF2J//E#  V8L8L(L(L$   #( : : #))#..#CRC[44E$VU33FFr L9999%::r/ct|tr+|dd}tt|}|S)z4 Just does torch.dtype(dtype) if necessary. torch.r)r+rZreplacerr5rPs r-_get_proper_dtyper's=%& h++u%% Lr/special_dtypes buffers_onlyc|t|}t|}| Compute the size of each submodule of a given model. Nc4i|]\}}|t|Sr)r'rkeydtyps r-rz(compute_module_sizes..s'___93#066___r/c4i|]\}}|t|Sr)r^r,s r-rz(compute_module_sizes..s&bbbic4sOD$9$9bbbr/TrrrRr )r'r^rrrGrrnumelrPrZrminrrangerr) r,rPr(r) dtype_sizespecial_dtypes_size module_sizes module_listrr_r: name_partsidxs r-compute_module_sizesr9s !%(($U++ !__H\H\H^H^___bb>K_K_KaKabbbs##LK 8*5$??? ))$)77 # = = f  %$.*@*@<<>>$7$==DD ]<<>>OFL$A$AADD    ) )*S T T S<<>>OFL$A$AADD<<>>C OFL4Q4Q$R$RRDZZ__ Z1,-- = =C *TcT"233 4 4 4 < 4 4 4 4 = r/cTt|||d}|ddS)zO Compute the total size of buffers in each submodule of a given model. T)rPr(r)rr)r9get)r,rPr(r5s r- compute_module_total_buffer_sizer<s1(U>hlmmmL   B " ""r/rr5no_split_module_classesc d}g}|}t|dkr|d\ }t|tjjr!t|ng}t|dks|j j |vr0| }||kr|} g}n-||kr| n fd|D|z}t|dk||fS)aO Utility function that will scan a list of named modules and return the maximum size used by one full layer. The definition of a layer being: - a module with no direct children (just parameters and buffers) - a module whose class name is in the list `no_split_module_classes` Args: modules (`List[Tuple[str, torch.nn.Module]]`): The list of named modules where we want to determine the maximum layer size. module_sizes (`Dict[str, int]`): A dictionary mapping each layer name to its size (as generated by `compute_module_sizes`). no_split_module_classes (`List[str]`): A list of class names for layers we don't want to be split. Returns: `Tuple[int, List[str]]`: The maximum size of a layer with the list of layer names realizing that maximum size. rc(g|]\}}d||fSrRr)rnr  module_names r-rz&get_max_layer_size..s0WWWdaK!5!5!!5!5q 9WWWr/) copyrpopr+r5rModulernamed_childrenrrr) rr5r=max_size layer_namesmodules_to_treatrwmodules_childrenr:rBs @r-get_max_layer_sizerKs&(HK||~~   ! # #.22155 V.[s'FFF:a3E3EF1FFFr/z) is not available, available devices are c@g|]}|v|Sr)rrrrLs r-rz"get_max_memory..ls- ] ] ]qa:??K\K\F\F\F\F\F\r/)rQrdiskzX is not recognized, available devices are integers(for GPU/XPU), 'mps', 'cpu' and 'disk'c"i|] }|| SrrrUs r-rz"get_max_memory..ss888q!Z]888r/)psutilrr2r5rN device_countr_r6 mem_get_infororinforrOrrPrrrr1rvirtual_memory availabler+rZrOrsortwarningrrK) rLrXirr- gpu_devices num_devicesr6 all_devicesrs ` r-get_max_memoryrdsMMM   ' 59113344   Qu|E1/E/EFFFA$)I$:$:1$=$=a$@JqMM KK h! h h hiiiH     59113344   Qu|E1/E/EFFFA$)I$:$:1$=$=a$@JqMM KK h! h h hiiiH    5:224455   Qu|FA/F/FGGGA$)J$;$;A$>$>q$AJqMM KK h! h h hiiiH     59113344   Qu|E1/E/EFFFA$$>q$AJqMM KK h! h h hiiiH    B & 5 5 7 7 AJu   & 5 5 7 7 AJu HH jos + + H6z#GGJsOGFjoo//FFFK 0i,,..   0i,,..   0j--//   0i,,.. j--// rr [ FQJJ NNpVppVZ[`al[m[mVnVnpp q q q ] ] ] ],B ] ] ]]K __   K  u!uuu  9888K888J s]AB(CCAE(FFAH(I?I;J==(K('K(?M(NNrrBc"dkrdndfd|D}tt|dkr2t|dkrfd|DD]}||=|d|<fd|D}tdkr%tddzndtfd|D}|D]}t || |S) z] Cleans a device_map by grouping all submodules that go on the same device together. rrRcDg|]\}}||Srr)rrr rs r-rz$clean_device_map..~s. G G GDAq!,,v2F2F Ga G G Gr/r c>g|]}||Srrg)rrrs r-rz$clean_device_map..s*@@@1<<+?+?@!@@@r/rc~g|]9}|t|tk7|:Sr)rr)rrrBrs r-rz$clean_device_map..sHkkka V8L8LkQTUVQWQWZ]^iZjZjQjQjQjQjQjr/c3xK|]4}d|ddV5dSrRN)rr)rrr8s r-rz#clean_device_map..sBRRA388AGGCLL#$677RRRRRRr/)rB)rrrrrclean_device_map)rrBrrchildren_moduleschildr8rs ` @@r-rlrlxsY "$$RR[*;*;*;F G G G GJ,,.. G G GF 3v;;1Vq@@@@Z@@@  A1 "() ;lkkkk:??#4#4kkk-0-=-=-A-A#k$$ % % ) )qCRRRRAQRRRRR!8877777 r/c|t|dkrdS|D]\}}d|vr d}d|vrw|dd|vrM|dd}t t j||d||}t j||d}t ||}t||d||dS)a Loads the weights from the offload folder into the model. Args: model (`torch.nn.Module`): The model to load the weights into. index (`dict`): A dictionary containing the parameter name and its metadata for each parameter that was offloaded from the model. offload_folder (`str`): The folder where the offloaded weights are stored. Nrrrz.datr)ryrz) rrr%rrospathrr) r,r3offload_folderrmetadatarz weight_name tensor_filers r-load_offloaded_weightsrvs }E a % m m H J    z ! !j&8&85&I&IUZZ\\&Y&Y$,,Xu==K3 ^ -A-A-ABBE+DVOgll>j3F3F3FGG &{H==#E:uF\klllll m mr/ci|D]E}|dksd|vr |ddd}|ddz|<Ffd|D}|S)NrrRr rcTg|]$}|ddk|dk"|%S)rr)r;)rrwmodule_childrens r-rz%get_module_leaves..s@ h h h?3F3Fvq3Q3QUV3V3V[aeg[g[gf[g[g[gr/)rsplitr;)r5rwparentleavesrys @r-get_module_leavesr}sOEE R<<3f,, sA&&q)"1"5"5fa"@"@1"D h h h h< h h hF Mr/low_zeroc du}ttrdn5trdn$trdnt rdndt fdD}|dkrS|d kr`d }|r\D]G}t|tr0|xxd zcc<t d |d nHt|||d|r|d z n|z} |g}nt|ttfs|g}t |dkri} D]\} } | dkr |} | dD]}t!| |} | jj}||vr || vr| | |<t'| t'|krnt | dkr!t)| nd}nd}t-fdDt-tt/fdDt)t d z }tdt)||z}| |z } tt1dD}|ddD]+}t3|r|dkrdn| ||<,|rYt)ddt/fdt5d |Dz }t3|dd<S)a Compute a `max_memory` dictionary for [`infer_auto_device_map`] that will balance the use of each available GPU. All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the meta device (as it would if initialized within the `init_empty_weights` context manager). Args: model (`torch.nn.Module`): The model to analyze. max_memory (`Dict`, *optional*): A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. Example: `max_memory={0: "1GB"}`. no_split_module_classes (`List[str]`, *optional*): A list of layer class names that should never be split across device (for instance any layer that has a residual connection). dtype (`str` or `torch.dtype`, *optional*): If provided, the weights will be converted to that type when loaded. special_dtypes (`Dict[str, Union[str, torch.device]]`, *optional*): If provided, special dtypes to consider for some specific weights (will override dtype used as default for all weights). low_zero (`bool`, *optional*): Minimizes the number of weights on GPU 0, which is convenient when it's used for other operations (like the Transformers generate function). NrNrOrPrr1cfg|]-}tj|jk|dk+|.Sr)r5r6r4)rdexpected_device_typerLs r-rz'get_balanced_memory..sAsssQ Q0DH\0\0\aklmanqrararqarararr/rr Fg?z(We will use 90% of the memory on device z for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).rPr(rrRc$i|] \}}|v || Srr)rrAr r|s r-rz'get_balanced_memory..s$MMMTQQf__Aq___r/c g|] }| Srr)rrAr5s r-rz'get_balanced_memory..!s;;;q<?;;;r/g?c3VK|]$\}}t|tr |dk |V%dS)rNrS)r device_id device_mems r-rz&get_balanced_memory..'sN  /)ZT]_bIcIc hruvhvhvIhvhvhvhv  r/r<c g|] }| Srr)rr`rLs r-rz'get_balanced_memory..0s1_1_1_A*Q-1_1_1_r/)rdrrrrrrr+rGrr[r9rtuplerrrrrrmaxrr}rrr1r2)r,rLr=rPr(r~user_not_set_max_memoryrbr-per_gpuno_split_childrenrr: submodulesubmodule_name class_namebuffer mean_leaves gpus_idx_listr8min_zerorr|r5s ` @@@r-get_balanced_memoryrsbJ)D0 ++J &$   &$   &%   &$%sssss*sssttKaa " !((  c3''sOOOs*OOOKKo3oooE (U>ZZZL2h#O;??KPG&"$ /$ ? ?<#:"; "##a''&,,..  JD$rzzI"&**S// ? ?#I~>> ",5J444K\9\9\04!*-$))++,,4K0L0LLLM478I4J4JQ4N4N&--//000TU| , ,FMMMM\%7%7%9%9MMML | , ,Fc;;;;F;;;< All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the meta device (as it would if initialized within the `init_empty_weights` context manager). Args: model (`torch.nn.Module`): The model to analyze. max_memory (`Dict`, *optional*): A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. Example: `max_memory={0: "1GB"}`. no_split_module_classes (`List[str]`, *optional*): A list of layer class names that should never be split across device (for instance any layer that has a residual connection). dtype (`str` or `torch.dtype`, *optional*): If provided, the weights will be converted to that type when loaded. special_dtypes (`Dict[str, Union[str, torch.device]]`, *optional*): If provided, special dtypes to consider for some specific weights (will override dtype used as default for all weights). verbose (`bool`, *optional*, defaults to `False`): Whether or not to provide debugging statements as the function builds the device_map. clean_result (`bool`, *optional*, defaults to `True`): Clean the resulting device_map by grouping all submodules that go on the same device together. offload_buffers (`bool`, *optional*, defaults to `False`): In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as well as the parameters. NrVcg|]}|dv| S))rrVrrr6s r-rz)infer_auto_device_map..s" J J JvF/,I,IF,I,I,Ir/rQrrrrThe model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.Frz Treating module rRcPg|]"}|k|dz |#Sr@rg)rrArs r-rz)infer_auto_device_map..s5dddd1<.s3WWWDAq 1eho8V8VW!QWWWr/cg|]:}tfd|Dtfd|D8|;S)c3,K|]}dz|dzvVdSrkrrrrs r-rz3infer_auto_device_map...s0==Q4#:S(======r/c3,K|]}dz|dzvVdSrkrrs r-rz3infer_auto_device_map...s5ItItde$QT*XY\_X_J_ItItItItItItr/)rallrr rs r-rz)infer_auto_device_map..sv   ====*===== GJItItItItisItItItFtFt    r/z' Found the relevant tied param groups c,g|]}fd|DS)c(g|]}dz|dzv |Sr@r)rprs r-rz4infer_auto_device_map...s+ A A AAtczS'@'@a'@'@'@r/rrs r-rz)infer_auto_device_map..s. e e ej A A A A A A A e e er/z4 So those parameters need to be taken into account zNot enough space on z to put z (space available z, module size z).z6This module cannot be split, going to the next device.r z Splitting c(g|]\}}d||fSr@r)rrAr rs r-rz)infer_auto_device_map..s,#T#T#T41a]]q]]A$6#T#T#Tr/cZg|](\}}t|tjj$||f)Srrrs r-rz)infer_auto_device_map..s3[[[1Jq%(/.s-$g$g$g91fq!WX\fWfWfQWfWfWfr/z It looks like z is going to fit on z7 but we have tied parameters to account for. - Names z - Module names zPutting z and z on cg|] }|d Srr)rrs r-rz)infer_auto_device_map..s+K+K+KQAaD+K+K+Kr/c,g|]\}\}}|k|Srrrr`rArtied_module_names r-rz)infer_auto_device_map..s-,u,u,u91fq!_`dt_t_tQ_t_t_tr/z, needed size c(g|]\}}d||fSr@r)rrAr rs r-rz)infer_auto_device_map...s2+l+l+ltqRS0@-F-F1-F-F,J+l+l+lr/c,g|]\}\}}|k|Srrrs r-rz)infer_auto_device_map../s-(q(q(qyq&1a[\`p[p[p[p[p[pr/cZg|](\}}t|tjj$||f)Srrrs r-rz)infer_auto_device_map..9s3___DAq 1eho@^@^_!Q___r/Tz?None of the tied module can be split, going to the next device.z (size=z) on z (available=zCurrent model requires z bytes of buffer for offloaded layers, which seems does not fit any GPU's remaining memory. If you are experiencing a OOM later, please consider using offload_buffers=True.)"rdr+rrrrrr9rrrrrrrFrrKrDprintrrrr5rrr enumerateziprEr<r;rlrr)-r,rLr=rPr(rrrdevicesgpus main_devicesr5tied_parametersrcurrent_devicecurrent_memory_useddevice_memory_useddevice_buffer_sizesrImax_layer_sizemax_layer_namesrw module_sizetied_param_goupsrr6current_max_sizecurrent_memory_reservedrJtied_module_names tied_modulestied_module_indexmodule_size_with_tiescurrent_buffer_sizesplit_happened tied_moduletied_module_childrennon_gpu_buffer_sizeis_buffer_fit_any_gpu gpu_devicegpu_max_memorygpu_memory_usedrrrs- @@@r-infer_auto_device_maprHs d ++J&"$ /$ ? ?<#:";:??$$%%G Wv J J J J JD }}w TQQ' w 'U>ZZZL*511O&u-- #o2F2F!2K2K  A   JN U # #E # 2 233 u##%% & & ' u""5"11 2 2 3 '99I "l 2 2/.@ &4 #  ',?+,MP`,`,`fbl3337A&%,7W7W3&//1122   ]7>+B]]D]]'*==]]MX]]]#$$))V-=-FJa-a-aTRSSS-@CZ-Z"6*!#%)6N#36F#F &'##0.t...///#'(?(?(?(N(N#O#ORb#b #T#T#T#TCS#T#T#TWg#g 2D[[(8[[[ +33/   ! ! " L) L L $g$g$g$g BR8S8S$g$g$ghi$j!!(()9:K)LQ)OPPP##$45F$G$JKKKK qtqqAXqq=Hqq]nqq %0 !03KAR0S0S c c, ,%6F)G,WaJb)bb%%'+>AV+VZj+j+jcaTaa0Aaaw~G^aaabbb#'<<##*>#: 4 (9KK$'+K+K:J+K+K+KKK,u,u,u,u JZ@[@[,u,u,u-)),,->???3:>3JJ/00&k:fbi+H+Hk*JeN+++'3F2I2I&RS2T2TWj2j'/ uw~/FuuPTuu[luu%58K%Kuu[puuu"'589JL5Y5Y1$k+/ 0J0J0L0L+M+M(/00A559N9W[r9r9r @>+;>>>???+/ 0L0LUZ0L0[0[+\+\_s+s(+l+l+l+lWk+l+l+l((q(q(q(qIFV?./++Q>U>UV\^_>`>``Q %*4*:*:*<*< 1 1 &JU""jF&:&:( 1"4"8"8Q"G"G!%8?%JJJ,0) t99q==!6= M)*=)))    r/c^d|D}|D],dkr|nfd|D}-t |dkr'd|}t d|dS)z Checks a device map covers everything in a given model. Args: model (`torch.nn.Module`): The model to check the device map against. device_map (`Dict[str, Union[int, str, torch.device]]`): The device map to check. cg|]\}}|Srrrrrs r-rz$check_device_map..sHHH'$HHHr/rcPg|]"}|k|dz |#Sr@rg)rrrBs r-rz$check_device_map..sE!!!{**4??;QTCT3U3U****r/rz, zOThe device_map provided does not give any device for the following parameters: N) state_dictrrclearrrrK)r,rall_model_tensorsnon_covered_paramsrBs @r-check_device_maprwsIHU-=-=-?-?-E-E-G-GHHH!((   "    # # % % % E!!!!-!!!    !!!YY'899 r^p r r   "!r/c |drt|d5}|}|}dddn #1swxYwY|"td|dddi}|ddvrtd|d |ddkrtd |dd |t|Stt| d krt| d }|}tr:tdddrt!dt#|t$rd|}t||Stt| dhz }d|vr|dd|D |D]0\ }||vr' | fd|D1 d fd|Di}t-r.t/dt1 fd|Ddd d} nd} |D]}|}tr:tdddrt!dt#|t$rd|}t|d|5} |D]^} | ,| |d| | || || <| | _ dddn #1swxYwY| | |St=j|t=j dS) a3 Load a checkpoint from a given file. If the checkpoint is in the safetensors format and a device map is passed, the weights can be fast-loaded directly on the GPU. Args: checkpoint_file (`str`): The path to the checkpoint to load. device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. z .safetensorspt) frameworkNz"The safetensors archive passed at zx does not contain metadata. Make sure to save your model with the `save_pretrained` method. Defaulting to 'pt' metadata.format)rtfflaxzf does not contain the valid metadata. Make sure you save your model with the `save_pretrained` method.z%The checkpoint passed was saved with z, we need a the pt format.r r safetensors= 0.4.2 for XPU. Please upgrade safetensors.rrrVrci|]}|gSrrrs r-rz#load_state_dict..s???Vfb???r/cPg|]"}|ks|dz |#Sr@rg)rrrBs r-rz#load_state_dict..s;hhhqA4D4D U`cfUfHgHg4D4D4D4Dr/c\g|](}|tgv&|)Sr)rr)rrdevice_weightss r-rz#load_state_dict..s;)o)o)oQcR`RgRgRiRikmNnNnEnEn!EnEnEnr/Fc:g|]}t|Sr)r)rr6rs r-rz#load_state_dict..s&QQQvs> ::QQQr/w)main_process_onlytotalunit smoothingleave)rr6)devrefresh) map_location)!rIr$rsrrrr;OSErrorrKsafe_load_filerrrrrr! ImportErrorr+rGrrextendrr r set_postfixset_description get_tensorupdatecloser5loadr6) checkpoint_filerfrs weight_namesr6 target_devicertensors progress_barr-rrBs @@r-load_state_dictr s//QM $ 7 7 7 $1zz||H6688L $ $ $ $ $ $ $ $ $ $ $ $ $ $ $   KKo_ooo   !$'H << ! !)= = =I_III h 4 ' 'sXhEWssstt t  !/22 23z((**++,,11j//112215 & #%%8' sGDDw)*uvvv!&#..8(7v %omLLLL3z002233vh>??GG##u%%%@?w???N'1'7'7'9'9  # VW$$"6*11hhhhLhhh 5 ! ( ()o)o)o)o\)o)o)o p p pG "" $#&+QQQQQQQRR      $ ! 2 2 & #%%8' sGDDw)*uvvv!&#..8(7v $}UUU2YZ-f522'3(444OOO(88==='(||C'8'8 '3(//111 2222222222222222'""$$$Nz/ U8K8KLLLLs$)AA#&A#A(M::M> M> cddlm}i}t}|D]I\}}|dkr t |drt |j|r|jjr|jj}d|j_ |j |n#t$rtddwxYw| }|j |tjg||j_n| }|D]T}||jtjdkr||d |zA||} | ||d |z<UK|D]}||vr|||rt&d ||S) z Returns the state dictionary for an offloaded model via iterative onloading Args: model (`torch.nn.Module`): The offloaded model we want to save r AlignDevicesHookr_hf_hookrz;Offloaded module must fit in CPU memory to call save_model!Nr}rRzMThe following tensors were not saved because they were still on meta device: )hooksrrrrr+roffloadexecution_device pre_forward MemoryErrorr post_forwardr5r_r6rrCremoverr_) r,rr placeholdersrrworiginal_devicemodule_state_dictr-paramss r-get_state_dict_offloaded_modelrs)(((((J55L++--22 f 2::  6: & & 4:foGW+X+X 4]c]l]t 4$o>O/4FO , k++F3333 k k k!"_``fjj k & 1 1 3 3  O ( (b1A1A B B B/>FO , , & 1 1 3 3 $ 2 2C %, V0D0DDD   C !1222&s+F+1Jti#ii' ( (  2  ""%% *      $ $ $wugsuuvvv s BB8rrdevice_to_put_offloadcddlm}|d|d}d}t|drUt |j|r@|jjr4|jj}||j_|j|d}| D]1}| |} |d|z|vr | ||d|z<2|r9|j |tj g||j_|S)aL Retrieve the state dictionary (with parameters) from an offloaded module and load into a specified device (defualts to cpu). Args: module: (`torch.nn.Module`): The module we want to retrieve a state dictionary from module_name: (`str`): The name of the module of interest state_dict (`Dict[str, Union[int, str, torch.device]]`): Dictionary of {module names: parameters} device_to_put_offload (`Union[int, str, torch.device]`): Device to load offloaded parameters into, defaults to the cpu. r r NrRFrT) rrrfindrr+rrrrrrr5r_) rwrBrrrroot preforwardrm_keyrs r-get_state_dict_from_offloadr" s3()((((( /**3/// 0DJvz""z&/CS'T'TY_YhYp /:+@(##F+++ ""$$44""$$U+ ;u;; : - --3Jtk%kk) *;$$VU\"-=-=>>>+:( r/ checkpointrroffload_state_dictkeep_in_fp32_modulesoffload_8bit_bnbstrictc #|rddlm} t|} t|r-t | dkrt d|t| ||'|%d|vrtd|.|,d|vrtj |d t|tr+|d d }tt |}d} d} tj|r+t|d r|} n|g} ntj|rQd tj|D}dtj|D}t |dkr(tj||dg} nt |dkr(tj||dg} ndtj|D}t |dkr"t|dt.dt0dt |dkr'tj||d} n%t|dtd|d| tj| d#t5| 5}t7j|}dddn #1swxYwYd|vr|d}t=t?tA|} #fd| D} i}|rtCj"}i}tA}tA|#$}d|%D}| D]}tM||}|P|&|| |'tA|$|z n,|(D]\}}d|vr ||vr|)|| s'|}t |dkrK||vrGd|ddd}t |dkr||vG|d krd |vrt|d||}|}|Pt!j*|r<|:|t j+kr*d} |D]}!|!|vr|!dz|vs|!|krd} n| r t j,}d|vr\|dd|$vr2|j-t j.kr||dd}"nd}"|dkrL|s||vrD||j-}|r| |||||||"t_||d |!ta||||"|d#krH|rF||j-}|r| |||||||"t_||d |!ta||||"t_||||||"$~tcj2| sAt |dkr.t 3d%|d&|j4j5d'|d(tm|||r%to|||tqj9|tu|| dS))ax Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are loaded. Once loaded across devices, you still need to call [`dispatch_model`] on your model to make it able to run. To group the checkpoint loading and dispatch in one single call, use [`load_checkpoint_and_dispatch`]. Args: model (`torch.nn.Module`): The model in which we want to load a checkpoint. checkpoint (`str` or `os.PathLike`): The folder checkpoint to load. It can be: - a path to a file containing a whole model state dict - a path to a `.json` file containing the index to a sharded checkpoint - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint. - a path to a folder containing a unique pytorch_model.bin or a model.safetensors file. device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. offload_folder (`str` or `os.PathLike`, *optional*): If the `device_map` contains any value `"disk"`, the folder where we will offload weights. dtype (`str` or `torch.dtype`, *optional*): If provided, the weights will be converted to that type when loaded. offload_state_dict (`bool`, *optional*, defaults to `False`): If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if the weight of the CPU state dict + the biggest shard does not fit. offload_buffers (`bool`, *optional*, defaults to `False`): Whether or not to include the buffers in the weights offloaded to disk. keep_in_fp32_modules(`List[str]`, *optional*): A list of the modules that we keep in `torch.float32` dtype. offload_8bit_bnb (`bool`, *optional*): Whether or not to enable offload of 8-bit modules on cpu/disk. strict (`bool`, *optional*, defaults to `False`): Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model's state_dict. r )quantize_and_offload_8bitrrNrVzeAt least one of the model submodule will be offloaded to disk, please pass along an `offload_folder`.T)exist_okr$rz.jsonc(g|]}|tk |Sr)rrrs r-rz,load_checkpoint_in_model..s"VVVQADUDUqDUDUDUr/c(g|]}|tk |Sr)rr,s r-rz,load_checkpoint_in_model..s#%b%b%bA1PaKaKaaKaKaKar/c<g|]}|d|S)z .index.json)rIr,s r-rz,load_checkpoint_in_model..s)^^^QAJJ}D]D]^q^^^r/z6 is not a folder containing a `.index.json` file or a z or a z filezI containing more than one `.index.json` file, delete the irrelevant ones.z`checkpoint` should be the path to a file containing a whole state dict, or the index of a sharded checkpoint, or a folder containing a sharded checkpoint or the whole state dict, but got rR weight_mapcPg|]"}tj|#Sr)rprqr)rrcheckpoint_folders r-rz,load_checkpoint_in_model..s*YYY1BGLL):A>>YYYr/cg|]\}}|Srrrs r-rz,load_checkpoint_in_model..s>>>WT1D>>>r/)r)r'rr<z doesn't have any device set.Frr}r&r2r)ryrPrzz(Some weights of the model checkpoint at z! were not used when initializing z: z. This may or may not be an issue - make sure that the checkpoint does not have unnecessary parameters, or that the model definition correctly corresponds to the checkpoint.);bnbr)rrrrrrrrKrpmakedirsr+rZr%rr5rqisfilerIisdirlistdirrrrropenjsonloadsreadrrrtempfilemkdtemprrrr rrris_floating_pointrgrdrPrjrrgccollectr_rrrrvshutilrmtreer")$r,r#rrrrPr$rr%r&r'r)rcheckpoint_filesindex_filenamepotential_state_binpotential_state_safetensorpotential_indexrr3 offload_indexstate_dict_folderstate_dict_indexunexpected_keys model_keys buffer_namesrloaded_checkpointrrrB param_device new_dtypeproceedr-rzr1s$ @r-load_checkpoint_in_modelrRKsj3222222&u--K&u-- #k2B2Ba2G2G  A   ,[*EEE*"8VzGXGXGZGZ=Z=Z s     # (>6ZM^M^M`M`C`C` NT2222%& h++u%%N w~~j!! z?? # #G , , ,'NN *|   z " " VV"*Z*@*@VVV%b%bJ1G1G%b%b%b" " # #q ( ( " Z9LQ9O P PQ   + , , 1 1 " Z9STU9V W WX  _^"*Z*@*@^^^O?##q(( !FFYeFFm~FFF_%%**!#j/!:L!M!M !lll vhr v v v   !GMM.99!< . ! ! )QJqvvxx((E ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) 5 ,'E!$s5<<>>':':";";<<YYYYHXYYYM$,..eeOU%%'',,..//J>>(;(;(=(=>>>L+KK+O SSS    ! !"3F ! C C C  " "3'8'='='?'?#@#@:#M N N N N%6%<%<%>%>A A ! EJ&&Z//#'' 333!! ( +&&**{*/L/L"%((;+<+bkbbRabb b  "o 5 5%:P    %  %  %  &  %  %    U ; ; >ckccSbcc c>MMkM_MM M%'''r/)NNNN)TFF)F)NrT)rTT)NNF)NNr)r)NNNNF)NNNNFTF)r)NNNFFNFF)FN)lrhr?rr9loggingrprXrAr<r collectionsrrtypingrrrrr r r5torch.nnrrjr constantsrr dataclassesrrrimportsrrrrrrrmemoryrrrrrrr rversionsr!r" torch_npu torch_mlu torch_musarr$safetensors.torchr%rWEIGHTS_INDEX_NAME getLoggerrrr.r9rGrZrOrPr^rr6rvrE HalfTensorrrSrrrrrrrrr rr"r'r9r<rKrdrlrvr}rrrrr rr_r"PathLikerRrkrr/r-r}s?    00000000:::::::::::::::: $$$$$$66666666EEEEEEEEEEA@@@@@@@NNNNNNNNNN))))))))88888888''''''%(((!!!!!!9999994  8 $ $]]])))6)5c?))))X5;8!4el!4uU\35K/L!4!4!4!4P%)/326MQJ>J> IJ>J> #sEL( )J> EL ! J> E#u{*+ , J> e./ J>d3U\5<-G(H#HIJJ>J>J>J>\kp## I#(,#>B#cg####<&&ry&4&&&&&.....t..."U29UUUU@;;;2,0! #`#` HO#` 3ux' (#` #` #`#`#`#`L%(/^b(6x6x6x6x6xr:::>U3 #45%+15DH '' 9' E#u|+, -'T#uS%,->'?"?@A' ''''X15DH # # 9 # E#u|+, - #T#uS%,->'?"?@A # # # #$! %UX_,- .$!>B38n$!gklogp$!$!$!$!NXXxU38_eCHo-M(NOXXXXvc5c5<1G+H&H!IX[,mmm<DH37/3DH || 9|eCHouS#X>?@|&d3i0| E#u{*+ , | T#uS%,->'?"?@A |  ||||~%58?%%%%(DH37/3CG!ll 9leCHouS#X>?@l&d3i0l E#u{*+ , l T#uS%+-='>">?@ l  llllll^  BI 4U3U\CY=Z8Z3[    4\M\M\M\M~-")----h