g0: ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZddlmZddlmZmZmZddlmZddlmZmZmZmZmZmZmZmZddl m!Z!ddl"Z"ddl#m$Z$dd l%m&Z&dd l"m'Z'm(Z(dd l)m*Z*m+Z+dd l,m-Z-d dl.m/Z/d dl0m1Z1d dl2m3Z3d dl4m5Z5m6Z6d dl7m8Z8m9Z9m:Z:d dl;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEd dlFmGZGmHZHd dlImJZJd dlKmLZLd dlMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnd dlompZpmqZqmrZrd dlsmtZtmuZumvZvmwZwd dlxmyZymzZze j{|dd}Z~e j{|dd}ZebrhddlmZmZmZdd lmZdd!lmZmZmZmZmZmZmZmZmZe&jej&d"Zee&jd#krdd$lmZeirdd%lmZdd&lmZdd'lmZeljeZd(ad)Zd*Zeur2ddlmcm"Zdd+lmZe&jee&jd,kZnd-Zegrd d.lMmZe(jje(jje(jje(jje(jje(jje(jje(jje(jje(jje(jje(jje(jje(jjd/Zedld0Zd1ee(jd2ffd3Zd1ee(jd2ffd4Zd1ee(jd2ffd5Zd6Zd7Zd8Zdmd:Zd;eYfdefd?Zdnd@Z dodAeee jfdBedCeeee"jfdDefdEZdFZdGe"j'dHefdIZdmdJe(jfdKZdLeeededSeedHefdTZGdUd2ZGdVdWe(jee6e\e8Ze^eje_ejj,ejjdXdYdZ[ej_Gd\d]e(jZGd^d_e(jZGd`dae(jZeGdbdce[ZGdddee(jZGdfdge(jZdpdXe(jdhedHe(jfdiZdjZdkZdS)sN)contextmanager) dataclass) lru_cachepartialwraps)Thread)AnyCallableDictListOptionalSetTupleUnion) is_zipfile)"split_torch_state_dict_into_shards)version)Tensornn)CrossEntropyLossIdentity) checkpoint)get_activation)PretrainedConfig)custom_object_save)GenerationConfigGenerationMixin)PeftAdapterMixindeepspeed_configis_deepspeed_zero3_enabled) LOSS_MAPPING)Conv1Dapply_chunking_to_forward find_pruneable_heads_and_indicesid_tensor_storage#is_torch_greater_or_equal_than_1_13prune_conv1d_layer prune_layerprune_linear_layer)AutoHfQuantizer HfQuantizer)get_module_from_name)auto_conversion)!ACCELERATE_MIN_VERSIONADAPTER_SAFE_WEIGHTS_NAMEADAPTER_WEIGHTS_NAME CONFIG_NAME DUMMY_INPUTSFLAX_WEIGHTS_NAMESAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMETF2_WEIGHTS_NAMETF_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEContextManagers ModelOutputPushToHubMixin cached_file copy_func download_urlextract_commit_hashhas_fileis_accelerate_availableis_bitsandbytes_availableis_flash_attn_2_availableis_offline_modeis_optimum_availableis_peft_available is_remote_urlis_safetensors_availableis_torch_sdpa_availableis_torch_xla_availableloggingreplace_return_docstrings strtobool)convert_file_size_to_intcreate_and_tag_model_cardget_checkpoint_shard_files)ENV_VARS_TRUE_VALUESis_sagemaker_mp_enabledis_torch_fx_proxyis_torchdynamo_compiling)BitsAndBytesConfigQuantizationMethod XLA_USE_BF160XLA_DOWNCAST_BF16)dispatch_modelinfer_auto_device_mapinit_empty_weights)add_hook_to_module) $check_tied_parameters_on_same_deviceextract_model_from_parallelfind_tied_parametersget_balanced_memoryget_max_memoryload_offloaded_weightsoffload_weightsave_offload_indexset_module_tensor_to_device accelerate0.31)get_state_dict_from_offload) safe_open) load_file) save_fileTc>tjotjoat t jdddko0t t jdddkS)NACCELERATE_USE_FSDPFalserFSDP_CPU_RAM_EFFICIENT_LOADING)torch distributed is_availableis_initializedrOosenvirongetW/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/modeling_utils.pyis_fsdp_enabledr}s &&(( V   , , . . V bjnn%:GDD E E J V bjnn%EwOO P PTU U r{ctjoNtjo0t t jdddkS)N LOCAL_RANKr)rsrtrurvintrwrxryrzr{r|is_local_dist_rank_0rsU &&(( 7   , , . . 7  |R00 1 1Q 6r{ __version__z1.10F)find_adapter_config_file)uniform_normal_ trunc_normal_ constant_xavier_uniform_xavier_normal_kaiming_uniform_kaiming_normal_uniformnormalxavier_uniform xavier_normalkaiming_uniformkaiming_normalc#Kt}|rDdad}tD]%\}}ttjj||& dV|a|r?tD]'\}}ttjj||&dSdS#|a|r?tD]&\}}ttjj||&wwxYw)z Context manager to globally disable weight initialization to speed up loading large models. TODO(Patrick): Delete safety argument `_enable=True` at next major version. . FcdSNrz)argskwargss r| _skip_initz#no_init_weights.._skip_inits Dr{N) _init_weightsTORCH_INIT_FUNCTIONSitemssetattrrsrinit)_enableold_init_weightsrname init_funcs r|no_init_weightsrs %5     499;; 5 5OD) EHM4 4 4 4 48 (  8#7#=#=#?#? 8 8i tY7777 8 8 8 8)  8#7#=#=#?#? 8 8i tY7777 8 8s BAC" parameterModuleUtilsMixinc< t|jS#t$ridtjdt tttffd}| |}t|}|djcYSwxYw)NmodulereturncLd|jD}|S)NcDg|]\}}tj|||fSrzrs is_tensor.0kvs r| zHget_parameter_device..find_tensor_attributes..0WWWAEOTUDVDVWq!fWWWr{__dict__rrtupless r|find_tensor_attributesz4get_parameter_device..find_tensor_attributes(WW)>)>)@)@WWWFMr{get_members_fnr) next parametersdevice StopIterationrModuler rstrr_named_membersrrgen first_tuples r|get_parameter_devicers %I((**++22  % % % 29 eCK>P9Q    &&6L&MM3ii 1~$$$$ %%(A0BBc< t|jS#t$ridtjdt tttffd}| |}t|}|djcYSwxYw)z` Returns the first parameter dtype (can be non-floating) or asserts if none were found. rrcLd|jD}|S)NcDg|]\}}tj|||fSrzrrs r|rzMget_first_parameter_dtype..find_tensor_attributes..rr{rrs r|rz9get_first_parameter_dtype..find_tensor_attributesrr{rr) rrdtyperrrr rrrrrs r|get_first_parameter_dtypers $I((**++11  $ $ $ 29 eCK>P9Q    &&6L&MM3ii 1~#### $rc,d}|D]}|j}|rttvrt rt jcSttvrTt rF|jt j krt jcS|jt j krt j cS|jcS||Sdtj dttt t"ffd}||}d}|D]-}|}|dr|djcS.| |djS|D]&}|j}|r |jcS'|S)zz Returns the first found floating dtype in parameters if there is one, otherwise returns the last dtype it found. NrrcLd|jD}|S)NcDg|]\}}tj|||fSrzrrs r|rzGget_parameter_dtype..find_tensor_attributes..s0SSSTQPQ@R@RS1a&SSSr{rrs r|rz3get_parameter_dtype..find_tensor_attributess(SSV_%:%:%<%<SSS r{rr)rris_floating_pointrYrSrLrsbfloat16r[floatdoublefloat32rrr rrrrbuffers)r last_dtypetrr last_tupletuples r|get_parameter_dtypersJ  ! ! # #W    3338N8P8P3~%%% $888=S=U=U87ek)) >)))7el** =(((7NNN ryT%V :L5M  " "2H " I ICJ"" 8 % % ' ' "8> ! ! ! "!}""   W    7NNN  r{c|D]}|r |jcS td)z_ Returns the first found floating dtype in `state_dict` or asserts if none were found. z5couldn't find any floating point dtypes in state_dict)valuesrr ValueError state_dictrs r|get_state_dict_float_dtyper*sT       7NNN  L M MMr{c|D]}|r |jcS t|jS)zt Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype. )rrrrrs r|get_state_dict_dtyper5sb   //    7NNN  J%%''((..r{c|tjkrdStjdt |}|t d|dt |d}|dzS)z Returns the size (in bytes) occupied by one parameter of type `dtype`. Example: ```py >>> dtype_byte_size(torch.float32) 4 ``` g?z [^\d](\d+)_?Nz`dtype` is not a valid dtype: .r)rsboolresearchrrrgroups)r bit_searchbit_sizes r|dtype_byte_sizerBsu  u?CJJ77JB%BBBCCC:$$&&q)**H q=r{ctfd|DdkrdStrdSt|dds)t|jjddSt| d}|z|vr1||zj ||j kSdS)a> Checks if `model_to_load` supports param buffer assignment (such as when loading in empty weights) by first checking if the model explicitly disables it, then by ensuring that the state dict keys are a subset of the model's parameters. Note: We fully disable this if we are using `deepspeed` c>g|]}||Srz startswith)rkey start_prefixs r|rz9check_support_param_buffer_assignment.._s* F F FC )E)E FC F F Fr{rF!_supports_param_buffer_assignmentTzA does not support param buffer assignment, loading will be slower) lenr!getattrloggerdebug __class____name__listrkeysr) model_to_loadrr first_keys ` r|%check_support_param_buffer_assignmentrVs F F F F: F F FGG1LLu!##u ="Et L L &/ r r r   u]--//446677:Ii:--,239]=U=U=W=WXa=b=hhh 5r{10GBrmax_shard_size weights_namec tdt|}ig}d}d}i}|D]\}}t |t rt |} | |vr1|jtjdkr|| } ||| |<_| t|j z} || z|kr0t|ddkr| id}||d|<|| z }|| z }t|dz || <t|dkr ||didfSi} i} t|D]\}}|dd|dzd d t|d d}|d d|dzd d t|d d }|| |<|D]}|| |<d |i}|| d }| |fS)a Splits a model state dictionary in sub-checkpoints so that the final size of each sub-checkpoint does not exceed a given size. The sub-checkpoints are determined by iterating through the `state_dict` in the order of its keys, so there is no optimization made to make each sub-checkpoint as close as possible to the maximum size passed. For example, if the limit is 10GB and we have weights of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not [6+2+2GB], [6+2GB], [6GB]. If one of the model's weight is bigger than `max_shard_size`, it will end up in its own sub-checkpoint which will have a size greater than `max_shard_size`. Args: state_dict (`Dict[str, torch.Tensor]`): The state dictionary of a model to save. max_shard_size (`int` or `str`, *optional*, defaults to `"10GB"`): The maximum size of each sub-checkpoint. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). weights_name (`str`, *optional*, defaults to `"pytorch_model.bin"`): The name of the model save file. zNote that `shard_checkpoint` is deprecated and will be removed in v4.44. We recommend you using split_torch_state_dict_into_shards from huggingface_hub libraryrmetarrN.bin-05dz-of- .safetensors total_sizemetadata weight_map)rwarningrPr isinstancerr&rrsnumelrrrappend enumeratereplacer)rrrsharded_state_dictslast_block_sizerstorage_id_to_blockrweight storage_idblock_id weight_sizer shardsidxshard shard_filer indexs r|shard_checkpointrus6 NN J.n==N$OJ!''))GG V fc " " 3 *622J , , ,%,vBVBV1V1V*:6H17  )# . llnnv|'D'DD  [ (> 9 9cBUVXBY>Z>Z]^>^>^  & &r * * *O'-B$;&k! *-.A*B*BQ*FJ'' 1$$1!45t;;J F 344)) U!))&2gc!e2g2g2gSI\E]E]2g2g2g2ghh '' [a[[[5H1I1I[[[[  #z::<< ) )C(JsOO )j)H! < .s#HHHCK1G1GC1G1G1Gr{cg|]}|v| Srzrz)rr model_keyss r|rz+load_sharded_checkpoint..s#KKKsS 5J5Js5J5J5Jr{r#Error(s) in loading state_dict for ,cg|]}d|d S"rzrrs r|rz+load_sharded_checkpoint.. s (H(H(HaQ(H(H(Hr{z Missing key(s): cg|]}d|d Sr,rzr.s r|rz+load_sharded_checkpoint.. s +N+N+NHHHH+N+N+Nr{ weights_only map_locationcpu)strict)!rwpathjoinr9r5isfilerJrrr openjsonloadrsetrrrrrr RuntimeErrorr'safe_load_filerrsload_state_dictgccollectrmodulesr_IncompatibleKeys)modelfolderr3 prefer_safe index_filesafe_index_file index_presentsafe_index_present filenames load_safe load_indexfr shard_files missing_keysunexpected_keys error_messagestr_missing_keysstr_unexpected_keysweights_only_kwargloaderrrr&r(s @@r|load_sharded_checkpointrUs0f&899Jgll6+BCCOGNN:..M88 c"4c9Q9S9Sc=U=W=W r !8 9 9^p]r a6;;y;Q;QaaX^aaabbbI   '))  llll I$-=:J j# 0 0 0A ! s5.55778899K %**,,K!!##((**JHHHH:HHHLKKKKkKKKO *3|$$q((C,@,@1,D,DXeo>VXX |  q "xx(H(H<(H(H(HII  E2BEEE EM   ! # #"%((+N+No+N+N+N"O"O  H2EHHH HM=)))3V^.$//\^( k^^gej.k.ku.kXj.k.kF! VBGLL<<==  j777   8  " 4 4\? S SSs<EE!$E!checkpoint_file is_quantizedr1r0c b|drtrvt|d5}|}dddn #1swxYwY|ddvrt d|dt |S |qtr@tj r"tj d kstrts|sd }nd }i}t|trM|d krGt!jtjt!jd krt'|rd di}t(rd|ini}tj|fd|i||S#t,$r} t/|5}|ddkrt dt3d|d|#1swxYwYn+#t4t2f$rt d|d|dwxYwYd}~dSd}~wwxYw)z] Reads a PyTorch checkpoint file, returning properly formatted errors if they arise. rpt frameworkNformat)rYtfflaxmlxz"The safetensors archive passed at zf does not contain the valid metadata. Make sure you save your model with the `save_pretrained` method.rrr22.1.0mmapTr0r1rzYou seem to have cloned a repository without having git-lfs installed. Please install git-lfs and run `git lfs install` followed by `git lfs pull` in the folder you cloned.zUnable to locate the file z_ which is necessary to load this pretrained model. Make sure you have saved the model properly.z9Unable to load weights from pytorch checkpoint file for 'z' at 'zZ'. If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True.)endswithrJrlr ryOSErrorr<r!rsrtrvget_rankr}rr rrparserrr'r9 Exceptionr7readrUnicodeDecodeError) rVrWr1r0rLr  extra_argsrSes r|r=r= sd// /4L4N4N / $ 7 7 7 $1zz||H $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ << ! !)D D DI_III o...0  /00 %)88:: %)2244q88#%%9.B.D.D9#9 & $    , , (&& e/00GM'4J4JJJ?++K!$J?bjnl;;hjz   % !      o&& !66!99 ))!& %N_NNN         #J/   jOjj&jjj        sZAAAC9F H. G8/=G,,G0 0G83G0 4G87H)8(H  H))H.ci}|D]\\}fd|D}dkrt|}||rd|_W||<]|S)z Sets the `_is_hf_initialized` flag in all submodules of a given model when all its weights are in the loaded state dict. crh|]3}|d|dd4Srr)rr)rr module_names r| z-set_initialized_submodules..msNtttATUT`T`doarararTsTstqyyK!2!2!2B77tttr{rT) named_modulesr: issupersetr_is_hf_initialized)rBstate_dict_keysnot_initialized_submodulesrr&ros @r|set_initialized_submodulesrvfs "$$2244== Vttttttt "  o..K  ! !&"3"3"5"5 6 6 =(,F % %6< &{ 3 3 %%r{tensorrc|rC|dd|z}n|}|S)Nr)nelementviewdata_ptr element_size)rwstops r|_end_ptrr~xs[ !{{2r"++--0C0C0E0EE   Kr{rcg}t|dd(fd|jD}||t|dd(fd|jD}|||D]4\}}rd|n|}|t ||5|S)N_tied_weights_keysc&g|] }rd|n|Srrzrrprefixs r|rz)_get_tied_weight_keys..s,UUUaF1FQUUUr{_dynamic_tied_weights_keysc&g|] }rd|n|Srrzrs r|rz)_get_tied_weight_keys..s,]]]aF1FQ]]]r{r)r)rrextendrnamed_children_get_tied_weight_keys)rrtied_weight_keysnamesr submodule local_prefixs ` r|rrsv+T22>UUUU6;TUUU&&&v3T::F]]]]6;\]]]&&&!0022WWi-3=&))4)))  5i U U UVVVV r{tensorscg}|D]}t|dkr||+g}|D]A}||}||t||fB||d\}}} || h|ddD]@\} } }| |kr||hn|d|| }Ag} g} |D]R}t|dkr(| |=| |S| | fS)Nrrr)rrr{r~sortaddpop)rrfiltered_tensorssharedareasrrw_ last_stop last_namestartr}disjoint_tensorsshared_tensorss r|_find_disjointrs v;;??  # #F + + +  F FD%F LL&//++Xf-=-=tD E E E E "'(9i ,,,!&qrr   E4 !! ''//// $((...II   N#++ w<<1    # #GKKMM 2 2 2 2  ! !' * * * * + ++r{cg}g}|D]}t|dkrtjt}|D]N}||}|j|t |f}|||Ot|dkr||||||fS)Nrr) r collections defaultdictr:rr{r~rr) rrr identicalrrrrwareas r|_find_identicalrsNI * * v;;?? ',, " "D%FM6??#4#4hv6F6FGD $KOOD ! ! ! ! u::??   V $ $ $ $  ! !& ) ) ) ) 9 $$r{c g}g}i}i}i}d|jjd} |D]v} d} d| vr| dd} |s| n||| <d| vr| dd} |s| n||| <| r*|| || wi||}|rJ| dz } |D]\} } | d| d | d z } | d z } t | t||D]\} } | | || <t|d d| }|_ g ddtjf fd ||||~ S)NzA pretrained model of type `z` gammarbetabiasrcontains parameters that have been renamed internally (a few are listed below but more are present in the model): * `` -> `` nIf you are using a model from the Hub, consider submitting a PR to adjust these weights and help future users. _metadatarFrc in ddi}||d<||dgg f}tfd|Ddkrtrddl}t |ddd fd|D}t|dkr`|j|d 5tj dkr |j |dddn #1swxYwYn |j ||j D]\}} |  | ||zd z|dS) Nrassign_to_params_buffersTc>g|]}||Srzrrrrs r|rz=_load_state_dict_into_model..load..s*DDDS^^F-C-CDDDDr{rF)rrecursec(g|]}|v|Srzrz)rrnamed_parameterss r|rz=_load_state_dict_into_model..load..s+#l#l#lAVW[kVkVk$4Q$7VkVkVkr{ modifier_rankr)ryrr! deepspeeddictrrzeroGatheredParametersrsrtre_load_from_state_dict_modulesr)rrrrlocal_metadatarrparams_to_gatherrchildr error_msgsr9r s ` @r|r9z)_load_state_dict_into_model..loads'/X\\&"+r5R5R5M12FND"b*M DDDDzDDD E E I I)++ 4    $((?(?vcrc{\a(?(b(b#c#c #l#l#l#lARAR#l#l#l '((1,,#::;K[\:]]@@ ,55771<<8F8$??@@@@@@@@@@@@@@@-,d33!?0022 W WKD% UJ (;=UVVV W Ws -DD D)rr)rF)rrrrrrr info_onceziprrcopyrrr)rrrrold_keysnew_keys renamed_keys renamed_gamma renamed_beta warning_msgrnew_keyold_keyrr9r s @@@r|_load_state_dict_into_modelrs=HHLMLU1H1QUUUK  % % c>>kk'844G0=!P=M#  S==kk&&11G/; ML   % OOC OOG $ $ $4m4|4L&M M , 2 2 4 4 = = GW <<<<<< z3_load_state_dict_into_meta_model..sHH^*j.>.>s.C.CCr{rrvaluer2rrrz doesn't have any device set.disk) param_device device_mapr)0typerrrrutilsparametrizationsrrrrrrrsrrrrranyfloat16rrrinspect signaturerhrrrr rr is_contiguous contiguousr5rrf requires_parameters_quantizationcheck_quantized_paramr}rcreate_quantized_paramr!r-datarr)(rBrr expected_keysroffload_folder offload_indexstate_dict_folderstate_dict_indexr hf_quantizeris_safetensorskeep_in_fp32_modulesrOpretrained_model_name_or_pathrrrrrrWrrrrris_torch_e4m3fn_availableparamroset_module_kwargsis_param_float8_e4m3fn old_paramsplitsrrr tensor_namerparam_tors( @r| _load_state_dict_into_meta_modelr@s@JHHMLt+L-U --K  %% c>>kk'844G0=!P=M#  S==kk&&11G/; ML  28,m < < WS  ++j2UVVS  ++j2UVV2c99++&I:VV2c99++&I:VV  % OOC OOG $ $ $4m4|4L&M M , 2 2 4 4 = = GW <<<<<< (>JuuF  (z< ] ]K\ ] ] ] ]  / /uj,Xbds t t t   4$>$@$@ 4&:5*&M&M#  44 "$$&-A-C-C&%H#U EJMM($;$;NNu~NN U333 }&6 66r{variantc|C|d}|dd|gz|ddz}d|}|S)Nrr)rr5)rr rs r| _add_variantr sS##C((y(6"##;6xx'' r{c eZdZdZedZedZdZdZe de j fdZ e de j fdZ d e de fd Zedd Z d d e deede j de jde f dZ d!dee dedede fdZdZd"dededefdZdeeee j effdefdZ d#deeee j effdedefdZd S)$rzH A few utilities for `torch.nn.Modules`, to be used as a mixin. c ddl}n#t$rtdwxYw|tj}|}|j|_dS)NrFYou need to install psutil (pip install psutil) to use memory tracing.)psutil ImportErrorProcessrwgetpid memory_inforssmem_rss_pre_forward)rrrrprocessmems r|_hook_rss_memory_pre_forwardz-ModuleUtilsMixin._hook_rss_memory_pre_forwardsz h MMMM h h hfgg g h..--!!##%(W"t!c8 ddl}n#t$rtdwxYw|tj}|}|j|_|j|jz }|t|dr|j ndz|_ dS)Nrr mem_rss_diff) rrrrwrrrmem_rss_post_forwardrrr)rrrrrrrs r|_hook_rss_memory_post_forwardz.ModuleUtilsMixin._hook_rss_memory_post_forward s h MMMM h h hfgg g h..--!!##&)g#2V5OO *WVUcEdEd.kf.A.Ajkltrc|D]6}||j||j7|dS)a% Add a memory hook before and after each sub-module forward pass to record increase in memory consumption. Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to zero with `model.reset_memory_hooks_state()`. N)r@register_forward_pre_hookrregister_forward_hookrreset_memory_hooks_stateselfrs r|add_memory_hooksz!ModuleUtilsMixin.add_memory_hookssfllnn M MF  , ,T-N O O O  ( ()K L L L L %%'''''r{c^|D]}d|_d|_d|_dS)z Reset the `mem_rss_diff` attribute of each module (see [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]). rN)r@rrrr"s r|r!z)ModuleUtilsMixin.reset_memory_hooks_state's@llnn + +F"#F *+F ')*F & & + +r{rc t|S)z `torch.device`: The device on which the module is (assuming that all the module parameters are on the same device). )rr#s r|rzModuleUtilsMixin.device0s $D)))r{c t|S)zw `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). )rr's r|rzModuleUtilsMixin.dtype8s #4(((r{encoder_attention_maskc(|dkr|dddddddf}|dkr|ddddddf}||j}d|z tj|jjz}|S)z Invert an attention mask (e.g., switches 0. and 1.). Args: encoder_attention_mask (`torch.Tensor`): An attention mask. Returns: `torch.Tensor`: The inverted attention mask. Nrr?)dimrrrsfinfomin)r#r)encoder_extended_attention_masks r|invert_attention_maskz&ModuleUtilsMixin.invert_attention_mask?s " % % ' '1 , ,.DQQQaaaQRQRQR].S + ! % % ' '1 , ,.DQQQdTUTUTUEU.V + +J*L*LSWS]*L*^*^'+.1P+PTYT_`d`jTkTkTo*o'..r{Nc8|tjdtn|j}|\}}t j||}|ddddf||d|ddddfk}||j}|j d|j dkrP|j d|j dz }t j t j |||f||j|gd}|dddddddf|ddddddfz}|S)NNThe `device` argument is deprecated and will be removed in v5 of Transformers.)rrrrraxis) warningswarn FutureWarningrrsarangerepeatrrshapecatones) input_shapeattention_maskr batch_size seq_lengthseq_ids causal_maskprefix_seq_lenextended_attention_masks r|*create_extended_attention_mask_for_decoderz;ModuleUtilsMixin.create_extended_attention_mask_for_decoderWse   M`bo    $*F!, J,z&999dD!!!m,33J ANNRYZ^`a`a`acgZgRhh "nn^%9::  Q ."6q"9 9 9+1!4{7H7KKN)J JGPV^i^oppp K#.aaaqqq!!!m"<~aaaQUW[]^]^]^N^?_"_&&r{rAr@rrc2||j}|dkr |jjs|t jdt |dkr|dddddddf}nm|dkr:|jjrt|||}n,|ddddddf}ntd|d|j d| |}d |z tj |jz}|S) a Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (`Tuple[int]`): The shape of the input to the model. Returns: `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. Nrr4r+z!Wrong shape for input_ids (shape z) or attention_mask (shape )r,r-)rr.config is_decoderr8r9r:rrHrr=rrsr/r0)r#rAr@rrrGs r|get_extended_attention_maskz,ModuleUtilsMixin.get_extended_attention_maskss^ =JE""$$))dk.D)! dfs     1 $ $&4QQQaaa]&C # #    ! !Q & &{% K*:*e*e++''+9D$9I*J''sKss\j\psss #:"<"<5"<"I"I#&)@#@EKPUDVDVDZ"Z&&r{F head_masknum_hidden_layersis_attention_chunkedcv|0|||}|dur|d}ndg|z}|S)a Prepare the head mask if needed. Args: head_mask (`torch.Tensor` with shape `[num_heads]` or `[num_hidden_layers x num_heads]`, *optional*): The mask indicating if we should keep the heads or not (1.0 for keep, 0.0 for discard). num_hidden_layers (`int`): The number of hidden layers in the model. is_attention_chunked (`bool`, *optional*, defaults to `False`): Whether or not the attentions scores are computed by chunks or not. Returns: `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or list with `[None]` for each layer. NTr)_convert_head_mask_to_5d unsqueeze)r#rNrOrPs r| get_head_maskzModuleUtilsMixin.get_head_masksR$  55iARSSI#t++%//33 !22Ir{cP|dkrh|dddd}||dddd}nS|dkr;|ddd}|dksJd|||j}|S)zD-> [num_hidden_layers x batch x num_heads x seq_length x seq_length]rrrrzhead_mask.dim != 5, instead r,)r.rSexpandrr)r#rNrOs r|rRz)ModuleUtilsMixin._convert_head_mask_to_5ds ==??a  !++A..88;;EEbIISSTVWWI!(():BBKKII ]]__ ! !!++A..88<<FFrJJI}}!###%UIMMOO%U%U###LLtzL22 r{only_trainableexclude_embeddingsc |r?d|D fd|D}n!t|}g}t |dd}|r"t rddl}ntd|D]}|js|s|rt||j j rrt|dr| }nt|d r |jj}nd }||d z|z||t%|S) a Get number of (optionally, trainable or non-embeddings) parameters in the module. Args: only_trainable (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of trainable parameters exclude_embeddings (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of non-embeddings parameters Returns: `int`: The number of parameters. cRg|]$\}}t|tj|d%S).weight)r r Embedding)rr module_types r|rz3ModuleUtilsMixin.num_parameters..sJ%%%%6T;R\]hjljvRwRw%   %%%r{c"g|] \}}|v | Srzrz)rrrembedding_param_namess r|rz3ModuleUtilsMixin.num_parameters..s/   -dIDXmLmLm LmLmLmr{is_loaded_in_4bitFrNzbitsandbytes is not installed but it seems that the model has been loaded in 4bit precision, something went wrong make sure to install bitsandbytes with `pip install bitsandbytes`. You also need a GPU. r| quant_storagerr)rqrrrrrD bitsandbytesr requires_gradr r Params4bitrr|rbitemsizerrsum) r#rXrYtotal_parameters total_numelrabnbr num_bytesr`s @r|num_parameterszModuleUtilsMixin.num_parameterss  7%%:>:L:L:N:N%%% !    151F1F1H1H      $DOO$5$566  #D*=uEE  (** ***** p & 6 6E" 6. 6% 6E36;L)M)M 6un55&$)$6$6$8$8  88&$)$7$@ $% &&u{{}}q'89'DEEEE&&u{{}}555;r{ input_dictct|dsi|_|j|vr||jSd|jvr$tdd|jd<dS)z Helper function to estimate the total number of tokens from the model inputs. Args: inputs (`dict`): The model inputs. Returns: `int`: The total number of tokens. warnings_issuedestimate_tokenszdCould not estimate the number of tokens of the input, floating-point operations will not be computedTr)rromain_input_namerrr )r#rms r|rpz ModuleUtilsMixin.estimate_tokensst.// &#%D  : - -d2399;; ; d&: : : NNv   7;D !2 3qr{Tc`d||z||zS)a Get number of (optionally, non-embeddings) floating-point operations for the forward and backward passes of a batch with this transformer model. Default approximation neglects the quadratic dependency on the number of tokens (valid if `12 * d_model << sequence_length`) as laid out in [this paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter re-use e.g. Albert or Universal Transformers, or if doing long-range modeling with very high sequence lengths. Args: batch_size (`int`): The batch size for the forward pass. sequence_length (`int`): The number of tokens in each line of the batch. exclude_embeddings (`bool`, *optional*, defaults to `True`): Whether or not to count embedding and softmax operations. Returns: `int`: The number of floating-point operations. )rY)rprl)r#rmrYs r|floating_point_opsz#ModuleUtilsMixin.floating_point_opss404'' 333d6I6I]o6I6p6pppr{r)NNFFFT)r __module__ __qualname____doc__ staticmethodrrr$r!propertyrsrrrr2rHrrrrMr rrTrRrlr rrr rprtrzr{r|rrsm  \   \  ( ( (+++* ***X*)u{)))X) /F/v////0'''\'8rv2'2'$2'38:2'GL|2'chcn2' 2'2'2'2'jaf!&)>AY] 6   4 4 T4 t4 `c4 4 4 4 l$sE%,:K4L/L*MRU,[_qqsE%,*;$<<=qSWq qqqqqqr{cHeZdZdZdZdZdZdZdZdZ dZ dZ dZ dZ dZdZdZdZdZdZdZdZdZdZedeeejffdZedefdZd effd Z d Z!d Z"d Z#de$e%eefddfdZ&e'dZ(e' dode)de*ej+de*e$eeee,ffde)fdZ-e'dej+dej+fdZ.ede/j0fdZ1e'de)fdZ2e' dpde*ej+de*e$eeee,ffde)de)def dZ3e'dqde)defdZ4dZ5dZ6de/j0fd Z7d!e/j0fd"Z8de/j0fd#Z9d$Z:d%Z;d&Zd,Z?defd-Z@ drd.e*e,d/e*e,d0e)de/jAfd1ZBdsd2ZC drd3e/jAd.e*e,d/e*e,d0e)de/jAf d4ZD dtd5e/jEd.e*e,d6e*e)d0e)de/jEf d7ZFd8ZG dqd9ZHd:ZId;ZJdZMd?ZNd@ee,e%e,ffdAZOdudBZPdeQfdCe)dDeRfdEZSdFZTede)fdGZUddejVddHddddf dIe$eeWjXfdJe)dKe*eYdLeRdMe)dNe$e,efdOe)dPe*edQe*e$ee)fdRe)fdSZZe[e\j]fdTZ]dvdUZ^e[ej/j0j_fdVZ_e[ej/j0j`fdWZ`fdXZafdYZbe'dddddddZddd[ d\e*e$eeWjXfd e*e$eeeWjXfd]e*e$eeWjXfd^e)d_e)d`e)dQe*e$ee)fdaedbe)dce)ddfddZce' dwdeZddxdfZee= dydgZfe'dzdiZgd{djZhdkZidlZjedmZkeeldnZmxZnS)|PreTrainedModela Base class for all models. [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading, downloading and saving models as well as a few methods common to all models to: - resize the input embeddings, - prune heads in the self-attention heads. Class attributes (overridden by derived classes): - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class for this model architecture. - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the TensorFlow checkpoint. - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to the model. - **path** (`str`) -- A path to the TensorFlow checkpoint. - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. - **is_parallelizable** (`bool`) -- A flag indicating whether this model supports model parallelization. - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP models, `pixel_values` for vision models and `input_values` for speech models). Nr input_idsFrc8dtjtiS)z^ `Dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network. r)rsrwr3r's r| dummy_inputszPreTrainedModel.dummy_inputsxs U\,7788r{cdS)z@ :str: Identifies that this is a PyTorch model. rYrzr's r|r[zPreTrainedModel.frameworks tr{rKctt|ts*t d|jjd|jjdt|dds)||tj d}||_ |j |_ i|_ |rtj|nd|_t%j|jj|_dS)NzParameter config in `zt(config)` should be an instance of class `PretrainedConfig`. To create a model from a pretrained model use `model = z(.from_pretrained(PRETRAINED_MODEL_NAME)`_attn_implementation_autosetF) torch_dtypecheck_device_map)super__init__r rrrrr_autoset_attn_implementationrsget_default_dtyperK name_or_pathro can_generaterfrom_model_configgeneration_configr_keep_in_fp32_modules)r#rKinputsrrs r|rzPreTrainedModel.__init__s &"233 ^(?^^ N3^^^  v=uEE 66E$;$=$=PU7F "/!OSO`O`ObOb!l!1!CF!K!K!Khl&*Yt~/S%T%T"""r{cV||dS)z A method executed at the end of each Transformer model initialization, to execute code that needs the model's modules properly initialized (such as weight initialization). N) init_weights._backward_compatibility_gradient_checkpointingr's r| post_initzPreTrainedModel.post_inits.  ;;=====r{cpt|dd}|td||S)z Potentially dequantize the model in case it has been quantized by a quantization method that support dequantization. rNz?You need to first quantize your model in order to dequantize it)rr dequantize)r#rs r|rzPreTrainedModel.dequantizes> t^T::  ^__ _&&t,,,r{c|jrAt|jddr-|t |jddSdSdS)Ngradient_checkpointingF)supports_gradient_checkpointingrrKgradient_checkpointing_enabledelattrr's r|rz>PreTrainedModel._backward_compatibility_gradient_checkpointingsc  / ;GDKIach4i4i ;  . . 0 0 0 DK!9 : : : : : ; ; ; ;r{tagsct|tr|g}|jg|_|D]%}||jvr|j|&dS)a\ Add custom tags into the model that gets pushed to the Hugging Face Hub. Will not overwrite existing tags in the model. Args: tags (`Union[List[str], str]`): The desired tags to inject in the model Examples: ```python from transformers import AutoModel model = AutoModel.from_pretrained("google-bert/bert-base-cased") model.add_model_tags(["custom", "custom-bert"]) # Push the model to your namespace with the name "my-custom-bert". model.push_to_hub("my-custom-bert") ``` N)r r model_tagsr)r#rtags r|add_model_tagszPreTrainedModel.add_model_tagssi, dC  6D ? " DO , ,C$/))&&s+++ , ,r{c |dtj}|dd}d}|||}t j|}|j|j}nd}|d||_t|dds| ||d|}trgddl }t d |jt! 5||fi|}dddn #1swxYwYn ||fi|}|tj||S) z All context managers that the model should be initialized under go here. Args: torch_dtype (`torch.dtype`, *optional*): Override the default `torch.dtype` and load the model under this dtype. ruse_flash_attention_2FNattn_implementationr)rrrr@Detected DeepSpeed ZeRO-3: activating zero.init() for this modelconfig_dict_or_path)rrsr_set_default_torch_dtyperdeepcopy_attn_implementation_internal_attn_implementationrrr!rrinforInitr set_default_dtype) clsrKrrr dtype_origrrrBs r| _from_configzPreTrainedModel._from_configsjj0G0I0IJJ & +BE J J  "55kBBJv&&  / ;#)"F  "& &,jj1FH[&\&\#v=uEE 55&;!&' 6F & ' ' *     KKZ [ [ [$$9I9K9K$LL . .F--f-- . . . . . . . . . . . . . . .C))&))E  !  #J / / / s D**D.1D.TrrrrcTd}t|dr|j|jdkr|rtd|jdt |jt s>|jdvr5d|jd}|jr|d z }|jr|d z }t|d z|j}|D]i}t t||trDt||} t |t s|n| |d} | | _j|r!t d d|_|jdkr| |||d |n|dvrts|||d nd}t jjk|jdkr`t jdkr>t dt jjd n$t |t rd|_nd|_d|_|S)az Automatically checks and dispatches to a default attention implementation. In order of priority: 1. An implementation specified in `config._attn_implementation` (due for example to the argument attn_implementation="sdpa" in from_pretrained). 2. DEPRECATED: if use_flash_attention_2 is set to `True` and `flash_attn` is available, flash attention. (`LlamaFlashAttention` for example) 3. SDPA implementation, if available and supported by the model type. (`LlamaSdpaAttention` for example) 4. The default model's implementation otherwise (`LlamaAttention` for example) . Nrflash_attention_2zBoth attn_implementation="z" and `use_flash_attention_2=True` were used when loading the model, which are not compatible. We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.)eagersdparz Specified `attn_implementation="zt"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)zT, `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)zf, `"attn_implementation=sdpa"` (implementation using torch.nn.functional.scaled_dot_product_attention)rzThe model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.F)rrhard_check_onlyr)NrT)rrrzUsing the `SDPA` attention implementation on multi-gpu setup with ROCM may lead to performance issues due to the FA backend. Disabling it to use alternative backends.r)rrrrr r_supports_flash_attn_2_supports_sdparrryr warning_once_check_and_enable_flash_attn_2rL_check_and_enable_sdparsrhipcuda device_countbackendsenable_flash_sdpr) rrKrrrrrequested_attn_implementationmessager sub_configcurr_attn_implementations r|rz,PreTrainedModel._autoset_attn_implementations&)-% 6: ; ; Q@d@p*.AAAF[A r1Lrrr f94@@ 0VE`iFF OV=XOOO-vuuG%H HHG 3///-3,P ) T TC'&#..0@AA T$VS11 &&CTJJF116::3EE)   B   +>F '  &*= = =  . .'% %!1 /    +n < *=F ' r{c|r;|jst|jdtst dtr|js|St |dd}|r|S|sd|_|S)a Checks the availability of SDPA for a given model. If all checks pass and `hard_check_only` is False, the method will set the config attribute `_attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module. a does not support an attention implementation through torch.nn.functional.scaled_dot_product_attention yet. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/28005. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`zSPyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1.rFr)rrrrKrrr)rrKrrs r|rz&PreTrainedModel._check_and_enable_sdpas  %  |SSS +,, !i'(( 0B M '-De L L  M 1*0F ' r{cdd}|||_dS)z Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping the model weights fixed. c0|ddSNT)requires_grad_)rinputoutputs r|make_inputs_require_gradszMPreTrainedModel.enable_input_require_grads..make_inputs_require_grads:s  ! !$ ' ' ' ' 'r{N)get_input_embeddingsr _require_grads_hook)r#rs r|enable_input_require_gradsz*PreTrainedModel.enable_input_require_grads4s<  ( ( ($(#<#<#>#>#T#TUn#o#o   r{c8|jdS)z4 Removes the `_require_grads_hook`. N)rremover's r|disable_input_require_gradsz+PreTrainedModel.disable_input_require_grads?s  '')))))r{clt||j|}||ur|St)z Returns the model's input embeddings. Returns: `nn.Module`: A torch module mapping vocabulary to hidden states. )rrrNotImplementedError)r#rs r|rz$PreTrainedModel.get_input_embeddingsEs;T4#94@@ T ! !2244 4% %r{rcrt||j|}||ur||dSt)z Set model's input embeddings. Args: value (`nn.Module`): A module mapping vocabulary to hidden states. N)rrset_input_embeddingsr)r#rrs r|rz$PreTrainedModel.set_input_embeddingsRsCT4#94@@ T ! !  + +E 2 2 2 2 2% %r{cdS)z Returns the model's output embeddings. Returns: `nn.Module`: A torch module mapping hidden states to vocabulary. Nrzr's r|get_output_embeddingsz%PreTrainedModel.get_output_embeddings_s tr{cdS)a] Initialize the weights. This method should be overridden by derived class and is the only initialization method that will be called when loading a checkpoint using `from_pretrained`. Any attempt to initialize outside of this function will be useless as the torch.nn.init function are all replaced with skip. Nrzr"s r|rzPreTrainedModel._init_weightshs  r{cdt|ddrdS||d|_dS)zM Initialize the weights if they are not already initialized. rsFNT)rrrsr"s r|_initialize_weightsz#PreTrainedModel._initialize_weightsqs@ 6/ 7 7  F 6"""$(!!!r{c,t|jddr>|}|(|||t|jddrnt|jddrXt ||jrt||j}||j|j |jd}||_ | D]&}t |dr| 'dS) z Tie the weights between the input embeddings and the output embeddings. If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the weights instead. tie_word_embeddingsTNis_encoder_decoderFtie_encoder_decoderencoder _tie_weights) rrKr_tie_or_clone_weightsrrr_tie_encoder_decoder_weightsrdecoderrr@r)r#output_embeddings tied_weightsrs r| tie_weightszPreTrainedModel.tie_weightszs! 4; 5t < < [ $ : : < <  ,**+W>W>Y>YZZZ 4; 4e < < ;VkmrAsAs ;tT344 =tT%;<<<< dlD,BIL /;D +llnn & &Fv~.. &##%%% & &r{rrrbase_encoder_namec g}g|j|jkr*t|jd|jd d dtjdtjdt dt d t t f fd |||||t|dkrtd |S) N and zZ are not equal. In this case make sure that all encoder weights are correctly initialized.rrdecoder_pointerencoder_pointerroruninitialized_encoder_weightsc t|tjrt|tjsJ|d|dt|drt|dsJ|j|_||dt|dr8t|dsJ||d|j|_dS|j}|j} t| dkrct|dksJd|d |fd | D} d} | D]\} } | rqtt| | z}| }t| |t||s&t|t| kr| d z} n| |vr|d krtd | x}}| |||dz| z|||d z|d||d|| dz|z|t#| z }dSdS)Nrz have to be of type nn.Modulerr\rz.biasrzEncoder module z does not match decoder module c h|] }dz|z S)/rz)rsub_nameros r|rpzkPreTrainedModel._tie_encoder_decoder_weights..tie_encoder_to_decoder_recursively..s#&k&k&k{S'88'C&k&k&kr{rizMax depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is a circular dependency between two or more `nn.Modules` of your model.rr)depthtotal_encoder_nametotal_decoder_name)r rrrrrrrrrrisdigitrrrrrr)rrrorrrrrencoder_modulesdecoder_modulesall_encoder_weightsencoder_layer_posrr encoder_name decoder_name"tie_encoder_to_decoder_recursivelyrs ` r|rzXPreTrainedModel._tie_encoder_decoder_weights..tie_encoder_to_decoder_recursivelys3ory99 Wj?? W W!VVVVV W W W11 99999)8)?&##'8$U:L$U$U$UVVV?F33@"?F;;;;; ''+<(W>P(W(W(WXXX+:+?O(-6O-6O?##a''((1,,,f_ffUdff-,,'l&k&k&kTcThThTjTj&k&k&k#$%!$3$9$9$;$;QQLD&||~~;'*3t997H+H'I'I '+ )/,*Go^jNkIlIlmm%ru+ss 11s2s2 .2-$_44 (e 7;: |66' 5' 5#c)D0)5#ai.@+Q+Q<+Q+Q.@+Q+Q<+Q+Q    (..{S/@  $ $fd 3 3 ?*,-*;*;!&+%,2158I8N8TUV8WW++  " ' $n 5 5 M'BRTd:e:e M-=-L  * * * M M M Mr{ct}|g}t|dkr|d}|jj|vrvt |t r=|jt|jjd|d|t|jz}|t| z }t|dkt|S)a Get the modules of the model that should not be spit when using device_map. We iterate through the modules to get the underlying `_no_split_modules`. Args: device_map (`str`): The device map value. Options are ["auto", "balanced", "balanced_low_0", "sequential"] Returns: `List[str]`: List of modules that should not be split rrNz does not support `device_map='z_'`. To implement support, the model class needs to implement the `_no_split_modules` attribute.) r:rrrrr r~_no_split_modulesrrchildren)r#rrmodules_to_checkrs r|_get_no_split_modulesz%PreTrainedModel._get_no_split_moduless EE 6"##a''%))"--F(0AAAfo66^/7(%/8ZZYcZZZ ->FD\@]@],]) D):):$;$;; "##a''%&&&r{new_num_tokenspad_to_multiple_of mean_resizingc||||}|||St|do|jdu}trR|sPddl}|j|jd5|jjd}dddn #1swxYwYn|jjd}||j _ ||_ | |S)a$ Resizes input token embeddings matrix of the model if `new_num_tokens != config.vocab_size`. Takes care of tying weights embeddings afterwards if the model class has a `tie_weights()` method. Arguments: new_num_tokens (`int`, *optional*): The new number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything. pad_to_multiple_of (`int`, *optional*): If set will pad the embedding matrix to a multiple of the provided value.If `new_num_tokens` is set to `None` will just pad the embedding to a multiple of `pad_to_multiple_of`. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities won't be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model. Nrrr) _resize_token_embeddingsrrr!rrrrr=rKget_text_config vocab_sizer)r#rr r! model_embedsrWrr%s r|resize_token_embeddingsz'PreTrainedModel.resize_token_embeddings s@H44^EWYfgg  !&8&@ t^44V9JRV9V % ' ' 6  6    22<3FVZ2[[ : :)06q9  : : : : : : : : : : : : : : :&,215J4> ##%%0$ s,B  BBc|}|||||}t|dr|j}t |||jj}||||t|do|j du}|rtrR|sPddl } | j |jd5|jjd}dddn #1swxYwYn|jjd}||jjs|} t%| t&jjr|| ||} n|| ||} t| dr| j}t | || jj} | | || |S)N_hf_hookrrr)r!)r_get_resized_embeddingsrr)r_rrdrrrr!rrrr=rrKrr rsrr]_get_resized_lm_headset_output_embeddings) r#rr r!old_embeddingsnew_embeddingshookold_embeddings_requires_gradrWr old_lm_head new_lm_headold_lm_head_requires_grads r|r#z(PreTrainedModel._resize_token_embeddings[ss224455 N,>    >: . . 5!*D ~t 4 4 4'5'<'J$%%&BCCC !!.111t^44V9JRV9V   ))++ @L @    ^66~7L\`6aaDD%3%:%@%CNDDDDDDDDDDDDDDD"0!6!= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if `new_num_tokens` is `None` Nz5Asking to pad the embedding matrix to a multiple of `z@`, which is not and integer. Please make sure to pass an integerrrzYou are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding dimension will be a.. This might induce some performance reduction as *Tensor Cores* will not be available. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tcrrzOld embeddings are of type , which is not an instance of zj. You should either use a different resize function or make sure that `old_embeddings` are an instance of rr5zThe new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`)r rrrr=rrrrr!rrrsizerr] TypeErrorrrrrr(_init_added_embeddings_weights_with_meanr0rr padding_idx) r#r-rr r!rWrold_num_tokensold_embedding_dimr.added_num_tokensnparamss r|r*z'PreTrainedModel._get_resized_embeddingss(V  )0#66  aL^aaa%!/!6!3HX\2]] Q Q4B4I4N4N4P4P1 1 Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q1?0E0J0J0L0L -N- ^ + +4N4P4P +! !.",77 $d>.B.B$$bdbn$$L$$$   !(/ '-     N * *= *   ~ . . . . n , , ,   =    .> )++ L     ^668M7N^b6ccAA&8I>[k =="N4E~Wg  / / % ' ' R  R    $+^-BCF2262KK V V4B4I4NrPQrSTSTSTu4U%*2A2qqq51 V V V V V V V V V V V V V V V1?0E0J2A2qqq50QN ! &rr111u - & ' ' 2  2    $+^-BCF2262KK 6 6(6(=%0>0E0J0PQR0S-"-9~PQ?QUcUo>o>o15N. 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6*8)>)CN ! &,:,A,F,LQ,ON ))5>A;MQ_Qk:k:k-1*sI>C''C+.C+ II  I 9,K11K58K5&AN77N;>N;r1 transposedc ||St|do|jdu}tr|sddl}|j|jd5|s|jn*|j\}}dddn #1swxYwYnI|s|jn*|j\}}||krts|St|tj s:tdt|dtj dtj d|s||fn||f} |jdu} tj | | |jj|jjd } ||kr|s|| n||kr|rt$d ||z } tr|sddl}|jg} | r | |jgz } |j| d5||| ||| || r||| | dddn #1swxYwYn3||| ||| || r||| | t-||}trn|slddl}|j|j| j| jg} |j| d5|| |||| dddn #1swxYwYn|| |||| | S) a Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end Args: old_lm_head (`torch.nn.Linear`): Old lm head liner layer to be resized. new_num_tokens (`int`, *optional*): New number of tokens in the linear matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just returns a pointer to the input tokens `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim, vocab_size` else `vocab_size, lm_head_dim`. mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is `None` Nrrrz#Old language model head is of type r5zg. You should either use a different resize function or make sure that `old_lm_head` are an instance of r)rrraThe new lm_head weights will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`)rrr!rrrrr6rr rLinearr7rrrrrrr%_init_added_lm_head_weights_with_mean"_init_added_lm_head_bias_with_meanr0!_copy_lm_head_original_to_resized)r#r1rr?r!rWrr:old_lm_head_dimnew_lm_head_shapehas_new_lm_head_biasr2r<r>num_tokens_to_copys r|r+z$PreTrainedModel._get_resized_lm_head sF  ! t^44V9JRV9V % ' '      22;3EUY2ZZ  5?bK&++---[EWEYEYE[E[E`E`EbEb0                2<^ "'')))ASAUAUAWAWA\A\A^A^ ,NO ^ + +4N4P4P + +ry11 !d;6G6G!!gigp!!I!!! FPv_n==VdfuUv*/t; i %%,$*     N * *= *   { + + + + n , , ,   =    .> )++ hL h    %,-'1{/00F^66vT6RRll>>#[/>Sceo,l?? [Zjkkk lllllllllllllll::o~O_ak(h;;KVfggg @@ % ' '      !(+*:K)covariance_matrix) sample_shaperr)rrrrsrmeanTlinalgeigvalsrall is_complex distributionsmultivariate_normalMultivariateNormalsamplerr<) r#r-r.r;r:r<old_embeddings_weightmean_embeddingsold_centered_embeddings covariance eigenvaluesis_covariance_psd distributions r|r8z8PreTrainedModel._init_added_embeddings_weights_with_mean s!/ 5 : = =em L L*%:CCC"7/"I,.1HH>Y l**:66  :< ' , , . . pu7G 7T7T3T pZehiZiYnYnYpYp     .BUU4*3DVLFREXEX.0FYFFb&,--  ! &r,<'<'>'>'A B B B  aaa(//0@!DDGGH]Hcdd  ! &r,<'<'>'>'A B B Br{c|r6|jjj|j_|jjj|j_|||||||r8|jjj|j_|jjj|j_dSdSr)rrrNr8)r#r1r2rEr:r<r?s r|rBz5PreTrainedModel._init_added_lm_head_weights_with_mean s  @&1&8&=&?K  #&1&8&=&?K  # 55 o~GW     @&1&8&=&?K  #&1&8&=&?K  # # # @ @r{c4tj|jjdtj}tj|jjdtj}|jjd|zd|d|zdS)Nr)r7rr6rrJ)rMstd)rsrMrrrr`rr)r#r1r2r< bias_meanbias_stds r|rCz2PreTrainedModel._init_added_lm_head_bias_with_mean sJ{/41EMRRR 9[-2;;;>>u}MMb#33556>>ISWZbSb>cccccr{c|s,|jjd|ddf|jjd|ddf<n+|jjddd|f|jjddd|f<|r%|jjd||jjd|<dSdSr)rrr)r#r2r1rHr?rGs r|rDz1PreTrainedModel._copy_lm_head_original_to_resized s n>I>P>UViWiViklklklVl>mK  #$7%7$7$: ; ;>I>P>UVWVWVWYlZlYlVl>mK  #AAA':(:':$: ;  d9D9I9NObPbOb9cK  !"5#5"5 6 6 6 d dr{new_num_position_embeddingsc \td|jd|jd|jjd)Nz4`resize_position_embeddings` is not implemented for B`. To implement it, you should overwrite this method in the class  in `modeling_.py`rrrx)r#rds r|resize_position_embeddingsz*PreTrainedModel.resize_position_embeddings sV! p4> p p26. p pPTP^Pi p p p   r{c \td|jd|jd|jjd)Nz1`get_position_embeddings` is not implemented for rfrgrhrir's r|get_position_embeddingsz'PreTrainedModel.get_position_embeddings sV! p p p26. p pPTP^Pi p p p   r{c|jjr||jjtr0||j|dSdS)z If needed prunes and maybe initializes weights. If using a custom `PreTrainedModel`, you need to implement any initialization logic in `_init_weights`. N)rK pruned_heads prune_headsrapplyrrr's r|rzPreTrainedModel.init_weights sk ; # 7   T[5 6 6 6   JJt/ 0 0 0           r{heads_to_prunec |D]^\}}t|jj|gt|z}t ||jj|<_|j|dS)a Prunes heads of the base model. Arguments: heads_to_prune (`Dict[int, List[int]]`): Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2. N)rr:rKrnryrr _prune_heads)r#rqlayerheads union_headss r|rozPreTrainedModel.prune_heads s+0022 @ @LE5dk6::5"EEFFUSK.2;.?.?DK $U + + $$^44444r{c|jst|jjd|ddi}t jt fi|}dtj|j j v}|s| d|nC| t |j dt dt|d d r|dSdS) az Activates gradient checkpointing for the current model. Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint activations". We pass the `__call__` method of the modules instead of `forward` because `__call__` attaches all the hooks of the module. https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 Args: gradient_checkpointing_kwargs (dict, *optional*): Additional keyword arguments passed along to the `torch.utils.checkpoint.checkpoint` function. z) does not support gradient checkpointing.N use_reentrantTr)enablegradient_checkpointing_funcrVYou are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model._hf_peft_config_loadedF)rrrr functoolsrrrr_set_gradient_checkpointingrrprr rr)r#gradient_checkpointing_kwargsrz_is_using_old_formats r|rz-PreTrainedModel.gradient_checkpointing_enable s3 d 7bbbcc c ( 0-*]]]   r{cZ|jr|dtj|jjv}|s|dnCt d|t|jdt|ddr| dSdS)z Deactivates gradient checkpointing for the current model. Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint activations". rF)ryr|r{r}N) rrrrrrr rprrr)r#rs r|gradient_checkpointing_disablez.PreTrainedModel.gradient_checkpointing_disableS s  / S$+g.?@`.a.a.l#l ' S000>>>>L 74#C5QQQRRR 415 9 9 /  , , . . . . . / /r{cXtd|DS)z Whether gradient checkpointing is activated for this model or not. Note that in other frameworks this feature can be referred to as "activation checkpointing" or "checkpoint activations". c3DK|]}t|do|jVdS)rN)rr)rms r|rz.r s6mmYZ71677TA}?fd4|?D}@|ruttjdd5krtd6td7d8|@D}A|@D]}B||B}t||B|A}A|A}@~Atjg|r3t|@tjY||>d9d:i;||@tjY||>|7>tjY||4}Ct *d<|Cn|rtnt}DtjY|t|D|}Dt|Dd=d>?5}Etjm|7d@dAdBz}F|En|Fdddn #1swxYwYt *dC|dDtI|6j]dE|Dd|rgt||jp| | F}G|GqtjY|dG|r||||| HdSdS)Ia Save a model and its configuration file to a directory, so that it can be re-loaded using the [`~PreTrainedModel.from_pretrained`] class method. Arguments: save_directory (`str` or `os.PathLike`): Directory to which to save. Will be created if it doesn't exist. is_main_process (`bool`, *optional*, defaults to `True`): Whether the process calling this is the main process or not. Useful when in distributed training like TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on the main process to avoid race conditions. state_dict (nested dictionary of `torch.Tensor`): The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only save parts of the model or if special precautions need to be taken when recovering the state dictionary of a model (like when using model parallelism). save_function (`Callable`): The function to use to save the state dictionary. Useful on distributed training like TPUs when one need to replace `torch.save` by another method. push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). max_shard_size (`int` or `str`, *optional*, defaults to `"5GB"`): The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). We default it to 5GB in order for models to be able to run easily on free-tier google colab instances without CPU OOM issues. If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard which will be bigger than `max_shard_size`. safe_serialization (`bool`, *optional*, defaults to `True`): Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). variant (`str`, *optional*): If specified, weights are saved in the format pytorch_model..bin. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). save_peft_format (`bool`, *optional*, defaults to `True`): For backward compatibility with PEFT library, in case adapter weights are attached to the model, all keys of the state dict of adapters needs to be pre-pended with `base_model.model`. Advanced users can disable this behaviours by setting `save_peft_format` to `False`. kwargs (`Dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. use_auth_tokenNignore_metadata_errorsFrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`.rr}r)rzThe model is quantized with z and is not serializable - check out the warnings from the logger on the traceback to understand the reason why the quantized model is not serializable. save_configze`save_config` is deprecated and will be removed in v5 of Transformers. Use `is_main_process` instead.zR`safe_serialization` requires the `safetensors library: `pip install safetensors`.zProvided path (z#) should be a directory, not a fileTexist_okcommit_messagerepo_idrrr)rKrzHMoving the following attributes in the config to the generation config: z. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.zhDetected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved.zTo match the expected format of the PEFT library, all keys of the state dict of adapters will be pre-pended with `base_model.model`.zbase_model.model.zMultiple active adapters detected, saving multiple active adapters is not supported yet. You can save adapters separately one by one by iteratively calling `model.set_adapter(adapter_name)` then `model.save_pretrained(...)` hf_device_mapr2rz|Attempting to save a model with offloaded modules. Ensure that unallocated cpu memory exceeds the `shard_size` (5GB default)rcRi|]#\}}tfd|D ||$S)c3 K|]}|vV dSrrz)rr tied_namess r|rz=PreTrainedModel.save_pretrained...^ s*HnHn`dQ[I[HnHnHnHnHnHnr{)r)rptrrrs r| z3PreTrainedModel.save_pretrained..] sR###'1sESHnHnHnHnhmHnHnHnEnEn#U###r{c@i|]\}}t|dk||Srr)rrrs r|rz3PreTrainedModel.save_pretrained..c s+[[[jc5CPUJJYZNNsENNNr{c3BK|]}tj|VdSrrr)rpatrs r|rz2PreTrainedModel.save_pretrained..o s/-a-asbiT.B.B-a-a-a-a-a-ar{z8The weights trying to be saved contained shared tensors z that are mismatching the transformers base configuration. Try saving using `safe_serialization=False` or remove this tensor sharing.rz {suffix}.binrz{suffix}.safetensors)filename_patternrr z(.*?)-\d{5}-of-\d{5}zSaving checkpoint shardsdesccFi|]}||Srz)r)rrwrs r|rz3PreTrainedModel.save_pretrained.. s,SSSVZ/::<<SSSr{rjzxYou need accelerate version to be greater or equal than 0.31 to save models with offloaded parameters. Detected version z<. Please upgrade accelerate with `pip install -U accelerate`ci|]}|dSrrz)rrs r|rz3PreTrainedModel.save_pretrained.. s#?#?#?D"#?#?#?r{r\rY)r zModel weights saved in wr"r#r)indent sort_keys z:The model is bigger than the maximum size per checkpoint (z) and is going to be split in z^ checkpoint shards. You can find where each parameters has been saved in the index located at )rrz README.md)rr)srr8r9r:rrr r,is_serializablequantization_config quant_methodrJrrwr4r6rerrormakedirsrsep _create_repo_get_files_timestamps unwrap_modelrrrKrrr architecturesr _auto_classr&_get_non_default_generation_parametersrr UserWarningrrrsave_pretrainedrget_adapter_state_dictactive_adapters peft_configrr:rrrqrIS_SAGEMAKER_MP_POST_1_10smpstatemodule_managertranslate_functions_keys_to_ignore_on_saverrrrrsrr&ridrbrsortedrrrrr intersection differencer;r6r:r r0r1rr is_shardedr tensor_to_filenamelistdirr5rcompilerfilename_to_tensors fullmatchrrMtqdmaccelerate_versionrrfrkr>r?safe_save_filer5r9r7r8dumpswriterQrsave_upload_modified_files)Jr#rrrrrrrr rrrrrr}rquantization_serializablerrfiles_timestamps model_to_savermisplaced_generation_parametersr param_valuepeft_state_dictrractive_adaptercurrent_peft_config module_maprmodule_state_dict smp_to_hfr ignore_keyptrsrw tied_params shared_ptrsr error_namesto_delete_namesrfoundmatches_pattern shared_namesdisjoint_namesidentical_namesinamesknownunknownrrstate_dict_splitrfilename full_filenameweights_no_suffixfilename_no_suffixregrrrrshard_state_dictropath_to_weightssave_index_filerLcontent model_cardrrsJ ` @@r|rzPreTrainedModel.save_pretrainedt s~ $4d;;!',De!L!L  % ME     l#E  #F7O!(/G!O!Ot^T::  $ T<55 T,,@R,SS "  #,B #Kd #u|/O/\uuu  F " " Mw   %jj77O  t&>&@&@ trss s 7>>. ) )  LL^>^^^ _ _ _ F NT2222  J#ZZ(8$??NjjN,@,@,M,Mb,QRRG'd'::6::G#99.II %T** $M22+.u::+;+;C+@+@+C (/<.E.N-O *=B 9   ' t^DK H H H H , D) E2?2F2m2m2o2o/$$&& H3/N+O+ORS+S+SMm:mmm$  4S3X3X3Z3ZHH/ K ?[YYY 4j$GGGG$44^DDD  "" P/??OOO% D ~+AACC #1KK_')O&0&6&6&8&8KK UEJ(AC(A(ABB!0J!%!5!5!7!7~&&**$u"0!2&*&6~&F##33NCCC   o.. >D.55778899A==d0779999VtGYG`G`GbGb=b=b S%2$?$?$A$A>>LD&rzz (.(9(9(;(;%0>>7= 4)c))#344>&1133J % 3 # 8 L 3 3 1&Yz22   ' 3": / / !2!222":. F *400D * 0 0 2 2 2 2 ffel332*6223::4@@@@F$++D1111t_-- \2488 %!,QJ####59ZZ\\###KK#%KK[[DJJLL[[[ "7t!>LL8Jd44PdL'//GGOOP^`vww= )9.     & ,5.AE  >22 ) )HGLLBBM!- 4 4VR @ @ H HY[ \ \ "*!1!1&"!=!=!E!EnVX!Y!Y *455C##$566 )GNN=11 )$4$H$M$M$O$OOO#PMM"455A -(((.BHHJJ  e"),/BIc"d"d"d #6 O O JSSSS7SSSE % f(=(===%VTfVVV $@#?#?#?#? #(jjK' 4F'B6;Xh'i'i$$)$ ! Oubgll>:&N&NZbdhYijjjjj eRW\\.*%M%MNNNN = gll><HHO KKC/CC D D D D9Kc55QcO gll><Y`;a;abbOosW=== !*U1EEEL    ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! KK7^77 0 DEE77$3777     2NdJ OOBGLLEE F F F  ' ' - (       s0oo oc|j|jng}|dg}t|tr|g}|D]}||vr|||r||d<t j|i|S)Nr)rryr rrrr)r#rrr tags_kwargsrrs r|rzPreTrainedModel.push_to_hub s"&/"=t2jj,, k3 ' ' (&-K ! !C$ C    "!F6N"uww"D3F333r{ctd|D}|r0td|D}||z}|S)a Get the memory footprint of a model. This will return the memory footprint of the current model in bytes. Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2 Arguments: return_buffers (`bool`, *optional*, defaults to `True`): Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2 c`g|]+}||z,Srzryr|)rrs r|rz8PreTrainedModel.get_memory_footprint.. s3ZZZu5>>##e&8&8&:&::ZZZr{c`g|]+}||z,Srzr)rbufs r|rz8PreTrainedModel.get_memory_footprint.. s1ZZZCCLLNNS-=-=-?-??ZZZr{)rgrr)r#return_buffersrmem_bufss r|get_memory_footprintz$PreTrainedModel.get_memory_footprint sdZZHYHYZZZ[[  !ZZ4<<>>ZZZ[[H.C r{ct|ddtjkrtdt|ddtjkrt|ddrtdt jtjdt jdkrtd|j d dStj |i|S) Nquantization_methodz2`.cuda` is not supported for HQQ-quantized models.is_loaded_in_8bitFzCalling `cuda()` is not supported for `8-bit` quantized models. Please use the model as it is, since the model has already been set to the correct devices.rc0.43.2zCalling `cuda()` is not supported for `4-bit` quantized models with the installed version of bitsandbytes. The current device is `L`. If you intended to move the model, please install bitsandbytes >= 0.43.2.) rrXHQQrBITS_AND_BYTESrrfrr rrr)r#rrrs r|rzPreTrainedModel.cuda s 4. 5 59K9O O OQRR R 4. 5 59K9Z Z Zt0%88  sy199.IIJJW][cMdMddd H.2kHHHed  577<000 0r{cd|v}|s#|D] }t|tjrd}n!t|ddtjkrt dt|ddtjkr|rt dt|ddrt dtj tj d tj d krt d |j d n0t|ddtj kr|rt d tj|i|S)NrTr z0`.to` is not supported for HQQ-quantized models.zYou cannot cast a bitsandbytes model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `torch_dtype` argument.r Fz`.to` is not supported for `8-bit` bitsandbytes models. Please use the model as it is, since the model has already been set to the correct devices and casted to the correct `dtype`.rcr zCalling `to()` is not supported for `4-bit` quantized models with the installed version of bitsandbytes. The current device is `r zYou cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `torch_dtype` argument.)r rsrrrXrrrrrfrr rGPTQrr)r#rrdtype_present_in_argsargrs r|rzPreTrainedModel.to. s!(6 1$   c5;//,0)E 4. 5 59K9O O OOPP P 4. 5 59K9Z Z Z$  V t0%88  ly199.IIJJW][cMdMddd H.2kHHHe T0$ 7 7;M;R R R$  Nuwwz4*6***r{cpt|ddrtdtj|S)NrWFz`.half()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)rrrhalfr#rrs r|rzPreTrainedModel.halfV sD 4 / / 'I   577<& &r{cpt|ddrtdtj|S)NrWFz`.float()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)rrrrrs r|rzPreTrainedModel.float` sD 4 / / (I  !577=$' 'r{main) rK cache_dirignore_mismatched_sizesforce_downloadlocal_files_onlyrrevisionuse_safetensorsr0rrrrrrrr0c 5X| dd} | dd}| dd}| dd}| dd}| dd}| d d}| d d}| d d}| d d}| d d}| dd}| dd}| dd}| dd}| dd}| dd}| dd}| dd}| dd} | dd}!| dd}"| dd}#| dd}$| dd}%| di}&| dd }'| d!d}(| d"d})| d#d}*d}+trd}|-tjd$t|t d%|}| |& d&|&vr||&d&<| t sd} |durtd'|*tst d(|$Xt|ts2t|t||||||||#ddd) },t|,|$}$nt|dd}$t!r|&d*d}-|-t#|f||||||$d+|&}-|-et$j|-rFt+|-d,d-.5}.|}-t-j|.d/}dddn #1swxYwYnd}-t|t0jrd|i}nt|t4r=|d0vr9 dt1j|i}nO#t6$rt d1|d2wxYwt|t8r|d3krt d4d|i}||d}n|st d5|rCt;rt d6tst=d7t>d8|!s| ri|"t d9d:| D}/i|/|!| d;}/tCj"d|/dd<| \}"} td=||z }0d>d?|d@}1|||1dA<tGr|st$dBd}t|ts)||n|}2|j%j&|2f|d|||||||#||dC | \}}3n5tOj(|}| dDd}4|4|4|_)| }3t|dddu}5|5s|"E|5r tUj+|j,|"|_,n|"|_,tUj-|j,|5E}6nd}6|6u|6.||||F|6/|}|60|}|6j,j1j2|1dG<|d}tdH|6du}7d}8d}9d}:dXd};|*|6t dI| |* t5|}t$j3|}<|<r |rqt$jt$j4||#tjdJzr+t$j4||#tjdJz}=n|rkt$jt$j4||#tlr(t$j4||#tl}=n|rkt$jt$j4||#tnr(t$j4||#tn}=n| durt$jt$j4||#tqtr|%r6t$j4||#tqtr|%}=n| durt$jt$j4||#tqtt|%r8t$j4||#tqtt|%}=d}8n| st$jt$j4||#tqtv|%r6t$j4||#tqtv|%}=ny| st$jt$j4||#tqtx|%r8t$j4||#tqtx|%}=d}8n| st$jt$j4||#tjdJzsCt$jt$j4||#tlr)t{dKtqtv|%dL|dM| slt$jt$j4||#tnr)t{dKtqtv|%dL|dN| r)t{dKtqtr|%dL|d2t{dKtqtv|%dOtqtr|%dOtldOtjdJzdPtndL|d2 t$jt$j4|#|r|}=d}<nt$jt$j4|#|dJzr?|st dQ|dJzdRt$j4|#|dJz}=d}<nt}|r|}>t|}?nc|rtl}>n9|rtn}>n/| durtqtr|%}>ntqtv|%}> |||||||1||#dd|$dS }@t||>fi|@}?|?|>tqtr|%krt|tqtt|%fi|@}?|?d}8n| r]|dTkrt|fi|@\}?}}8||@dU<|?>t{|dVtqtr|%dPtqtt|%dWn#tqtv|%}>t||>fi|@}?|?>|>tqtv|%kr%t|tqtx|%fi|@}?|?d}8|stGs|?u|>tvtxfvrc|8rttntr}A|||||dX}B|||||1|#dd|$dY |B}@t||Afi|Bs/tt|fdZdi|@d[\Cn|||||dX}Bt|tlfi|Br(t{|dVtqtv|%dMt|tnfi|Br(t{|dVtqtv|%dN|%>t|tvfi|Br+t{|dVtqtv|%d]|%d^t{|dVtqtv|%dOtqtr|%dOtldOtjdPtnd2 na#tz$rt$rJ}Ct{d_|d`|datqtv|%dOtldOtjdPtnd2 |Cd}C~CwwxYw|dc|?na|*r]dddelEmF}Dt$j|*r|*}+n|||||||1||#dd|$dS }@t||*fi|@}+|D|+dfdg} d}?d}8nd}?|8rt||?|||||||1||#|$h \}?}9t r.t|?t4r|?Hdirt|?djk5}.|.J}Edddn #1swxYwY|EKdldjkrn|EKdldmkrd}t$dnnu|EKdldokrd}t$dpn?|EKdldqkrn%t dr|EKdl||z }0|0r|8s| t|?| s} d}F| t|t4r|dtkrt|dr-|jN&|jN}t$du|dvn|8r dw|9vr |9dw}n9|8st| }n't|?d3| s}Gt|G}~Gt$dxn=tt0|rtt0|}nt dy||P|}F|jQduo|t0jRkpt|6dz};|8r |9d{}Hn!t| T}H|+|s|;rtrd} ||_Ut||g}It;rL|7sJd3dlW}Jt$d}|JjXYt~g|Iz}InI|rGtst=d7t>d8|I[ttOj(|}t|dds|]||(||}t|I5||g| Ri|3}Kdddn #1swxYwY|Kj_}|;r&trt;sd}|KjQXngX|6|6`|K|X||_at|t4r{i}L|6)|Lb|6c|K||LbXfd|KdD|}M|6|6e|M}M|Kf|}N|d0vrt dd|Ni}Odtjhtjjvr|L|Od<n-t|Ld3krtd|dkrt|Kf|M|dk|d|O}nt|}|6|6n|}||Od<|Kot|Kfdw|Mi|O}|6|6.|n5|3|Kot|K}Pt|P||rx|?HdJr |r|K||?dd}Kn dddlsmt}Q|Q|K|?dd\}K}:n#t<$rtudwxYw|r= dddlvmw}R|R|K|?}Knn#t<$rtudwxYw|0rC|Ft1jx|F|y|K| |H|?|||9|||||||6X|+| \}K}S}T}U}V}W|Ko|Kz|K{rN|)Lt$d|Kj|"|)}|K_|nd|K{rP|N tj&|f||||||||#||d | |K_|n*#t$rt$dYnwxYw||||V|d}Odtjhtjjvr |Kj|Od<dtjhtjjvr"|6 |6j,j1tjkrd|Od<|6b|6j,j1tjkrGt|t r1d|vsd|vrd|Od<tst;st|Kfi|O|6|6|K|6|K_|-|K|-|'||&|r |:|S|T|U|Wd}:|K|:fS|KS)aB Instantiate a pretrained pytorch model from a pre-trained model configuration. The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train the model, you should first set it back in training mode with `model.train()`. The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning task. The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those weights are discarded. If model weights are the same precision as the base model (and is a supported model), weights will be lazily loaded in using the `meta` device and brought into memory once an input is passed through that layer regardless of `low_cpu_mem_usage`. Parameters: pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): Can be either: - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. - A path to a *directory* containing model weights saved using [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In this case, `from_tf` should be set to `True` and a configuration object should be provided as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to `True`. - `None` if you are both providing the configuration and state dictionary (resp. with keyword arguments `config` and `state_dict`). model_args (sequence of positional arguments, *optional*): All remaining positional arguments will be passed to the underlying model's `__init__` method. config (`Union[PretrainedConfig, str, os.PathLike]`, *optional*): Can be either: - an instance of a class derived from [`PretrainedConfig`], - a string or path valid as input to [`~PretrainedConfig.from_pretrained`]. Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when: - The model is a model provided by the library (loaded with the *model id* string of a pretrained model). - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the save directory. - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a configuration JSON file named *config.json* is found in the directory. state_dict (`Dict[str, torch.Tensor]`, *optional*): A state dictionary to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and [`~PreTrainedModel.from_pretrained`] is not a simpler option. cache_dir (`Union[str, os.PathLike]`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. from_tf (`bool`, *optional*, defaults to `False`): Load the model weights from a TensorFlow checkpoint save file (see docstring of `pretrained_model_name_or_path` argument). from_flax (`bool`, *optional*, defaults to `False`): Load the model weights from a Flax checkpoint save file (see docstring of `pretrained_model_name_or_path` argument). ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`): Whether or not to raise an error if some of the weights from the checkpoint do not have the same size as the weights of the model (if for instance, you are instantiating a model with 10 labels from a checkpoint with 3 labels). force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. resume_download: Deprecated and ignored. All downloads are now resumed by default when possible. Will be removed in v5 of Transformers. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (i.e., do not try to download the model). token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. mirror (`str`, *optional*): Mirror source to accelerate downloads in China. If you are from China and have an accessibility problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety. Please refer to the mirror site for more information. _fast_init(`bool`, *optional*, defaults to `True`): Whether or not to disable fast initialization. One should only disable *_fast_init* to ensure backwards compatibility with `transformers.__version__ < 4.6.0` for seeded model initialization. This argument will be removed at the next major version. See [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information. attn_implementation (`str`, *optional*): The attention implementation to use in the model (if relevant). Can be any of `"eager"` (manual implementation of the attention), `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)), or `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)). By default, if available, SDPA will be used for torch>=2.1.1. The default is otherwise the manual `"eager"` implementation. > Parameters for big model inference low_cpu_mem_usage(`bool`, *optional*): Tries not to use more than 1x model size in CPU memory (including peak memory) while loading the model. Generally should be combined with a `device_map` (such as `"auto"`) for best results. This is an experimental feature and a subject to change at any moment. If the model weights are in the same precision as the model loaded in, `low_cpu_mem_usage` (without `device_map`) is redundant and will not provide any benefit in regards to CPU memory usage. However, this should still be enabled if you are passing in a `device_map`. torch_dtype (`str` or `torch.dtype`, *optional*): Override the default `torch.dtype` and load the model under a specific `dtype`. The different options are: 1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified `dtype`, ignoring the model's `config.torch_dtype` if one exists. If not specified - the model will get loaded in `torch.float` (fp32). 2. `"auto"` - A `torch_dtype` entry in the `config.json` file of the model will be attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in the checkpoint that's of a floating point type and use that as `dtype`. This will load the model using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32. 3. A string that is a valid `torch.dtype`. E.g. "float32" loads the model in `torch.float32`, "float16" loads in `torch.float16` etc. For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or reach out to the authors and ask them to add this information to the model's card and to insert the `torch_dtype` entry in `config.json` on the hub. device_map (`str` or `Dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank like `1`) on which the model will be allocated, the device map will map the entire model to this device. Passing `device_map = 0` means put the whole model on GPU 0. To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more information about each option see [designing a device map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). max_memory (`Dict`, *optional*): A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU and the available CPU RAM if unset. offload_folder (`str` or `os.PathLike`, *optional*): If the `device_map` contains any value `"disk"`, the folder where we will offload weights. offload_state_dict (`bool`, *optional*): If `True`, will temporarily offload the CPU state dict to the hard drive to avoid getting out of CPU RAM if the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` when there is some disk offload. offload_buffers (`bool`, *optional*): Whether or not to offload the buffers with the model parameters. quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*): A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g bitsandbytes, gptq). There may be other quantization-related kwargs, including `load_in_4bit` and `load_in_8bit`, which are parsed by QuantizationConfigParser. Supported only for bitsandbytes quantizations and not preferred. consider inserting all such arguments into quantization_config instead. subfolder (`str`, *optional*, defaults to `""`): In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. variant (`str`, *optional*): If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. `variant` is ignored when using `from_tf` or `from_flax`. use_safetensors (`bool`, *optional*, defaults to `None`): Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors` is not installed, it will be set to `False`. weights_only (`bool`, *optional*, defaults to `True`): Indicates whether unpickler should be restricted to loading only tensors, primitive types, dictionaries and any types added via torch.serialization.add_safe_globals(). When set to False, we can load wrapper tensor subclass weights. kwargs (remaining dictionary of keyword arguments, *optional*): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., `output_attentions=True`). Behaves differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with `config`, `**kwargs` will be directly passed to the underlying model's `__init__` method (we assume all relevant updates to the configuration have already been done) - If a configuration is not provided, `kwargs` will be first passed to the configuration class initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that corresponds to a configuration attribute will be used to override said attribute with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's `__init__` function. Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to use this method in a firewalled environment. Examples: ```python >>> from transformers import BertConfig, BertModel >>> # Download model and configuration from huggingface.co and cache. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased") >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable). >>> model = BertModel.from_pretrained("./test/saved_model/") >>> # Update configuration during loading. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", output_attentions=True) >>> assert model.config.output_attentions == True >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable). >>> config = BertConfig.from_json_file("./tf_model/my_tf_model_config.json") >>> model = BertModel.from_pretrained("./tf_model/my_tf_checkpoint.ckpt.index", from_tf=True, config=config) >>> # Loading from a Flax checkpoint file instead of a PyTorch model (slower) >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", from_flax=True) ``` * `low_cpu_mem_usage` algorithm: This is an experimental function that loads the model using ~1x model size CPU memory Here is how it works: 1. save which state_dict keys we have 2. drop state_dict before the model is created, since the latter takes 1x model size CPU memory 3. after the model has been instantiated switch to the meta device all params/buffers that are going to be replaced from the loaded state_dict 4. load state_dict 2nd time 5. replace the params/buffers from the state_dict Currently, it can't handle deepspeed ZeRO stage 3 and ignores loading errors rNfrom_tfF from_flaxresume_downloadproxiesoutput_loading_infortrust_remote_codemirror_from_pipeline _from_auto _fast_initTrlow_cpu_mem_usager max_memoryroffload_state_dictoffload_buffers load_in_8bit load_in_4bitr subfolderr _commit_hashr adapter_kwargs adapter_namedefaultrr gguf_filerrrzgThe argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.zIaccelerate is required when loading a GGUF file `pip install accelerate`.) rrr"r#rrrr0 _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errors_adapter_model_path)rrr"r#rr1r!r"r#base_model_name_or_path)autobalancedbalanced_low_0 sequentialzWhen passing device_map as a string, the value needs to be a device name (e.g. cpu, cuda:0) or 'auto', 'balanced', 'balanced_low_0', 'sequential' but found rrznYou can't pass device_map as a negative int. If you want to put the model on the cpu, pass device_map = 'cpu' z>Passing along a `device_map` requires `low_cpu_mem_usage=True`z`DeepSpeed Zero-3 is not compatible with `low_cpu_mem_usage=True` or with passing a `device_map`.z`Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install 'accelerate>=z'`zwYou can't pass `load_in_4bit`or `load_in_8bit` as a kwarg when passing `quantization_config` argument at the same time.cZi|](\}}|tjtjv%||)Srz)rrrWrrs r|rz3PreTrainedModel.from_pretrained..s7tttDAqa7CTUgChChCs>s>s1a>s>s>sr{)r/r.) config_dictreturn_unused_kwargszThe `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.rBpytorch) file_typer[from_auto_classusing_pipelinez+Offline mode: forcing local_files_only=True) rrArr"r#rrrr0r(r'r) pre_quantized)rr r!rquantzK`low_cpu_mem_usage` was None, now default to True since model is quantized.zYou cannot combine Quantization and loading a model from a GGUF file, try again by making sure you did not passed a `quantization_config` or that you did not load a quantized model from the Hub.z.indexzError no file named z found in directory zf but there is a file for TensorFlow weights. Use `from_tf=True` to load this model from those weights.zb but there is a file for Flax weights. Use `from_flax=True` to load this model from those weights.z, r z$We found a TensorFlow checkpoint at z:, please set from_tf to True to load from this checkpoint.) rrr#r"rr user_agentrr0r6r7r1rrz& does not appear to have a file named z and thus cannot be loaded with `safetensors`. Please make sure that the model has been saved with `safe_serialization=True` or do not set `use_safetensors=True`.)rr#rrr) rrr"rrHr0r6r7r1ignore_errors_during_conversionzThread-autoconversion)targetrrrz) but there is a file without the variant z;. Use `variant=None` to load this model from those weights.zCan't load the model for 'z'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure 'z=' is the correct path to a directory containing a file named zloading weights file z from cache at r)load_gguf_checkpoint)return_tensorsr) rrr#r"rrrHrr0r1rrYrZr\r]zAA TensorFlow safetensors file is being loaded in a PyTorch model.r^z;A Flax safetensors file is being loaded in a PyTorch model.r_zTIncompatible safetensors file. File metadata is not ['pt', 'tf', 'flax', 'mlx'] but r0r;zWill use torch_dtype=z$ as defined in model's config objectrzSince the `torch_dtype` attribute can't be found in model's config object, will use torch_dtype={torch_dtype} as derived from model's weightszh`torch_dtype` can be one of: `torch.dtype`, `"auto"` or a string of a valid `torch.dtype`, but received use_keep_in_fp32_modulesall_checkpoint_keys)rrrr)rrr)rBrrchi|]-\}tfdD tj.S)c3 K|]}|vV dSrrz)rrrs r|rz=PreTrainedModel.from_pretrained..."s'CC19CCCCCCr{)rrsr)rrrrs @r|rz3PreTrainedModel.from_pretrained..sVaCCCC.BCCCCC%-r{ziIf passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or 'sequential'.no_split_module_classesspecial_dtypeszThis model has some weights that should be kept in higher precision, you need to upgrade `accelerate` to properly deal with them (`pip install --upgrade accelerate`).r>r=)rlow_zeror+)ri)$load_tf2_checkpoint_in_pytorch_model)allow_missing_keysr$zLoading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.)%load_flax_checkpoint_in_pytorch_modelzLoading a Flax model in PyTorch, requires both PyTorch and Flax to be installed. Please see https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation instructions.) rsharded_metadatar)r*rrr,rrr gguf_pathr0z\The user-defined `generation_config` will be used to override the default generation config.) rrr"r#rrrr0r(r'zZGeneration config file not found, using a generation config created from the model config.)r offload_dirrr- skip_keys force_hooksr2r)r3rr2)rNrOmismatched_keysrrz)rr}r8r9r:rrJrr rCr rr>r2rArrHrrwr4r6r7r8r9rsrrr;rr!rr/rrW from_dictrFr config_classfrom_pretrainedrrrr+merge_quantization_configsr from_configvalidate_environmentupdate_torch_dtypeupdate_device_maprrisdirr5r8r7r4r r6r5r:r9EnvironmentErrorrIr@r.rBrrrgmodeling_gguf_pytorch_utilsrKrRrcrlr ryr=rrrrrrrrrrrrrr rr^rr;rKpreprocess_model_pre_quantization_dtypeupdateget_special_dtypes_updateradjust_target_dtyperrrr]rrrcrdadjust_max_memoryrrbr`load_tf_weightsmodeling_tf_pytorch_utilsrUrmodeling_flax_pytorch_utilsrWr_load_pretrained_modelevalrrto_dictrrdr\_skip_keys_device_placementrXr FBGEMM_FP8rrpostprocess_modelr load_adapter)YrrrKrrrrrrrr0 model_argsrrr r!r"r#r$rr%r from_pipelinerDr)rr*rr+rr,r-r.r/rr0 commit_hashr r2r3rrr5rYresolved_config_filer9rLr@from_ptrH config_path model_kwargskwarg_attn_imprFrrWrrX loading_inforNis_local archive_filerresolved_archive_filecached_file_kwargssafe_weights_namehas_file_kwargsrkrKr rone_state_dictr init_contextsrrBrS target_dtypeno_split_modulesdevice_map_kwargsrrUrWrNrOr]rrrsY @r|r`zPreTrainedModel.from_pretrainedj s"#LZZ d33 **Y..JJ{E22  **%6==**Y--$jj)>FF$4d;;"JJ':DAA JJx & & #3T::  **\599ZZ d33 jj55 "JJ':DAAZZ d33 ZZ d33 $4d;;#ZZ(>zz.%88 zz.%88 $jj)>EEJJ{B// jj66 **Y--$4b99zz.)<< & +BE J J"JJ':DAAJJ{D11     % $   % ME     l#E  !;~@]@]&+N7 #  "+C+E+E "#O  $ $ NN     )@)B)B hii i  f&677 D'21'#1$3#%5%'5::?1 ''#1$3#%5!, ' '% ' '##.27>>BU3V3V.-sWEEE\*G'48IaLLAZ4[1\\\\\\\\\\\\\\\#'  j%, / / .j)JJ  C ( ( .Z?s-s-s  %,z":":;     bT^bbb   C ( ( .A~~ E!*-  ! ($(!!& c !abbb  )++  v-.. !RxNRRR  < ". G utFLLNNtttKe[e,XdeeeK*<*F+'d++FL++ '  NNh    *+#*Wfgg  $+8J' (    $%5 $ KKE F F F# &"233 "$*$6&& ?]eKeff  K$&7<<0MyZiltZt#u#uLLFGLL!> K[\\""F$&7<<0MyZj#k#kLLA27>>GLL!> K\]]$$A$&7<<0MyZk#l#lLL$E11bgnnGLL!> K\]]..+B|L'/R/RBB9BBB % *=|F.z:4<&6'D%v%v(45F(P(P%v%vVbcz}DWEWE%v%v%v'"'"!" =(4L''J'JH4? =x55K]551-4\R^`gEhEh9h9h0;9();WEE11111- 1<)-J+C"O4E4EC"0<'L:L+MMMPZ4p4K4K_p 108/6-21:8H 3"3"2;6D7F8H2<1:HMMR4? 6" 6"'6 6" 2(00MO`'t'tds't't!.$*/>.K-M0QSW/n[m/n-D %&%&%& ',eggg -5+2).-64D //O ((EGWkk[jkk"&6'D%a%a(4\7(K(K%a%a%a'"'"!" "**GIZ!n!n^m!n!n"&6'D%_%_(4\7(K(K%_%_%_'"'"!" ")!4 =|::O^::!4'7'D%m%m(4\7(K(K%m%m(/%m%m%m'"'"!" '7'D%f%f(4\7(K(K%f%fO[\movOwOw%f%f(8%f%fa>aZZ- ZZ1@ ZZFW ZZZ   f BLBBCCC(4%% dHddMbddeeee  ) I I I I I Iw~~i(( h% "+&4&'6(8"", (!*8==B$/ & &"((EyggTfgg --iMMMiXJ$( !JJ$( !  6P-%#- /!1%!#( 7 7 7 3 !#3 % & & 0#66 &..~>>  0DAAA (Q::<< ( ( ( ( ( ( ( ( ( ( ( ( ( ( (||H%%--h''4// _````h''611  YZZZZh''500 DkskwkwyAlBlBDD*+ 5 " _*"4,-BQ]^^^ J&k3//"f,,"6=99f>P>\*0*w.G %/3.B:.N.N 1@AVWXAYht1u1u1u.B>.R.R $2"KK!e! 44&-e[&A&A (UHSUU!99+FF ),(A(M( -c',Hb2c2c % A)9:O)P&&)-joo.?.?)@)@& &7 #%  #  ) ) I] *   .9F * j# & &; JN'%%l&L&LUT_&`&`aaa  ! !#(#9#9#;#;   'L'+?? MM $:::FF !UUU $ ";!B!B!MMM , 4AEWE[[[37!-0( 4AEWEbbbz400cj//1111Vz?P?P?R?R5R5R7;!"34"$$ ;-G-I-I ;u::(9:::  #  * *5 1 1 1!-E   *   #)-      '#$0'6'6",    ,& & s:P##P'*P'*RRJ=AA?A?ACBAACCACG0AHHAHHAHV AVVAV!V$AV!_8A``&A`7`=Aaa&Aa6e"Aff$Af)f(Af)cL=>?@ABCDEFGHIJKd}|du}d}d}| d| vrjt|ttfr|dn|??d}| |st d| t j| d| d} |o|du}|| }t| }|j I|| |||}d=|}=fd |D}tIdkr7tIfd |D}tIfd |D}nd}d}| o|}|o| }|r"Id >>fd |DA>fd|D}n|rIfd|D}tt!|t!|z }t!|t!|z }d|D} |r>fd| D} n|rIfd| D} t|| z }|| t%st'st)jt}!| D]/\F}"t/|"}#|!|#F0d|!D}$nt3|}$|$D]rC|r>fdCDCn|rIfdCDCCfd|DEtEdkr.tEtCkrEfd|D}sjjD]HHfd|D}jjD]HHfd|D}||||I}| r|D]̊DDt| vrDDnƉId Dt| vrId DDnDIrd Dd ddt| vr0d Dd ddD|D}%| }&|7| t@j!kr'tDfd|Dr t@j"}&|%j#tAj#dkr}tAj$|%%d|&i}'|r*tM|dds|'||'DistQ|Dd |'|)||'Dd |||r|s|rIfd!|D}(n|rIfd"|D}(n|}(tU||(})tW|j,d#r@|j,j-r4|.}*|*tW|*d$r|*j/d|*_0n!tc|2})t'r|sddl3}+tt!thj56d%|)D},|+j78|,d&5|9|j:dddn #1swxYwYn|9|j:|^|;D]I\F}%tFfd'|Dr)|%j<=t@j"|%_<Jd(J|}-tj dkr!tW|j s |r j d zJtj dkrtW|j r|stM|j }-t|- @t@Afd)|Drt d*| fd+| D} Ifd,}.|Ut j>j?|dt j>j?dd-BndB| |rt| |JG| #t| Bd.d(nd/K|5t|ttfr|dn|??fd0|D}/n&Bfd1|d2D}/GJKfd3|/D}0nd}0|Z|.||||||}1|r#t|-|J|| | |0||| ||||4\}2}0}nAt|-|J}3t|-|J|3}2nt|ts|g}g}2g}1|s| d| vrind}0| rtjG}i}nd}d}|r!t| |J5}4Bfd6|4D}4ng}4t|dkrtjJ|d78}d}3|D]}5|5|4vrd}6| b|`|jKjLtjNkrF|jKjOd9kr6tAj#d:| Dd}6t|5||6|;}|1|.||||||z }1| rt%rts|s}|- D]U\D}%|%j#tAj#dkr3tQ|-Dd tAj$|%%d| iVnOt|-|J|| | |0||| ||||4\}7}0}|2|7z }2n(|3t|-|J}3|2t|-|J|3z }2~tjS|0t|0dkr||-krj I|s_|0D]\}8tjUt j>| |8d<t j>| Id |8d<]Ifd=|0D}0|st|0| d}0| r%t|-||tjX|t|2dkr=d>|2}9d?|9vr|9d@z }9tdA|jZj[dB|9t|dkrz|j,j\gn |j,j\}:|jZj[|:vr tj^n tj_};|;dC|dD|jZj[dE|dF|jZj[dG|jZj[dH n(t_dI|jZj[dJt|dkr/t^dK|jZj[dL|dM|dNnKt|1dkr8t_dO|jZj[dP|dQ|jZj[dRt|1dkrMdSdT|1D}._fix_keys}}{{66222#~~{{7H555rx0-@@ X$$;;z3VWWW$$;;z3VWWW%7#==;;'JJWWW6#==;;'JJWWWJr{c&g|] }|Srzrz)rrrs r|rz:PreTrainedModel._load_pretrained_model..3s!<<<xx}}<<.6s/#N#NQALL$8$8#N#N#N#N#N#Nr{c3BK|]}|VdSrrrs r|rz9PreTrainedModel._load_pretrained_model..7s/'T'T V(<(<'T'T'T'T'T'Tr{rc>g|]}||Srzrrr_prefixs r|rz:PreTrainedModel._load_pretrained_model..Cs+)`)`)`!,,W^J_J_)`!)`)`)`r{clg|]0}|r|tdn|1Srrrrs r|rz:PreTrainedModel._load_pretrained_model..Ds?fffST!,,w2G2GNQs7||~~..Qfffr{c>g|]}d|gSrr5rs r|rz:PreTrainedModel._load_pretrained_model..Fs)JJJqSXXvqk22JJJr{ch|]\}}|Srzrz)rr=rs r|rpz9PreTrainedModel._load_pretrained_model..Ms===tq!===r{clh|]0}|r|tdn|1Srrrrrs r|rpz9PreTrainedModel._load_pretrained_model..Os@nnnY\CNN74K4KTSW00QTnnnr{c>h|]}d|gSrrrs r|rpz9PreTrainedModel._load_pretrained_model..Qs)NNNSXXvsm44NNNr{c>g|]\}}t|dk|Srr)rrrs r|rz:PreTrainedModel._load_pretrained_model..\s'PPPXQUa5r{clg|]0}|r|tdn|1Srrrs r|rz:PreTrainedModel._load_pretrained_model..cs?bbbUXw0G0GPS\\^^,,Sbbbr{c>g|]}d|gSrrrs r|rz:PreTrainedModel._load_pretrained_model..es)BBBS63-00BBBr{cg|]}|v| Srzrz)rrgroups r|rz:PreTrainedModel._load_pretrained_model..fsFFFa1:::::r{cg|]}|v| Srzrz)rrmissing_in_groups r|rz:PreTrainedModel._load_pretrained_model..hs$UUUa1DT;T;T;T;T;Tr{c>g|]}tj||Srrrrrs r|rz:PreTrainedModel._load_pretrained_model..ns*UUUa29S!;L;L;T;T;T;Tr{c>g|]}tj||Srrrs r|rz:PreTrainedModel._load_pretrained_model..rs+"["["[3PQARARAZ1AZAZAZr{rc3FK|]}|dvVdSrr)rrrs r|rz9PreTrainedModel._load_pretrained_model..sDE[.#))C..@r{rrr)rrrr2cg|] }d| Srrzrs r|rz:PreTrainedModel._load_pretrained_model..s##I#I#IvOOOO#I#I#Ir{cDg|]}|tdzdS)rNrrs r|rz:PreTrainedModel._load_pretrained_model..s.#N#N#NQAc&kkAo&7&7$8#N#N#Nr{rrc3BK|]}|dVdS)F)rN)r)rrs r|rz9PreTrainedModel._load_pretrained_model..sD66DMI000??666666r{rc3FK|]}|dvVdSrr)rrrs r|rz9PreTrainedModel._load_pretrained_model..s4ttE[-C@ttttttr{rc3(K|] }|vo|vV dSrrz)rrbase_model_expected_keysexpected_keys_not_prefixeds r|rz9PreTrainedModel._load_pretrained_model..s5ttad344\D\9\ttttttr{zjThe state dictionary of the model you are trying to load is corrupted. Are you sure it was properly saved?cTi|]$\}}|jdd|%Srn)rr)rrrrs r|rz:PreTrainedModel._load_pretrained_model..s<kkkPTPQSTaii3+@(C(C(CRHH!kkkr{cg}|r|D]}||vr|}|r d|}n2|r0d|ddd}||vr||j||jkr||jddkr:||dz||kr||||j||jf||=|S)Nrrrr)r5rr=rr) rmodel_state_dictr&add_prefix_to_modelremove_prefix_from_modelrr]checkpoint_key model_keyrs r|_find_mismatched_keyszEPreTrainedModel._load_pretrained_model.._find_mismatched_keyssF!O& ;&1;;N%Z77 .I/L'-$@$@$@$@ ,L$'HH^-A-A#-F-Fqrr-J$K$K "%555&~6<@PQZ@[@aaa'~6 : @ @ B BQ FJZ[dJeJkJkJmJm m m!+22!/N1K1QScdmSnSt u!+> :" "r{rztorch.rci|]}|Srzrz)rprs r|rz:PreTrainedModel._load_pretrained_model.. sLLL!aLLLr{cXi|]&\}}|tj|'Srzrwr4r5)rrrLrCs r|rz:PreTrainedModel._load_pretrained_model.. s/lllTQafa!8!8lllr{r ci|]W\}}|r=|tddk;|td||dXS)Nr)safetensors_file weight_namerr)rrrLparam_device_mapr str_dtypes r|rz:PreTrainedModel._load_pretrained_model.. sAq<< --3C1SEVEVEXEXCY2Z^d2d2d#l##%%&Qq[d(e(e2d2d2dr{) rrrrrrrrrrO)rXrcPg|]"}tj|#Srzr)rrLrCs r|rz:PreTrainedModel._load_pretrained_model..Ns)(`(`(`Qfa)@)@(`(`(`r{zLoading checkpoint shardsrint4_weight_onlycg|]}|dv| S))r2rrz)rds r|rz:PreTrainedModel._load_pretrained_model..`s$0l0l0lqST\kSkSkSkSkSkr{)rWr1r0z.datc&i|] \}}d||Srrz)rrrrs r|rz:PreTrainedModel._load_pretrained_model..s-$f$f$f*#u%6%6%6%6$f$f$fr{z z size mismatchz_ You may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method.r)z: z(Some weights of the model checkpoint at z! were not used when initializing z: z, - This IS expected if you are initializing z from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing z from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).z9All model checkpoint weights were used when initializing z. zSome weights of z3 were not initialized from the model checkpoint at z and are newly initialized: zo You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.zAll the weights of z/ were initialized from the model checkpoint at zf. If your task is similar to the task the model of the checkpoint was trained on, you can already use z* for predictions without further training.rc .g|]\}}}d|d|d|dS)z- z: found shape z in the checkpoint and z in the model instantiatedrz)rrshape1shape2s r|rz:PreTrainedModel._load_pretrained_model..sJ+VVnmmFmm6mmmr{z= and are newly initialized because the shapes did not match: )`rr rrrcrrwrrrrrupdate_expected_keysrrrr: named_buffersr}r!rrrr&rrb_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpectedupdate_missing_keysrr5rrsrrrrr6rrrhrrvrrKrrrrsrrqr itertoolschain from_iterablerrrprrrrr4rexpand_device_maprrrrrtempfilemkdtempget_disk_only_shard_filesrMrrrrXTORCHAO quant_typer=rr>r?shutilmovergrermtreer;rrrrr r)LrrBrr&rrrrXr)r*rrr,rrrrYr0rrWrris_sharded_safetensorsrroriginal_loaded_keyshas_prefix_moduleexpects_prefix_modulerrrNrO model_buffersrrw id_tensorrrrr _loaded_keysrurrnot_initialized_parametersrrr rr]rrdisk_only_shard_filesrr1new_error_msgsr error_msgarchswarnermismatched_warningrrrrrrCrrrrrrrrrsL` @@@@@@@@@@@@@@@r|rrz&PreTrainedModel._load_pretrained_models*#4/    !f 0A0A0C0C&C&C,67LtUZm,\,\w%a((bw *22>BBN%n% : ) NT::::!)%)"!/!P4DD4P !++---224455 (  #(==e]T_``M   & +<<<< <<< v;;?? ##N#N#N#N+#N#N#N N N $''T'T'T'Tm'T'T'T$T$T ! ! % $) !(9#8#R=R /M8M4M # KlllG)`)`)`)`])`)`)` &ffffXefffMM  KJJJJMJJJMc-003{3C3CCDD k**S-?-??>=u':':'<'<=== # Onnnn`mnnnMM  ONNNN NNNM =!@AA   o&7&7 @Z@\@\ *400D % 0 0 2 2 8 8 : : - - f-f55 Y&&t,,,,QPPPPKK/u55K  V VE' Cbbbb\abbb$ CBBBBEBBBFFFF<FFF #$$q((S1A-B-BSZZ-O-OUUUU<UUU   . :: V VUUUU<UUU  1 == \ \"["["["[o"["["[  #';;E>!""#566C(- % (4.._s/ $)=L<5<#7#777!KJ\JJE( s#L2TV[\\ s ,AA!uQS B   s4E3uMMMM$;;E5#uV`bqrrr  7* I+/#I#I#I#I[#I#I#ILL(/#N#N#N#N+#N#N#NLL#.L-G|-\-\*5<)>??HELDdH(-(C(C(E(E%(4&'8&AAHEVE[EcCG-@-1%2E2E2G2G-H-H*)++ 7L 7    -1!5566QkQrQrQtQt666..*^667Qab6cc;;KK 9:::;;;;;;;;;;;;;;; E5666 +$5577 > > etttt_sttttt>!&u}!=!=EJ  s$ % % ) )'%AV2W2W )\m )036L s$ % % ) )geS=R.S.S )\m )#E3+@AAM'+M,D,D,F,F,K,K,M,M'N'N $ttttthsttttt  &%kkkkXbXhXhXjXjkkk & #& #& #& #& #P ! ,W[%%&;A&>&D&DRW[&Q&QRUSURU&VWWFFF  !n !0=QS_`` ^! !)#1"/&7%5!-#1)=$3???; M+;+;$,Q!:|,,(9!:|=U 3T:: @)>(?%J O! i&0&<:K\K\K^K^A^A^dh ! ($,$4$6$6!#%  $(!#' % +(A1AP\)))%)a(`(`(`J_(`(`(`%%(*%())A--(/ 5JQl(m(m(m%'+ $3? ?  !666# *$0$8EI[Iccc$8CGYYY#(<0l0lJ18=8P      ! # #!%/>""  NN<5?#;<<1<<.<<<   lO_mU___s*\\\cd|D}|d|D}g}|D]\}}|r9|jd}||r|t |dn|}n8|r6t |dkrd|j|gn|j}||vr|||S)Ncnh|]2}d|ddd3S)rNr)r5rrrs r|rpz>PreTrainedModel.retrieve_modules_from_names..s7FFFsxx #ss 344FFFr{ch|]_}t|dk|d/d|ddd`S)rrrN)rr r5rrs r|rpz>PreTrainedModel.retrieve_modules_from_names..sY b b bss3xx!||PSTVPWP_P_PaPa|SXXciinnSbS) * *|||r{rr)unionrqrrrr5r) r#r add_prefix remove_prefix module_keysretrieved_modulesrrrs r|retrieve_modules_from_namesz+PreTrainedModel.retrieve_modules_from_namess FFFFF "'' b be b b b   ..00 1 1LD& m!3666/3w/G/GQtCLLNN++T mCFt99q==sxx!7 >???VZVl{""!((000  r{cvt|||t||}|}t|||||} | S)a This is an experimental function that loads the model using ~1.x model size CPU memory Before you call it do: 1. save which state_dict keys are available 2. drop state_dict before model is created, since the latter takes 1x model size memory Here then we continue: 3. switch to the meta device all params/buffers that are going to be replaced from the loaded state_dict 4. load state_dict 2nd time 5. replace the params/buffers from the state_dict Currently, it doesn't handle missing_keys, unexpected_keys, mismatched_keys. It can't handle deepspeed. To handle bitsandbytes, needs non-empty hf_quantizer argument. rM)rr)rr=r) rBrrrrrr0rrrs r|_load_pretrained_model_low_memz.PreTrainedModel._load_pretrained_model_low_memsZ8 E#9<HHH$%:VVV . 5   '%    r{ AutoModelct|ts|j}ddlmcm}t ||st|d||_dS)a Register this class with a given auto class. This should only be used for custom models as the ones in the library are already mapped with an auto class. This API is experimental and may have some slight breaking changes in the next releases. Args: auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`): The auto class to register this new model with. rNz is not a valid auto class.) r rrtransformers.models.automodelsr;rrr)r auto_class auto_modules r|register_for_auto_classz'PreTrainedModel.register_for_auto_classsn *c** -#,J666666666{J// I GGGHH H$r{ctstdddlm}t j|t jdkrtd|dddlm}||S)a( Converts the model to use [PyTorch's native attention implementation](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html), integrated to Transformers through [Optimum library](https://huggingface.co/docs/optimum/bettertransformer/overview). Only a subset of all Transformers models are supported. PyTorch's attention fastpath allows to speed up inference through kernel fusions and the use of [nested tensors](https://pytorch.org/docs/stable/nested.html). Detailed benchmarks can be found in [this blog post](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2). Returns: [`PreTrainedModel`]: The model converted to BetterTransformer. =1.7.0 to use Better Transformer. The version was found.BetterTransformer) rGroptimum.versionrrrfoptimum.bettertransformerr transformr#optimum_versionrs r|to_bettertransformerz$PreTrainedModel.to_bettertransformer6s$%% ^\]] ]BBBBBB = ) )GM',B,B B BtXgttt  @????? **4000r{ctstdddlm}t j|t jdkrtd|dddlm}||S)a Reverts the transformation from [`~PreTrainedModel.to_bettertransformer`] so that the original modeling is used, for example in order to save the model. Returns: [`PreTrainedModel`]: The model converted back to the original modeling. rrrrrrr) rGrrrrrfrrreversers r|reverse_bettertransformerz)PreTrainedModel.reverse_bettertransformerRs$%% ^\]] ]BBBBBB = ) )GM',B,B B BtXgttt  @????? ((...r{c ht|s,tjst rdS| |jjdS|jj|ddddgfvrd}|jj|jj|jjksL|jj|jj|jjks&|jj T|jj |jjkr:|d|jjd|jjd|jjd|jj d z }t |dSdS) zv Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given. NrrzWe strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.z5 You may ignore this warning if your `pad_token_id` (z&) is identical to the `bos_token_id` (z), `eos_token_id` (z), or the `sep_token_id` (z ), and your input is not padded.) rUrsjit is_tracingrVrK pad_token_id bos_token_id eos_token_id sep_token_idrr)r#rrA warn_strings r|%warn_if_padding_and_no_attention_maskz5PreTrainedModel.warn_if_padding_and_no_attention_maskhsn Y ' ' 59+?+?+A+A E]E_E_  F  &DK,D,L F ; #yRG'< < <F )5$+:RVZVaVn:n:nK,8T[=UY]YdYq=q=qK,8T[=UY]YdYq=q=qiT[Meii.2k.Fii[_[f[sii.2k.Fiii    , , , , ,) =  -II/I ,,;#((<"8"8;;; J{DN4KLL y>>A%% )! II $I    = ='$+WbdhBiBiBu   >i>>>   &II&&r{)FNNT)NNTFru)NNTr)NFTrrw) FNTFNNNNNNNTrv)rNNT)r)rr~)orrxryrzr_rrqrrrrurrrrris_parallelizabler _is_statefulrr_supports_cache_class_supports_static_cache_supports_quantized_cacher|r rrsrrr[rrrrrrr r classmethodrrr rrrrrrrrrrrrrrrrrrr{rrrr]r'r#r*rAr+r8rBrCrDrjrrlrrorrr rrrrrwPathLikerrrr=rrrrrrr`rrrrrr r rrrr! __classcell__rs@r|r~r~4s 6L!OJK"& '+#*.&#&+#L#N""!& 9d3 #45999X9 3X U/UUUUUU.>>> - - -;;; ,5cC#8,T,,,,@55[5n',-1;?!% ]] $]ek* ] U3S#X#678 ]  ]]][]~U[U[[4;BI;;;X; $T$$$[$L.2;?!% % [[ek*[U3S#X#678 [  [  [ [[[[[zTN^[: p p p*** &bi & & & & &") & & & &ry   )))&&&6WW%'YWCFW[^WWW\WrMMM('''''>)-,0" 99  9%SM9 9  9999v%+%+%+%+T)-,0" \\ \! \%SM \  \  \\\\B)-%*" ~~Y~! ~TN ~  ~  ~~~~@D@@@@0ddd d d d c     r|U2<=P/P)Q    "5$sDI~*>5555"(.(.(.(.T:>gq$\d,///.n4nnnXn!%%)"'*!*/#'!%,0!%CCc2;./CCTN C  C  Cc3hC!C#Cc4i()CCCCCJ  U> %&& 4 4 4 4'& 4$ U58?   1111! 1$ U58? %+%+%+%+%+N'''''((((( GK7;(-$!&,0 $!{{{'/c2;6F0G'H{/bkABC { E#r{"234 { "& {{{c4i(){{{{ {{{[{z#!&!%r`r`r`[r`h!!!!. &*%%%\%N%%%[%211118///,!-!-!-F . .X .''YX'''''r{r~rBrz model file)object object_class object_filescjeZdZdZdeffd Z d dejdeejdejfdZ xZ S) PoolerStartLogitsz Compute SQuAD start logits from sequence hidden states. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model. rKcttj|jd|_dS)Nr)rrrrA hidden_sizedenser#rKrs r|rzPoolerStartLogits.__init__s3 Yv1155 r{N hidden_statesp_maskrc||d}|:t|tjkr|d|z zd|zz }n|d|z zd|zz }|S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): The final hidden states of the model. p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token should be masked. Returns: `torch.FloatTensor`: The start logits for SQuAD. rNrꌠ9Y>)F)r2squeezerrsr)r#r4r5xs r|forwardzPoolerStartLogits.forwardss JJ} % % - -b 1 1  "4((EM99V$uv~5V$tf}4r{r) rrxryrzrrrs FloatTensorr r;r)r*s@r|r/r/s6/666666 W[".8@AR8S  r{r/c eZdZdZdeffd Z d dejdeejdeej deejd ejf d Z xZ S) PoolerEndLogitsz Compute SQuAD end logits from sequence hidden states. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps` to use. rKcNttj|jdz|j|_tj|_tj|j|j |_tj|jd|_ dS)Nr)epsr) rrrrAr1dense_0Tanh activation LayerNormlayer_norm_epsdense_1r3s r|rzPoolerEndLogits.__init__sz y!3a!79KLL '))f&8f>STTTy!3Q77 r{Nr4 start_statesstart_positionsr5rcZ| | Jd|a|jdd\}}|ddddfdd|}|d|}|d|d}|t j||gd}||}||}|| d}|:t|tj kr|d|z zd|zz }n|d|z zd|zz }|S) a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): The final hidden states of the model. start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*): The hidden states of the first tokens for the labeled span. start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): The position of the first token for the labeled span. p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token should be masked. One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides `start_states`. Returns: `torch.FloatTensor`: The end logits for SQuAD. N7One of start_states, start_positions should be not Nonerrr.rr7r8) r=rWgatherrArsr>rCrDrFr9rr)r#r4rGrHr5slenhszr:s r|r;zPoolerEndLogits.forwardsB<  $(C(C D)D(C(C  &%+BCC0ID#-aaatm<CCBCPPO(//ODDL'..r4<r>s8/888888596:.2 11(1u011"%"23 1 *+ 1   11111111r{r>c eZdZdZfdZ d dejdeejdeejdeejdejf d Z xZ S) PoolerAnswerClassz Compute SQuAD 2.0 answer class from classification and start tokens hidden states. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model. cttj|jdz|j|_tj|_tj|jdd|_dS)NrrF)r) rrrrAr1rArBrCrFr3s r|rzPoolerAnswerClass.__init__)sc y!3a!79KLL '))y!3QUCCC r{Nr4rGrH cls_indexrc`|jd}| | Jd|K|ddddfdd|}|d|d}|L|ddddfdd|}|d|d}n|dddddf}|t j||gd}||}||d}|S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): The final hidden states of the model. start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*): The hidden states of the first tokens for the labeled span. start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): The position of the first token for the labeled span. cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Position of the CLS token for each sentence in the batch. If `None`, takes the last token. One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides `start_states`. Returns: `torch.FloatTensor`: The SQuAD 2.0 answer class. rNrJrrK) r=rWrLr9rArsr>rCrF)r#r4rGrHrUrNcls_token_stater:s r|r;zPoolerAnswerClass.forward/s>:!"%  $(C(C D)D(C(C  &-aaatm<CCBCPPO(//ODDLLRPPL  !!!!T4-077BDDI+222yAAII"MMOO+AAAr111H5O LLL/#BKKK L L OOA   LLOO # #B ' 'r{rO) rrxryrzrrsr<r rQr;r)r*s@r|rSrS sDDDDD596:04 //(/u01/"%"23 / E,- /   ////////r{rSceZdZUdZdZeejed<dZ eejed<dZ eej ed<dZ eejed<dZ eej ed<dZeejed<dS) SquadHeadOutputa Base class for outputs of question answering models using a [`~modeling_utils.SQuADHead`]. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided): Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses. start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Log probabilities for the top config.start_n_top start token possibilities (beam-search). start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Indices for the top config.start_n_top start token possibilities (beam-search). end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search). end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search). cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided): Log probabilities for the `is_impossible` label of the answers. Nlossstart_top_log_probsstart_top_indexend_top_log_probs end_top_index cls_logits)rrxryrzrZr rsr<__annotations__r[r\rQr]r^r_rzr{r|rYrYas*)-D(5$ %,,,7;%"34;;;26OXe./66659x 1299904M8E,-444.2J*+22222r{rYceZdZdZfdZeee ddej de ej de ej d e ej d e ej d e ej d e d e eeej ffdZxZS) SQuADHeadz A SQuAD head inspired by XLNet. Args: config ([`PretrainedConfig`]): The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps` to use. ct|j|_|j|_t ||_t ||_t||_ dSr) rr start_n_top end_n_topr/ start_logitsr> end_logitsrS answer_classr3s r|rzSQuADHead.__init__sc !-)-f55)&11-f55r{) output_typer_NFr4rH end_positionsrU is_impossibler5 return_dictrc|||}||||||fD]1} | -| dkr| d2||||} t } | ||} | | |} | | zdz }|A|?||||}t j}|||}||dzz }|rt| n|fS| \}}}t j |d }tj ||jd \}}|ddd|}tj|d |}|dd|dd}|d|}||dnd}|||| } t j | d }tj ||jd \}}|d|j|jz}|d|j|jz}tjd ||}||||}|s|||||fSt|||||S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): Final hidden states of the model on the sequence tokens. start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Positions of the first token for the labeled span. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Positions of the last token for the labeled span. cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Position of the CLS token for each sentence in the batch. If `None`, takes the last token. is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Whether the question has a possible answer in the paragraph or not. p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token should be masked. return_dict (`bool`, *optional*, defaults to `False`): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Returns: )r5Nrr)rHr5r)rHrUg?)rZrKr)rGr5z blh,bl->bh)rGrU)r[r\r]r^r_)rfr.squeeze_rgrrhrBCEWithLogitsLossrYr6rsoftmaxrstopkrdrSrWrL expand_asrerzeinsum)r#r4rHrjrUrkr5rlrfr:rgloss_fct start_lossend_loss total_lossr_ loss_fct_clscls_lossbszrMrNstart_log_probsr[r\start_top_index_exprGhidden_states_expanded end_log_probsr]r^s r|r;zSQuADHead.forwards7>((v(FF  &=+D%}iO # #=QUUWWq[[JJrNNN`fggJ'))H!,@@Jx M::H$x/14J$)B!..}oir.ss !355 '< MBBhn, 7BU? 3333 U+//11NCs m33Lb3IIO38:!1r444 0 #2";";B"?"?"F"Fr2s"S"S  < r;NOOL'11!44;;Bb"MML%2%<%OQ^`jkk&(;$3&7"/) r{)NNNNNF)rrxryrzrrNrYrrsr<r rQrrrr;r)r*s@r|rbrbs*66666?IYZZZ7;480448.2!^^(^"%"23^  01 ^ E,- ^   01 ^*+^^ e&7 88 9^^^[Z^^^^^r{rbcjeZdZdZdeffd Z d dejdeej dejfdZ xZ S) SequenceSummarya Compute a single vector summary of a sequence hidden states. Args: config ([`PretrainedConfig`]): The config used by the model. Relevant arguments in the config class of the model are (refer to the actual config class of your model for the default values it uses): - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are: - `"last"` -- Take the last token hidden state (like XLNet) - `"first"` -- Take the first token hidden state (like Bert) - `"mean"` -- Take the mean of all tokens hidden states - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2) - `"attn"` -- Not implemented now, use multi-head attention - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction. - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes (otherwise to `config.hidden_size`). - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output, another string or `None` will add no activation. - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation. - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation. rKc.tt|dd|_|jdkrtt |_t|drW|jrPt|dr|j r|j dkr|j }n|j }tj |j ||_t|dd}|rt|n t |_t |_t|dr)|jdkrtj|j|_t |_t|d r+|jdkr"tj|j|_dSdSdS) N summary_typelastattnsummary_use_projsummary_proj_to_labelsrsummary_activationsummary_first_dropoutsummary_last_dropout)rrrrrrsummaryrrr num_labelsr1rrArrC first_dropoutrDropout last_dropoutr)r#rK num_classesactivation_stringrs r|rzSequenceSummary.__init__s #FNFCC   & && %zz 6- . . F63J Fv788 1V=Z 1_e_pst_t_t$/ $0 9V%7EEDL#F,@$GGIZ$jN3D$E$E$E`h`j`j%ZZ 62 3 3 J8TWX8X8X!#F,H!I!ID $JJ 61 2 2 Hv7RUV7V7V " 6+F G GD    H H7V7Vr{Nr4rUrc:|jdkr|dddf}n-|jdkr|dddf}n|jdkr|d}n|jd kr|=tj|d ddddf|jd dz tj }nl|dd}|d |dz z| dfz}| d | d }n|jdkrt| |}||}||}||}|S)ak Compute a single vector summary of a sequence hidden states. Args: hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`): The hidden states of the last layer. cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*): Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token. Returns: `torch.FloatTensor`: The summary of the sequence hidden states. rNrfirstrrMrrKrU.rr,)rr)rrMrs full_liker=longrSrWr.r6rLr9rrrrCr)r#r4rUrs r|r;zSequenceSummary.forward,s   & &"111b5)FF  ' ) )"111a4(FF  & ( ("''A'..FF  + - - !O!#rr111*-!'+a/* &//33==bAA %,,Uimmoo6I-JmN`N`acNdNdMf-fgg "))"i88@@DDFF  & ( (% %##F++f%%((""6** r{rrPr*s@r|rrs2H/HHHHHH<Y]))".);CEDT;U)  ))))))))r{r recursivectr4i}|r#tdstd||d<t|fi|St|drt |jS|S)a Recursively unwraps a model from potential containers (as used in distributed training). Args: model (`torch.nn.Module`): The model to unwrap. recursive (`bool`, *optional*, defaults to `False`): Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers recursively, not just the top-level distributed containers. z0.29.0zsSetting `recursive=True` to `unwrap_model` requires `accelerate` v0.29.0. Please upgrade your version of acceleraterr)rCr;rarrr)rBrrs r|rrXs    0*844 0"J'0{#*5;;F;;; 5( # #  -- -Lr{ci}fd|D}|D]'\|fd|D(|S)zT Expand a device map to return the correspondance parameter name to device. chg|].}||td/Srr)rrrs r|rz%expand_device_map..{s=]]]a!,,|B\B\]1S&&(()]]]r{c^i|])}|ks|dsdk&|*Srnr)rrrrs r|rz%expand_device_map..~sE i i i1qF{{allf<<<>X>X{\bfh\h\hQ\h\h\hr{)rrk)r param_namesrnew_device_maprrs ` @@r|rrvsN]]]];]]]K$**,,   i i i i i i i i     r{cfd|dD}tjt}|D]\}}t |dkrK||vrGd|ddd}t |dkr||vG||||d|DS)zT Returns the list of shard files containing only weights offloaded to disk. cpi|]2\}}||td|3Srr)rrrrs r|rz-get_disk_only_shard_files..sW&*aXYXdXdeqXrXr #l    r{r rrNrc@g|]\}}t|dhk|S)r)r:)rfnamedevicess r|rz-get_disk_only_shard_files..s0 Z Z ZneWWRXQYAYAYEAYAYAYr{)rrrrrr5rr)rrXrr  files_contentrrs ` r|rrs .>|.L.R.R.T.TJ +D11M!+!1!1!3!3@@ X+""{*'D'D((;#4#4S#9#9#2##>??K+""{*'D'Dh&&z+'>???? Z Z (;(;(=(= Z Z ZZr{rwr)TT)FNTru) NNNNNNNFNNNr)rrr~r>importlib.metadatarrrr8rwrrrr8 contextlibr dataclassesrrrr threadingrtypingr r r r r rrrzipfilerrshuggingface_hubr packagingrrrtorch.nnrrtorch.utils.checkpointr activationsrconfiguration_utilsrdynamic_module_utilsr generationrr integrationsrr r!loss.loss_utilsr" pytorch_utilsr#r$r%r&r'r(r)r* quantizersr+r,quantizers.quantizers_utilsr-safetensors_conversionr.rr/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrO utils.hubrPrQrRutils.import_utilsrSrTrUrVutils.quantization_configrWrXrxryupperrYr[rir\r]r^accelerate.hooksr_accelerate.utilsr`rarbrcrdrerfrgrhrfr raccelerate.utils.modelingrk safetensorsrlsafetensors.torchrmr<rnr get_loggerrrrr}r!smdistributed.modelparallel.torch modelparallelrsmdistributed.modelparallelr SMP_VERSIONrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrUr(rrr=rvr~rrrrrrrr rr~rrzr\r/r>rSrYrbrrrrrzr{r|rs+     %%%%%%!!!!!!//////////IIIIIIIIIIIIIIIIIIII >>>>>>////////------''''''11111144444499999999XXXXXXXXXX))))))                    54444444======333333""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""Fgfffffffff NMMMMMMMz~~nc2288:: JNN#6<<BBDDJTTTTTTTTTT333333                      'y'9'A'A,'O'OPP]W]62222IIIIII>%%%%%%============  H % % &333333333FFFFFF - k : :mgmF>S>S S %0////// wW*"w.g,0w.wgng,W*w.g,$88888 %E")5G*G$H % % % %$ry:L/L)M$$$$$.54F)F#G....bNNN / / /(@LRgsSSS%,&'S9>sCxSadSSSSlRTRTRTRTn7; CC3 +,CC5el!234C CCCCL&&&$U\c  ")    ,DSN,S%,=N8O,TYZ^_bcf_gZhjnorjsZsTt,,,,>%T#c(^%c5<>O9P%UZ[_`cdg`h[iknorks[sUt%%%%&KKKK\###.444> "&p7p7p7p7fsXc]ctqtqtqtqtqtqtqtqp w9'w9'w9'w9'w9'bi!1?NTdw9'w9'w9'ts(i(CDD&2*9*E*M*T*T[|+U++O' ##### ###LBBBBBbiBBBJ>>>>> >>>B 33333k33 3