
    קgl                     
   U d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlm	Z	 d dl
mZmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z' d dl(m)Z)m*Z*m+Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d d	l2m3Z3m4Z4 d d
l5m6Z6 d dl7m8Z8 d dl9m:Z; d dl<m=Z= g dZ>dZ?dZ@dZAdZBeeC         ZDee6eejE        eFeGeCf         ZHeeHeeH         eeH         eeCdf         f         ZIeeCeIf         ZJeeJ         ZKeeCeeJeKf         f         ZL eM            ZNee         eOd<   e jP        d             ZQe G d d                      ZRe G d deR                      ZS ejT        d          	 	 dIdejU        deCdeVdeVd eDf
d!            ZW G d" d#          ZXd$ ZYddd%dejU        d&eejZ        j[        d'f         d(eVd)eeejU                          d*eeR         d eSfd+Z\d,eeCeIf         d-eLd.eSd dfd/Z]d0eejU        ejZ        j[        f         d1eCd efd2Z^d3eeCef         d.eSd eeCef         fd4Z_ ej`                    dejU        d.eSd eeCeIf         fd5            Za ej`                    dejU        d3eeCeIf         d.eSd e8fd6            Zbd7ejZ        j[        d dfd8Zcd3eLd eeCeIf         fd9Zdd7ejZ        j[        d3eeCeIf         d.eSd eLfd:Ze ej`                    dejU        d;eejZ        j[        d'f         d.eSd eLfd<            ZfdejU        d7ejZ        j[        d-eLd.eSd eLf
d=Zg ej`                    dejU        d;eejZ        j[        d'f         d3eLd.eSd df
d>            Zhddd%dejU        d)eeejU                          d*eeR         d eeCeIf         fd?Ziddd%dejU        d;eejZ        j[        eejZ        j[                 f         d)eeejU                          d*eeR         d eLf
d@Zjddd%dejU        d;eejZ        j[        eejZ        j[                 f         d)eeejU                          d*eeR         d eeeCeIf         eLf         f
dAZkdejU        d3eeejU        eeCeIf         f         eeCeIf         f         d eeCeIf         fdBZlddCdejU        d,eeCeIf         d*eeR         d e8fdDZmddCdejU        d;eejZ        j[        eejZ        j[                 f         d-eLd*eeR         d df
dEZnddCdejU        d;eejZ        j[        eejZ        j[                 f         d,eeCeIf         d-eLd*eeR         d e8fdFZoeddCdejU        d*eeR         d dfdG            ZpeddCdejU        d;eejZ        j[        d'f         d*eeR         d dfdH            ZqdS )J    N)asdict	dataclassfield)chain)AnyCallablecastDict	GeneratorIterableListno_type_checkOptionalSetTupleUnion)ShardedTensor)_broadcast_state_dict_distribute_state_dict_flatten_state_dict_gather_state_dict_offload_state_dict_to_cpu_unflatten_state_dict)_CHECKPOINT_PREFIX)FullOptimStateDictConfigFullStateDictConfigFullyShardedDataParallelOptimStateDictConfigShardedOptimStateDictConfigShardedStateDictConfigStateDictConfigStateDictType)._get_module_fsdp_state_if_fully_sharded_moduleFSDP_WRAPPED_MODULE)DTensor)_IncompatibleKeys)DistributedDataParallel)tree_map_only)FQNS_TPrimitiveType	ValueTypeDictValueTypeListDictValueTypeOptimizerStateTypeStateDictOptionsget_model_state_dictget_optimizer_state_dictget_state_dictset_model_state_dictset_optimizer_state_dictset_state_dict_flat_paramparam_groupsparamsstater+   _patched_state_dictc               #      K   t          j                    } t          j                     	 d V  | rt          j                     d S d S # | rt          j                     w w xY wN)gc	isenableddisableenable)
is_enableds    c/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/checkpoint/state_dict.py_gc_contextrC   ]   sh      JJLLL 	IKKKKK	 	: 	IKKKK	s   A Ac                   v    e Zd ZU dZdZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed<   dZeed	<   dZeed
<   dS )r/   ap  
    This dataclass specifies how get_state_dict/set_state_dict will work.

    - ``full_state_dict``: if this is set to True, all the tensors in the
      returned state_dict will be gathered. No ShardedTensor and DTensor
      will be in the returned state_dict.

    - ``cpu_offload``: offload all the tensors to cpu. To prevent CPU OOM, if
      ``full_state_dict`` is also true, then only the rank0 will get the
      state_dict and all other ranks will get empty state_dict.

    - ``ignore_frozen_params``: if the value is True, the returned state_dict
      won't contain any frozen parameters -- the ``requires_grad`` is False.
      The default value is False.

    - ``keep_submodule_prefixes`` (deprecated): when ``submodules`` is not None, this option
      indicates whether to keep the submodule prefixes from the state_dict keys.
      or example, if the submodule is ``module.pretrain`` and the full FQN of
      the parameter is ``pretrain.layer1.weight`` of the param. When this option
      is True, the parameter's key in the returned state_dict will be
      ``pretrain.layer1.weight``. If the options is False, the key will be
      ``layer1.weight``.
      Note that if ``keep_submodule_prefixes`` is False, there may be conflicted
      FQNs, hence there should be only one submodule in ``submodules``.

    - ``strict``: the ``strict`` option when ``set_state_dict`` calls
      model.load_state_dict().

    - ``broadcast_from_rank0``: when the option is True, rank0 should receive a
       full state_dict and will broadcast the tensors in the state_dict/
       optim_state_dict one by one to other ranks. Other ranks will receive
       the tensors and shard according to the local shards in the model and
       optimizer. ``full_state_dict`` must be set to True when using this option.
       This option currently only supports DTensor, not the legacy ShardedTensor.
    Ffull_state_dictcpu_offloadignore_frozen_paramsTkeep_submodule_prefixesstrictbroadcast_from_rank0flatten_optimizer_state_dictN)__name__
__module____qualname____doc__rE   bool__annotations__rF   rG   rH   rI   rJ   rK        rB   r/   r/   h   s         " "H "OT!!!K!&$&&&$(T(((FD!&$&&&). $.....rS   r/   c                   v   e Zd ZU  ee          Zeeee	j
        f         eee	j
        f         f         ed<    ee          Zeeee	j
        f         eee	j
        f         f         ed<    ee          Zee         ed<   dZeed<   dZeed<   ej        Zeed<    ee          Zeej                 ed	<   d
S )_StateDictInfo)default_factoryfqn_param_mappingshared_params_mappingsubmodule_prefixesThandle_modelhandle_optimfsdp_contextfsdp_modulesN)rL   rM   rN   r   dictrW   r
   r   strtorchTensorr)   rQ   rX   setrY   r   rZ   rP   r[   
contextlibnullcontextr\   r   listr]   r   nnModulerR   rS   rB   rU   rU      s&         	d### tc5< %(<"== $ $ $
 	d### 4c5< %(<"== $ $ $ $)5#=#=#=C===L$L$'3L(333$)E$$?$?$?L$ry/?????rS   rU   )maxsizeTmodelnameskip_ddp_prefixskip_compiler_prefixreturnc                 z  
 |                     t          d          }d|vr|hS |                    d          }g }| }t          |          D ]\  }}t	          |t
                    r'|dk    sJ |j        }|s|                    |           Bt	          |t                    r|t          |          dz
  k     rZ||dz            t          k    rFd                    |          
t          |t                    }	
r
 d

fd|	j        D             c S t          |t                    }|t          k    r%|                    |           t          ||          }t	          |t          j        j        j                  r(|dk    sJ |j        }|s|                    |           Z|                    |           |t(          j        j        j        k    r'|t          |          dz
  k    rt/          d          t          ||          }d                    |                               t          d          hS )a  
    This API is used to convert the name of a parameter to the FQNs. For FSDP
    without `use_orig_params`, the name of FlatParameter can be mapped to
    multiple original parameters. As a result, the return type of this function
    is `Set[str]`.

    Args:
        module (nn.Module): the root model.
        name (str): the name
        skip_ddp_prefix (bool): whether to skip DDP's `module` prefix

    Returns:
        The canonical FQNs based on the model traversal.
     .module   c                     h | ]} | 	S rR   rR   ).0fqnprefixs     rB   	<setcomp>z_get_fqns.<locals>.<setcomp>   s$    EEES6(3((EEErS   	_orig_modz-Expect `_extra_state` to be the last obj name)replacer   split	enumerate
isinstanceDDPrq   appendFSDPlen_FLAT_PARAMjoingetattr_fqnsr$   r`   _dynamo
eval_frameOptimizedModulerx   rf   modules_EXTRA_STATE_KEY_SUFFIXRuntimeError)ri   rj   rk   rl   	obj_namesfqn_obj_namescurr_objicurr_obj_name
flat_paramrv   s             @rB   	_get_fqnsr      sP   . <<*B//D
$v

3IMH%i00 < <=h$$ 	< H,,,,H" 4$$]333$'' 	<3y>>A%%%)AE*:k*I*I-00$X{;;
 * &\\\FEEEEJ4DEEEEEEx)<==H 333$$]333"8];;%-":"JKK 	< K////)H' 4$$]333  ///
 1 IIII***&'VWWW + #8];;HH]##++,>CCDDrS   c                       e Zd ZdS )_EXTRA_STATEN)rL   rM   rN   rR   rS   rB   r   r      s        DrS   r   c              #      K   t                      dt          j        dt          dt          ffd | d          E d {V  d S )Nrq   curr_fqnrm   c              3   *  K                        |            |r| dnd}|                                 D ]"\  }}|v r
| | } ||          E d {V  #t          |                     d          |                     d                    D ]\  }}|| j        v r| | }||fV  t          | j        dt          j	        j
                  t          j	        j
        k    r.| t          j        j        j         }|t                      fV  d S d S )Nrp   ro   F)recurseget_extra_state)addnamed_childrenr   named_buffersnamed_parameters_non_persistent_buffers_setr   	__class__rf   rg   r   r   rq   r   r   )rq   r   rj   	submodulenew_fqnobjr   visited_moduless         rB   r   z+_iterate_valid_model_state.<locals>.recurse   sy     F###%-5h>>>>2%4466 	3 	3OD)O++!)4))Gwy'2222222222   //1H1HQV1H1W1W
 
 	 	ID# v999!)4))G3, F$&79RSSy() ) "N2:#4#LNNG<>>))))))	) )rS   ro   )rb   rf   rg   r_   r   )ri   r   r   s    @@rB   _iterate_valid_model_stater      sw      &)eeO*	 *S *Y * * * * * * *2 wub!!!!!!!!!!!rS   )
submodulesoptionsoptims.
optim_onlyr   r   c                d   |rt          j        dt                     |r|st          d          |pt	                      }i }i }t          |           D ]\  }}t          |t                    rt          | |          }	|	                    |d          }
|
Et          t          t                   ||                                       |	           ||         ||<   n|	                                ||<   |	D ]}
t          |t                    s|||
<   t          |                                          D ]'\  }}|D ]}
t          t"          j        |          ||
<    (t'                      }|rzt'          |          }|                                 D ]V\  }}||vr
t          | |          }	t+          |	          dk    s
J d            |                    d |	D                        W|j        r|j        st1          d          t3          j        |           }|r|j        rJt7          |j        |j                  }t;          |j        |j        p|j                  }t<          j        }n6tA          |j        	          }tC          |j        	          }t<          j"        }tF          j$        d
             }tK          j&        || |||          }ntF          j'        }tQ          di tS          |          ||||t          tT          tV          j,                 |          | t+          |          dk    dS )zW
    Verify the model and options passed by the user and generates _StateDictInfo.
    zGetting submodules only model/optim state_dict is deprecated and will be removed in 2.5. This feature can be achieved by manually filtering out the state_dict returned from get_state_dict.z;Optimizers are not passed in but optim_only is set to True.Nrr   z)Submodule FQN should only have 1 instancec              3       K   | ]	}| d V  
dS )rp   NrR   )rt   ru   s     rB   	<genexpr>z"_verify_options.<locals>.<genexpr>C  s(      %@%@Ciii%@%@%@%@%@%@rS   z?full_state_dict must be True when broadcast_from_rank0 is True.)offload_to_cpu
rank0_only)r   c              3     K   t          j                    5  t          j        ddt                     t	          j        | |||          5  d V  d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NignorezFSDP.state_dict_type)messagecategoryrq   state_dict_typestate_dict_configoptim_state_dict_config)warningscatch_warningsfilterwarningsFutureWarningr   r   r   s       rB   $fsdp_state_dict_type_without_warningz=_verify_options.<locals>.fsdp_state_dict_type_without_warninga  s#      (** 
 
'&<}    )!$3&7,C	     EEE              	
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s5   5A4AA4A 	 A4#A 	$A44A8;A8r   r   )rW   rX   rY   r\   r]   rZ   r[   rR   )-r   warnr   r   r/   r   r|   r   r   getr	   r   r_   updatecopyre   itemsr`   ra   rb   named_modulesr   rJ   rE   
ValueErrorr   r]   r   rF   r   r"   FULL_STATE_DICTr    r   SHARDED_STATE_DICTrc   contextmanager	functoolspartialrd   rU   r   r   rf   rg   )ri   r   r   r   r   rW   rX   rj   paramfqnsru   param_fqns_rY   rq   r]   r   r   r   r   r\   s                        rB   _verify_optionsr     s     
I 		
 	
 	
  
& 
I
 
 	
 +)++G 	 
 	  2%88 / /ee\** 	%%##E400?S,U344;;DAAA+<U+C!%(( (,yy{{e$ 	/ 	/Ce\22 /).!#&	/ 399;;<< D D 	D 	DC)-elF)C)C!#&&	D $'55 A__
!//11 	A 	ALD&Z''UD))Dt99>>>#N>>>%%%@%@4%@%@%@@@@@# 
G,C 
M
 
 	
 $U++L  /." 	? 3&2w?R! ! ! '?&2#/O73O' ' '# ,;OO 6&2! ! ! 'B&2' ' '# ,>O		"	 	 
#	"	$ !(0+/$;
 
 
 "- 	 	
//	+3-!$ry/<88#^&kkAo	 	 	 	 	rS   model_state_dictoptim_state_dictinfoc                    |j         D ]}t          |          }|
J d            |j        rP| sN|j        sG|j        s@|j        r|j        s2|j        r+|j        s$t          dt          j                    d          |j        r)|s'|j        r|j        s|j        st          d|           |                                 D ]%}t          |v rt          | dt           d          &d S )Nz)Expected a fsdp_state with a fsdp module.z}The option indicates that model state_dict is required to save or load, but model state_dict is empty.rank = dist.get_rank()=rp   zgThe option indicates that model state_dict is required to save, or load but optim state_dict is empty. z
 contains z6. This can happen if the model is not the root module.)r]   r#   rZ   rY   rG   rF   rE   rI   rJ   r   distget_rankr[   keysr   )r   r   r   rq   
fsdp_statekeys         rB   _verify_state_dictr     s   
 # S SCFKK
%%'R%%%%
 	
 
 '
 )	

 !

 '+&:
 K
 )
 *moo* * *
 
 	
  	 	%	*.*>	 .	
 M:JM M  
  $$&&  # * *+ * * *    rS   r   apic                     t          | |          }|t          v r)t          j        t          | j        |          |           }|S )N)self)r   r:   r   r   r   )r   r   calls      rB   _state_dict_fnr     sC    3D""" !<!<3GGGKrS   
state_dictc                     |j         r@|j        rt          j                                        sdnd}t          | |j        |          S |j        rt          |           S | S )NrR   )r   )rF   
ranks_only)rE   rF   r`   distributedis_initializedr   r   )r   r   r   s      rB   _maybe_full_or_cpu_state_dictr     s       $,1,=,L,L,N,NBB 	
 "D$4
 
 
 	
 
	 )*555rS   c                 p   |j         si S |                                5   t          | d                      }d d d            n# 1 swxY w Y   t          |                                          D ]}t          | |          }t          |          dk    sJ ||f            t          t          |                    }||k    rDdt          fd} |||          st          d| d|           |                    |          ||<   |j        rpi }|                                D ]W}|j        D ]M}|                    |          s|j        r||         ||<   +|t          |          d          }	||         ||	<   NX|}|j        rL|                                 D ]7\  }}
|
j        rt          | |          }|D ]}|                    |           8t          |                                          D ]5\  }}t'          j        |          r|j        r|                    |           6t-          ||          S )Nr   rr   rm   c                 T   t          |          t          |           k    rdS |                    d          }|                     d          }d}t          |          D ]I\  }}|||         k    r1|dz  }|t          |          k    r|t          |          dz
  k    c S B|dv rG dS dS )NFrp   r   rr   )rq   rx   T)r   rz   r{   )r   ru   	fqn_split	key_splitfqn_idxkey_idxkey_names          rB   verifyz%_get_model_state_dict.<locals>.verify  s    s88s3xx'' 5IIcNN	IIcNN	)29)=)= % %%GX9W#5551"c)nn44#*c)nnq.@#@@@@ 5!%<<< $uutrS   zAn unexpected key, z, exists. FQN is )rZ   r\   r   re   r   r   r   nextiterrP   r   poprY   
startswithrH   rG   r   requires_gradr   r`   	is_tensoris_metar   )ri   r   r   r   r   ru   r   new_state_dictrv   r   r   ps               rB   _get_model_state_dictr     s     						 ; ;8^E<88::
; ; ; ; ; ; ; ; ; ; ; ; ; ; ; JOO%%&& 2 2$$4yyA~~~T{~~~4::#::D    " 6#s## V"#T#T#Ts#T#TUUU(nnS11JsO $/1??$$ 	> 	>C1 > >~~f-- / >*4S/N3''!#f++--0G.8oN7++> $
  $0022 	$ 	$JC" UC((D $ $s####$ z''))**    Q?1 	 !) 	 NN3(T:::s   AA
Ac           	      .   |j         r	|s|j        st          i i           S i }t          |           D ]\  }}t	          | |          }t	          | |dd          }t          ||          D ]F\  }}	|j        rt          j                    dk    r||	k    r|                    |          ||	<   |||	<   Gd}
|j        s|j	        rd }|
                                D ]H\  }}t          j        |          r/|                                dk    r||j        };||j        k    sJ I|J |t          j        d          k    r t          j                                        }d}
|j        rt#          ||||j                   n|j	        rt'          |||           |
                                D ]
\  }}|||<   |                                5  t+          t           t-          | d          ||j        |
	                    cd d d            S # 1 swxY w Y   d S )
NF)rk   rl   r   metaT)devicerI   r   load_state_dict)r   rI   assign)rZ   rJ   r&   r   r   zipr   r   r   rE   r   r`   r   dimr   distributed_c10d_get_pg_default_devicer   rI   r   r\   r	   r   )ri   r   r   local_state_dictr   valuer   fqns_with_prefixru   fqn_with_prefixr   r   local_states                rB   _load_model_state_dictr    s     )Z )8Q ) R(((077 6 6
U$$$3E
 
 
 %(.>$?$? 	6 	6 C-B15A1E1E((.8nnS.A.A
?+05_--	6 F  *D$8 **0022 	2 	2JCu%% 2%))++//>"\FF!U\1111!!!U\&))))*AACCFF$ 	P!,VDK     ! 	P":/?OOOO 0 6 6 8 8 	* 	*C)JsOO					 
 
4N5"344%dk&  
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   	4H

HHoptimc                    | j         rdS | j        D ]}|t                   D ]}|j          dS | j        D ]2}|t                   D ]"}|j        rt          j        |          |_        #3g }| j        D ]&}d|v r |                    |d                    d|d<   '|                     d           | j        D ]}d|v r|	                    d          |d<   | 
                    d           dS )zH
    Initialize optim states by calling the step() with zero grads.
    Nlrg        )closurer   T)set_to_none)r9   r7   _PARAMSgradr   r`   
zeros_liker~   stepr   	zero_grad)r  param_groupr   lrss       rB   _init_optim_stater  J  sI    {  )   ) 	 	Ez% &	 ) 5 5 ) 	5 	5E" 5"-e44
	5 C) $ $;JJ{4())) #K	JJtJ ) + +; #

K	OOO%%%%%rS   c           
      ,   d }i }t          t          | t                                                             D ]O\  }}t          t          |                                          D ]"\  }} ||           ||t           d| d| <   #Pt          t          | t
                             D ]k}|                    t                    }t          t          t                   |          D ].}|                                D ]\  }}||t
           d| d| <   /l|S )aI  
    This API flattens the optimizer state_dict to support optimizer resharding for
    MPMD, e.g., pipeline parallelism.

    Without the API, the original optimizer state_dict looks like:
    {
        "state": {
            "layer1.weight": {
                "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
            },
            "layer2.weight": {
                "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
            },
        },
        "param_group": [
            {
                "lr": 0.0,
                "betas": (0.9, 0.95), ...,
                "params": ["layer1.weight", "layer2.weight"]
            }
        ]
    }

    With this API, the optimizer state_dict looks like:
    {
        "state.layer1.weight.step": 10,
        "state.layer2.weight.step": 10,
        "state.layer1.weight.exp_avg": SomeTensor,
        "state.layer2.weight.exp_avg": SomeTensor,
        "state.layer1.weight.exp_avg_sq": SomeTensor,
        "state.layer2.weight.exp_avg_sq": SomeTensor,
        "param_group.layer1.weight.lr" : 0.1,
        "param_group.layer2.weight.lr" : 0.1,
        "param_group.layer1.weight.betas" : (0.9, 0.95),
        "param_group.layer2.weight.betas" : (0.9, 0.95),
    }

    Note that if any of the value is a container, like the betas in the example,
    this API won't flattent it.
    c                     t          | t          j        t          t          f          s t          dt          |            d          d S )NzUFlattening optimizer state_dict only supports tensor, int, float states now. Type is rp   )r|   r`   ra   intfloatNotImplementedErrortype)vs    rB   _raise_if_type_not_supportedz?_flatten_optim_state_dict.<locals>._raise_if_type_not_supported  sV    !elC788 	%&77& & &  	 	rS   rp   )
r	   r,   _STATEr   r-   _PGr   r  r   r_   )	r   r  retru   r9   kr  r  r   s	            rB   _flatten_optim_state_dictr  p  sJ   T   !#C=*V*<==CCEE + +
U..4466 	+ 	+DAq((+++)*C6%%C%%!%%&&	+ -z#?? , ,w''S	4(( 	, 	,C#))++ , ,1*+s&&S&&1&&'',	, JrS   c                 z   i }g }t           |t          |i}| j        D ]}|                    t          g i           |t                   D ]}|j        |         D ]}|d         t                   }	t          |	t                    sJ |	                    |           |j        sIi ||<   | j	        |         
                                D ]3}
|t            d| d|
          t          t          ||                   |
<   4t          t          t                   |d         t                             d         }|
                                D ]s}|t          k    r|t           d| d|          }||d         vr||d         |<   9|d         |         |k    r(t          d| d| d| d|d         |          d	          t|S )z
    This API unflattens the state_dict generated by _flatten_optim_state_dict().
    See the docstring of _flatten_optim_state_dict() for more detail.
    rp   r   zaAll the parameters in the same parameter group should have the same saved param_group value. But z is z while other(s) is )r  r  r7   r~   r  rW   r|   re   r   r9   r   r	   r,   r   r_   r   )r  r   r   r9   pg_state
return_osdr  r   ru   r8   
state_namefirst_param_fqnr  r   s                 rB   _unflatten_optim_state_dictr"    s1    E"$H&,eS(%CJ)  "&&& ) 	 	E-e4 
 
!"g.!&$/////c"""* c
"'+e"4"9"9";";  JBL!66C66*66CDc
33J??
 tCy(2,w*?@@C!!## 	 	AG||#====!==>E$$"'Q"aE))"G=LG GOPG GG G4<RLOG G G   *	 rS   
optimizersc                    |j         si S t          i t          g i}|D ]}t          |            t	          |d                      }|j        r|                                5  t          j        | ||          }d d d            n# 1 swxY w Y   |svt          |t                   
                                          D ]H}d|v rB|t                                       |          |t                   |                    dd          <   I|t                   D ]#}d |t                   D             }||t          <   $nqt          t          j        d |j        D                                 }t#          t%          |t'          t)          |                                        }	i |                                 D ]]\  }
}t-          | |
          }t)          |          dk    sJ t/          t1          |                    }||	vrK|	|         }||<   ||<   ^t          |t                   
                                          D ]8}
|
         }|t                                       |
          |t                   |<   9|t                   D ]#}fd|t                   D             |t          <   $|st3          t4          |t                                                 |t                              t3          t8          |t                                                 |t                              |j        r"t3          t>          tA          |                    }tC          ||          S )	Nr   rx   
_orig_mod.ro   c                 :    g | ]}|                     d d          S )r%  ro   ry   rt   r  s     rB   
<listcomp>z)_get_optim_state_dict.<locals>.<listcomp>  s&    JJJ!!))L"55JJJrS   c              3   0   K   | ]}|t                    V  d S r<   )r  )rt   gs     rB   r   z(_get_optim_state_dict.<locals>.<genexpr>  s&      -U-UQaj-U-U-U-U-U-UrS   rr   c                      g | ]
}|         S rR   rR   )rt   pidfqn_pid_mappings     rB   r)  z)_get_optim_state_dict.<locals>.<listcomp>  s    !Q!Q!Q3/#"6!Q!Q!QrS   )"r[   r  r  r  r   r]   r\   r   r   re   r   r   ry   r  r   from_iterabler7   r^   r   ranger   r   r   r   r   r	   r,   r   r-   extendrK   r.   r  r   )ri   r#  r   r   r  osdr  r+  r8   param_pid_mappingr   r   r   ru   r-  groupr.  s                   @rB   _get_optim_state_dictr5    s     	,2BR+@ ,H ,H%   1nUL1133 #	R""$$ ? ?+E5#>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?  #f+**,,-- R R!##?B6{q?Q?QCK		, ; ;<X $ $JJqzJJJ#'

$ %--U-U%BT-U-U-UUUVVF $Ss6{{1C1C%D%D E E O#4466 + +
U ,,4yyA~~~~4::&& 111'.'*$'*$$CK,,..// 8 8%c*#&v;??3#7#7FC  S R R!Q!Q!Q!Q%.!Q!Q!Qg 	],V455<<S[III 0 566==c#hGGGG( 
 9:J K K
 
 ))94@@@s   "BB		B		c           
         i }g }t           |t          |i}i }t          d t          t          |t                                                              D                       r|S |j        D ]}|                    t          g i           |t                   D ]f}	|j	        |	         D ]T}
|
|j
        v rWd}t          t          |t                             D ]3}|
t          t          t                   |t                             v rd} n4nd}|sh|d         t                   }t          |t                    sJ |                    |
           |	j        r)t          t          |t                              |
         ||
<   t          t          |t                             D ]\}|
t          t          t                   |t                             v r-t#          |t                             dz
  |t%          |          <   ]Vht          t          |t                             D ]]}|                    t%          |          d          }|dk    r,|                                D ]\  }}|t          k    r|||         |<   ^|S )a  
    Extract the corresponding optim state_dict from ``optim_state_dict`` for
    ``optim`` and return the result optim state_dict.

    Args:
        model (nn.Module): the root model.
        optim (torch.optim.Optimizer): the optimizer.
        optim_state_dict (Dict[str, ValueType]): the superset optim state_dict that
            contains the optim state_dict of ``optim``.
        info (_StateDictInfo): state dict information.

    Returns:
        The optim state_dict of ``optim``.
    c              3   @   K   | ]}t          |t                    V  d S r<   )r|   r  r(  s     rB   r   z*_split_optim_state_dict.<locals>.<genexpr>7  s=         
1c     rS   FTr  rr   )r  r  allr	   r,   r   r7   r~   r  rW   rX   r-   r   r_   r|   re   r   r   idr   r   )ri   r  r   r   r9   r  r  
pg_mappingr  r   ru   	in_paramsloaded_param_groupr8   idxr   r   s                    rB   _split_optim_state_dictr>    s   * E"$H&,eS(%CJ!#J
  $(8H8P$Q$Q$V$V$X$X        ) V V"&&& ) 	V 	VE-e4 V V$444 %I.2)+;C+@/ / " "* $tCy2DW2M"N"NNN(,I!E O !%I  !"g.!&$/////c"""& T!%m5Ef5M!N!Ns!SE#J*.%'7'<+ + V V& d49.@.IJJJJ=@C=Q=QTU=U
2&8#9#9:	V'V	V4 -/?/DEE ' 'nnR__b11"99%++-- 	' 	'JCg~~!&HSM#		' rS   c           	      ,   |j         sd S |D ]}t          |           |rSt          |v rt          | |||          }n9t	          |t          t          t          t          f         |          |          }ni }|j	        r| 
                                D ];\  }}t          | |          }t          | |d          }	||	k    r/t          |          dk    sJ |                                |	                                |t                   D ]M}
t          t          t          t          f         |
          }fd|t                    D             }||t           <   Nt          t"          |t                             }t%          |                                          D ]2}|v r,|                    |          ||                              <   3=|                                5  t-          j        | ||          }d d d            n# 1 swxY w Y   n|j        rd|_        t3          | |f|          }d|_        d fd}t5          t6          j        ||          }J t;          |          \  }}t;          |          \  }}|j        rt?          ||           ntA          ||           |                                D ]"}||vr||v sJ ||         ||<   ||         ||<   #tC          ||          } tE          |d          |	           d S )
NF)rl   rr   c                 <    g | ]}|                               S rR   r'  )rt   r   ru   fqn_with_compilers     rB   r)  z*_load_optim_state_dict.<locals>.<listcomp>  s5       @CC):;;  rS   Tc                     |                                  dk    r$| j        n| j        k    rt          d          | S )Nr   zDevice mismatch)r   r   r   )tr   s    rB   _devicez'_load_optim_state_dict.<locals>._device  sC    5577Q;;~!"18++():;;;rS   r   r   )r   )#r[   r  r  r>  r"  r	   r
   r_   r+   r]   r   r   r   r   r  r   r  r,   re   r   ry   r\   r   optim_state_dict_to_loadrE   r5  r(   r`   ra   r   rJ   r   r   r   r   )ri   r#  r   r   r  r   original_fqn_r   fqns_with_compilerr+  valr8   	osd_stater  r   rD  flatten_osdosd_mappingflatten_local_osdlocal_osd_mapping	optim_keyr   ru   rA  s                         @@@rB   _load_optim_state_dictrP  e  s      QN QN%    
	"###:5*d$ $   $?4S)^ 4jAA4$ $    " ?	 $)#9#9#;#; X Xa 55%.<e& & &" ---4yyA~~~~hhjj$6$:$:$<$<!)#. * *AtCH~q11C    GJ7|  F $*CLL 0@0HII	inn..// X XAaxxGP}}UVGWGW	!))C1B"C"CDX ""$$  #'#@5"2$ $                ! "	#(D 4UUHdKK#'D F     elG5EFFA%%%':;K'L'L$K3FGW3X3X00( V%k3DVTTTTT&{4EfUUUU
 )--// J J	$555$33333>y3I%i03>y3I%i04!#4    	1u/00<LMMMMMcQN QNs   %HH	H	c                    t                      5  t          | dd||          }t          | |          }t          |i |           |cddd           S # 1 swxY w Y   dS )aH  
    Return the model state_dict of ``model``.

    See ``get_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        submodules (deprecated): Optional[Set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        The state_dict for ``model``.

    :rtype: typing.Dict[str, ValueType]
    rR   Fr   r   r   N)rC   r   r   r   )ri   r   r   r   r   s        rB   r0   r0     s    0 
 
  
 !
 
 
 1==+R666
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
 s   7AAAc                $   t                      5  t          |t          j        j                  r|fnt          |          }t          | |d||          }t          | ||          }t          i ||           |cddd           S # 1 swxY w Y   dS )a  
    Return the combined state_dict for optimizers.

    See ``get_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        submodules (deprecated): Optional[Set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        The state_dict for ``optimizers``.

    :rtype: OptimizerStateType
    TrR  N)	rC   r|   r`   r  	Optimizertupler   r5  r   )ri   r#  r   r   r   r   s         rB   r1   r1     s    6 
     *ek&;<<#ZMMz"" 	
 !
 
 
 1
DII2/666                                   s   A)BB	B	c                H   t                      5  t          |t          j        j                  r|fnt          |          }t          | |d||          }t          | |          }t          | ||          }t          |||           ||fcddd           S # 1 swxY w Y   dS )a  
    Return the model state_dict and optimizers state_dict.

    ``get_state_dict`` can process any module that is parallelized by PyTorch
    FSDP/fully_shard, DDP/replicate, tensor_parallel/parallelize_module, and any
    combination of these parallelisms. The main functions of ``get_state_dict``
    are: 1.) returning a model and optimizer state_dict that can be resharded
    with a different number of trainers and/or different parallelisms.
    2.) hiding the parallelism-specific state_dict APIs. Users don't have to call
    these APIs.
    3.) sanity checking the result state_dict.

    The keys of the result state dictionary are the canonical FQNs (Fully
    Qualified Names).  A canonical FQN refers to the FQN based on a parameter's
    position in an nn.Module hierarchy. More specifically, a canonical FQN to a
    parameter is the FQN returned by ``module.named_parameters()`` or
    ``module.named_buffers()`` when the module is not distributed by any
    parallelisms. Since the optimizer internally uses parameter IDs to represent
    a parameter, there will be a conversion from the parameter IDs to the
    canonical FQNs when calling this API.

    ``get_state_dict`` can also process a module that is not parallelized. In
    such a case, ``get_state_dict`` only performs one function -- converting the
    optimizer parameter IDs to the canonical FQNs.

    Example:
        >>> # xdoctest: +SKIP
        >>> import torch
        >>> from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        >>> from torch.nn.parallel import DistributedDataParallel as DDP
        >>> from torch.distributed.checkpoint.state_dict import get_state_dict

        >>> fsdp_model = FSDP(copy.deepcopy(model))
        >>> fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)
        >>> ddp_model = DDP(copy.deepcopy(model))
        >>> ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)


        >>> ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim)
        >>> fsdp_state_dict, fsdp_optim_state_dict = get_state_dict(fsdp_model, fsdp_optim)

        >>> # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(),
        >>> # the asserts will fail.
        >>> assert ddp_state_dict == fsdp_state_dict
        >>> assert ddp_optim_state == fsdp_optim_state_dict


    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        submodules (deprecated): Optional[Set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        ``Tuple`` that contain model state_dict and optimizer state_dict.

    :rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType]
    FrR  N)
rC   r|   r`   r  rT  rU  r   r   r5  r   )ri   r#  r   r   r   r   r   s          rB   r2   r2     s   L 
 2 2 *ek&;<<#ZMMz"" 	
 !
 
 
 1==0
DII+-=tDDD!11!2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   A;BBBc                   	 |si S t          t          t          |                                                    t          j                  rt          j        dt                     t          t          t          j        t          t          t          f         f         |          }i }|                                D ]\  }}|                                 D ]\  }}||k    rt          | |          }t!          |          dk    s
J d            t          t          |                     d	|                    	fd|                                D                        |S t          t          t          t          f         |          S )NzPassing model_state_dict as a ``Dict[nn.Module, Dict[str, Any]]``is deprecated and will be removed in 2.5. If you need this feature, please preprocessing the model_state_dict to achieve the same functionality.rr   z/FQNs for a submodule should only have 1 elementrp   c                 "    i | ]\  }}|z   |S rR   rR   )rt   subfqnr   rv   s      rB   
<dictcomp>z/_unflatten_model_state_dict.<locals>.<dictcomp>  s#    XXXVf_eXXXrS   )r|   r   r   r   rf   rg   r   r   r   r	   r
   r_   r+   r   r   r   r   r   )
ri   r   cast_state_dictr   r   sub_state_dictrj   mr   rv   s
            @rB   _unflatten_model_state_dictr^  n  s     	$tJOO--..//;; 6" 	
 	
 	
 tBItCN/C$CDjQQ/1)8)>)>)@)@ 
	 
	%I~ ..00 	 	a	>> --4yyA~~~'X~~~ d,,///%%XXXXAUAUAWAWXXX   	 Di(*555rS   )r   c                    t          | |          }t                      5  t          | dd|          }t          |i |           t	          | ||          cddd           S # 1 swxY w Y   dS )a=  Load the model state_dict.

    The counterpart of ``get_model_state_dict`` to set the state_dict to the
    model. See ``set_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        model_state_dict: (Dict[str, ValueType]):
           the model state_dict to load. If the key of the ``model_state_dict``
           is nn.Module, the key is a submodule of ``model`` and the value should
           be the state_dict of the submodule. When loading the state_dict,
           the prefix of the submodule will be append to the state_dict.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
            * **missing_keys** is a list of str containing the missing keys
            * **unexpected_keys** is a list of str containing the unexpected keys

    :type model_state_dict: typing.Dict[str, ValueType]
    rR   Fr   r   N)r^  rC   r   r   r  )ri   r   r   r   s       rB   r3   r3     s    : .I. . 
 E EubUGLLL+R666%e-=tDD	E E E E E E E E E E E E E E E E E Es   5A!!A%(A%c                "   t                      5  t          |t          j        j                  r|fnt          |          }t          | |d|          }t          i ||           t          | |||           ddd           dS # 1 swxY w Y   dS )a  Load the optimizers state_dict.

    The counterpart of ``get_optimizer_state_dict`` to set the state_dict to the
    optimizers. See ``set_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        optim_state_dict: OptimizerStateType:
            the optimizer state_dict to load.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        None

    :type optim_state_dict: typing.OptimizerStateType
    Tr`  N)	rC   r|   r`   r  rT  rU  r   r   rP  )ri   r#  r   r   r   s        rB   r4   r4     s    6 
 	J 	J *ek&;<<#ZMMz"" 	
 ujT7SSS2/666uj2BDIII	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	Js   A(BBBc                d   t          | |          }t                      5  t          |t          j        j                  r|fnt          |          }t          | || |          }t          |||           t          | |||           t          | ||          cddd           S # 1 swxY w Y   dS )a4  Load the model state_dict and optimizers state_dict.

    The counterpart of ``get_state_dict`` to set the state_dict to the model and
    optimizers.  The given ``model_state_dict`` and ``optim_state_dict`` do not
    have to be returned by ``get_state_dict`` but must meet the following
    requirements: 1) all FQNs are canonical FQNs as defined in ``get_state_dict``,
    2) if a tensor is sharded, it must be either a ShardedTensor or DTensor,
    3) optimizer state_dict cannot contain the parameter IDs; the keys should be
    the canonical FQNs.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        model_state_dict: (Union[Dict[nn.Module, Dict[str, ValueType]], Dict[str, ValueType]]):
           the model state_dict to load. If the key of the ``model_state_dict``
           is nn.Module, the key is a submodule of ``model`` and the value should
           be the state_dict of the submodule. When loading the state_dict,
           the prefix of the submodule will be append to the state_dict.
        optim_state_dict: OptimizerStateType:
            the optimizer state_dict to load.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
            * **missing_keys** is a list of str containing the missing keys of the model state_dict.
            * **unexpected_keys** is a list of str containing the unexpected keys of the model state_dict.

    :type model_state_dict: typing.Dict[str, ValueType]
    :type optim_state_dict: typing.OptimizerStateType
    r`  N)r^  rC   r|   r`   r  rT  rU  r   r   rP  r  )ri   r#  r   r   r   r   s         rB   r5   r5     s-   T .I. . 
 E E *ek&;<<#ZMMz"" 	
 :.>*>
 
 
 	+-=tDDDuj2BDIII%e-=tDDE E E E E E E E E E E E E E E E E Es   A9B%%B),B)c                F   t          j        t          | |          fd}|| _        t          j        t          | |          dt
          t          t          f         ffd}|| _        t          
                    |           t          
                    |           dS )a  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model``.

    Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model`` to
    be a partial function to call ``get_state_dict`` and ``set_state_dict``.

    Example:
        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        from torch.distributed.checkpoint.state_dict import patch_model_state_dict

        model = fsdp(model)
        patch_model_state_dict(model)

    Args:
        model (nn.Module): the nn.Module to the model.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.
    Returns:
        None
    )ri   r   c                                    S r<   rR   _state_dict_calls   rB   state_dict_callz0_patch_model_state_dict.<locals>.state_dict_call<      !!!rS   r   c                       |            d S )N)r   rR   r   _load_state_dict_calls    rB   load_state_dict_callz5_patch_model_state_dict.<locals>.load_state_dict_callG      z::::::rS   N)r   r   r0   r   r3   r
   r_   r   r   r:   r   )ri   r   rg  rl  rk  rf  s       @@rB   _patch_model_state_dictrn    s    6 !(  " " " " " 'E%-  ;c3h ; ; ; ; ; ; 1EO,,,011111rS   c                   t          j        t          | ||          fd}t          j        t          | ||          dt          t
          t          f         ffd}t                              |           t                              |           t          |t          j        j                  r|fnt          |          }|D ]}||_        ||_        dS )a  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers``.

    Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers`` to
    be a partial function to call ``get_state_dict`` and ``set_state_dict``.

    Note that if there are multiple optimizers, all of the optimizers will be patched.
    So users only need to call one of the state_dict() to get the full result.

    Example:
        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        from torch.distributed.checkpoint.state_dict import patch_model_state_dict

        model = fsdp(model)
        patch_model_state_dict(model)

    Args:
        model (nn.Module): the nn.Module to the model.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.
    Returns:
        None
    )ri   r#  r   c                                    S r<   rR   re  s   rB   rg  z4_patch_optimizer_state_dict.<locals>.state_dict_callx  rh  rS   r   c                       |            d S )N)r   rR   rj  s    rB   rl  z9_patch_optimizer_state_dict.<locals>.load_state_dict_call  rm  rS   N)r   r   r1   r4   r
   r_   r   r:   r   r|   r`   r  rT  rU  r   r   )ri   r#  r   rg  rl  r  rk  rf  s         @@rB   _patch_optimizer_state_dictrr  R  s   > !( 	  " " " " " &- 	  ;c3h ; ; ; ; ; ; O,,,0111 j%+"788	: 
  5 5* 45 5rS   )TT)rrc   r   r=   r   dataclassesr   r   r   	itertoolsr   typingr   r   r	   r
   r   r   r   r   r   r   r   r   r`   torch.distributedr   r   torch.nnrf   'torch.distributed._shard.sharded_tensorr   #torch.distributed._state_dict_utilsr   r   r   r   r   r   ;torch.distributed.algorithms._checkpoint.checkpoint_wrapperr   torch.distributed.fsdpr   r   r   r   r   r   r    r!   r"   $torch.distributed.fsdp._common_utilsr#   r$   torch.distributed.tensorr%   torch.nn.modules.moduler&   torch.nn.parallelr'   r}   torch.utils._pytreer(   __all__r   r  r  r  r_   r)   ra   r  r  r*   r+   r,   r-   r.   rb   r:   rQ   r   rC   r/   rU   	lru_cacherg   rP   r   r   r   r  rT  r   r   r   r   no_gradr   r  r  r  r"  r5  r>  rP  r0   r1   r2   r^  r3   r4   r5   rn  rr  rR   rS   rB   <module>r     se            				  0 0 0 0 0 0 0 0 0 0                                                     A A A A A A                    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	        - , , , , , 5 5 5 5 5 5 < < < < < < - - - - - -  " 
		Sg}elCKL4&m(<d3CS>TT	 S)^$' #u]4E%EFFG  &)SUU S] * * *    +/ +/ +/ +/ +/ +/ +/ +/\ @ @ @ @ @% @ @ @ T""" !!%	;E ;E9;E
;E ;E 	;E
 ;E ;E ;E #";E|	 	 	 	 	 	 	 	" " "H ,0*.  9%+',- 
 RY( &'    D*3	>**(* * 
	* * * *Zbi)>>? c h    S#X&4	#s(^   $ @;9@;*@;	#y.@; @; @; @;F 2
92
S)^$2
 2
 	2
 2
 2
 2
j#&U[2 #&t #& #& #& #&L=*< =c9nAU = = = =@*; *S)^$* * 	* * * *Z <A9<Aek+S01<A <A 	<A <A <A <A~E9E; E )E 	E
 E E E EP ZN9ZNek+S01ZN #ZN 	ZN
 
ZN ZN ZN ZN@ ,0*.	"  "  " 9"  RY("  &'	" 
 
#y."  "  "  " R ,0*.*  *  * 9* ek+Xek6K-LLM*  RY(	* 
 &'*  *  *  *  * b ,0*.V2 V2 V29V2ek+Xek6K-LLMV2 RY(	V2
 &'V2 4Y!334V2 V2 V2 V2r696d29d3	>&::;T#y.=QQR6 
#y.6 6 6 6J +/	$E $E $E9$E3	>*$E &'	$E
 $E $E $E $EX +/$J $J $J9$Jek+Xek6K-LLM$J )$J
 &'$J 
$J $J $J $JZ +/9E 9E 9E99Eek+Xek6K-LLM9E 3	>*	9E
 )9E &'9E 9E 9E 9E 9E|  +/12 12 12912 &'12 
	12 12 12 12l 
 +/	;5 ;5 ;59;5 ek+S01;5 &'	;5
 
;5 ;5 ;5 ;5 ;5 ;5rS   