
    קg2              	          U d dl Z d dlmZmZmZmZmZ d dlZd dlm	Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ dd	lmZmZmZmZmZmZmZ dd
lmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& dgZ'ee(         e)d<   dej*        defdZ+dedefdZ,dedede!fdZ-de(dede"fdZ.de(dedede"fdZ/de(dej*        de"fdZ0de(defdZ1d Z2d Z3de(dedee         dee         fdZ4d ede fd!Z5de(d"edee"         fd#Z6dedefd$Z7dej*        dee         fd%Z8de(d&ed'edee         fd(Z9d ee(ef         defd)Z:d*ed+ed,ed-efd.Z;dS )/    N)AnyCallablecastDictList)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__tensorreturnc           	          t          t          j        dgt          |                                           z            |                                           S )Nr   offsetssizes)r   torchSizelensize)r   s    h/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensorr*   '   sG    
A3V[[]]!3!3344FKKMM       shard_mdc                     t          t          j        | j                  t          j        | j                            S Nr"   )r   r%   r&   shard_offsetsshard_sizes)r,   s    r)   _chunk_for_shardr1   -   s9    
8122j-..   r+   sharded_tensorc                     |                                  j        }t          |j        |j        |j        |j        |j                  }t          t          |          ||                                  j
                  S )N)dtypelayoutrequires_gradmemory_format
pin_memorychunk
propertiesr(   )metadatatensor_propertiesr   r4   r5   r6   r7   r8   r   r1   r(   )r2   r,   shard_propertiesr;   s       r)   _sharded_tensor_metadatar?   4   s     &..00B!$&&4&4#.  J x(($$&&+   r+   fqnc                    t          |j        |j        |j                  \  }}t	          j        |          t	          j        |          }}t          t          | |          t          j	        t          t          ||          t          j        |                                          |                                                    S )Nr"   r9   indextypetensor_data)r   shapedevice_mesh
placementsr%   r&   r   r   r   SHARDr   r   r   create_from_tensorto_localr(   )r@   r   r$   r#   s       r)   _create_write_items_for_dtensorrL   H   s    :f(&*; NE7 Z&&
7(;(;7EC)) #&   (:6??;L;LMM
 
 
   r+   c                     t          j        |j                  }t          t	          | |          t
          j        t          ||                    S )NrB   )r%   r&   r/   r   r   r   rI   r?   )r@   r2   r,   r#   s       r)   _create_write_item_for_shardrN   \   sM     j/00GC)) ,^XFF   r+   c                 n   t          j        dgt          |                                          z            }t	          t          | |          t          j        t          t          ||                                          t          j        |          |                                                    S )Nr   r"   r9   rB   )r%   r&   r'   r(   r   r   r   TENSORr   r   r   rJ   )r@   r   r#   s      r)   _create_write_item_for_tensorrQ   g   s    j!s6;;==11122GC))!#&wfkkmmLLL':6BB
 
 
   r+   bytesc                 R    t          t          |           t          j                  S )N)rC   rD   )r   r   r   BYTE_IO)r@   rR   s     r)   _create_write_item_for_bytesiorU   t   s*    C  "   r+   c           
          t          t          j        | t          j        |f          |t          j        |f          t          j        |f                    S N)rD   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rT   r%   r&   rX   dest_offsetrZ   storage_offsetlengths        r)   _create_read_item_for_byteiora   {   sU     !Z//#
N#455
F9%%   r+   c           
          t          t          j        | t          j        |          |t          j        |          t          j        |                    S rW   )r   r   rP   r%   r&   rX   rY   rZ   r[   r\   s        r)   _create_read_item_for_tensorrd      sN      Z--#
?33
7##   r+   checkpoint_mdlocal_chunksc                    g }t          |          D ]\  }}t          |j                  D ]\  }}t          ||          sg }g }	g }
t          ||          D ]F\  }}}}|                    |           |	                    |           |
                    |           G|                    t          t          | |j        |          |	t          | |j        |          ||
                     Č|S )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )saved_shardcurrent_shardrc   )	enumeratechunksr   r   appendrd   r   r#   )r@   re   rf   
read_itemsidxshardstorage_idx
storage_mdr[   rY   r\   dimoffset_for_saved_tensoroffset_for_current_tensorr`   s                  r)   r   r      sD   * J--  
U'01E'F'F 	 	#K5eZHH  OLG =&e  
' 
' ')  &&'>???##$=>>>v&&&&,,S%-EE!-"/Z5G"U"U$3#     '	8 r+   
state_dictc           	      8   g }|                                  D ]\  }}t          |t                    r$|                    t	          ||                     >t          |t
                    rA|                                j        D ]&}|                    t          |||                     't          |t          j
                  r$|                    t          ||                     |                    t          ||                     t          |          S )N)items
isinstancer   rl   rL   r
   r<   shards_metadatarN   r%   TensorrQ   rU   r   )ru   requestsr@   objr,   s        r)   "_create_default_metadata_only_planr}      s   H$$&& 	F 	FSc7## 	FOO;CEEFFFF]++ 	FLLNN: R R <S#x P PQQQQRU\** 	FOO9#sCCDDDDOO:3DDEEEEHr+   objectc                 6    t          d          r                               S t          t                    r! fd                                D             S t          t
          j                  rt                     gS t                     gS )N__create_write_items__c                 <    g | ]}t          |j                  S  )rN   r<   ).0ro   r@   r~   s     r)   
<listcomp>z'_create_write_items.<locals>.<listcomp>   s7     
 
 
 )fenEE
 
 
r+   )	hasattrr   rx   r
   local_shardsr%   rz   rQ   rU   )r@   r~   s   ``r)   _create_write_itemsr      s    v/00 =,,S&999	FM	*	* =
 
 
 
 
,,..
 
 
 	
 
FEL	)	) =-c6::;;.sF;;<<r+   c                     t          | j        | j        | j                  \  }}t	          j        |          t	          j        |          }}t          ||          S r.   )r   rF   rG   rH   r%   r&   r   )r   r$   r#   s      r)   _create_chunk_from_dtensorr      s`    :f(&*; NE7 Z&&
7(;(;7E   r+   c                 L   t          | d          r|                                 }n~t          | t                    rd |                                 D             }nJt          | t
          j                  rt          |           g}nt          dt          |                      |S )N__create_chunk_list__c                 6    g | ]}t          |j                  S r   )r1   r<   )r   ro   s     r)   r   z&_create_chunk_list.<locals>.<listcomp>   s0     
 
 
16U^,,
 
 
r+   zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )
r   r   rx   r
   r   r%   rz   r*   
ValueErrorrD   )r   rf   s     r)   _create_chunk_listr      s    v.// 
3355	FM	*	* 


 
:@:M:M:O:O
 
 
 
FEL	)	) 
1&99:'V' '
 
 	

 r+   mdr|   c                 @   t          |t                    s[	 t          |          }n9# t          $ r,}t          d|  ddt	          |           z             |d }~ww xY wt          | ||          S t          t          |           dt          |           dd          gS )Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   r]   )rx   r   r   r   rD   r   ra   r   )r@   r   r|   rf   exs        r)   _create_read_itemsr     s    b.// 
	-c22LL 	 	 	:3:::GT"XXGGH  	 0RFFF )(--+C00   
 	
s   ' 
A'AAc                 v    dt           fd}dt          fd}dt          j        fd}t	          | |||           dS )zP
    Initializes meta tensor if the meta tensor is DTensor or torch.Tensor.
    valuec                    t          | dd           }|t          j        d          k    rt          j                                        j        }t          t          j        t          |          	                                          }t          j
        |                                 |          }t          j        || j        | j        |                                 |                                           }|S | S )Ndevicemetar   )rG   rH   rF   stride)getattrr%   r   distdistributed_c10d_get_pg_default_devicerD   r   r   current_device
empty_likerK   r   
from_localrG   rH   r(   r   )r   r   device_typenew_local_tensordtensors        r)   dtensor_funcz&_init_state_dict.<locals>.dtensor_func$  s    $//U\&))))/FFHHMK0==LLNN F  %/0@0@PPP ( !- +jjll||~~  G NLr+   c                     t          | dd           }|t          j        d          k    r t          dt	          |            d          | S )Nr   r   zFound unsupported type z for meta device loading.)r   r%   r   RuntimeErrorrD   )r   r   s     r)   sharded_tensor_funcz-_init_state_dict.<locals>.sharded_tensor_func9  sU    $//U\&))))P$u++PPP   Lr+   c                 @   t          | dd           }|t          j        d          k    rtt          j                                        j        }t          t          j        t          |          	                                          }t          j
        | |          }|S | S )Nr   r   r   )r   r%   r   r   r   r   rD   r   r   r   r   )r   r   r   r   s       r)   tensor_funcz%_init_state_dict.<locals>.tensor_funcB  s    $//U\&))))/FFHHMK0==LLNN F %eF;;;FMLr+   N)r   r   r%   rz   _iterate_state_dict)ru   r   r   r   s       r)   _init_state_dictr     s    
G    *3    
5< 
 
 
 
 	    r+   iter_objectr   r   r   c                 t   t          | t                    r |           S t          | t                    r |           S t          | t          j                  r |           S t          | t
          t          t          t          t          j
        f          s| | S t          | t                    r1|                                 D ]\  }}t          |          | |<   | S t          | t          t          f          r6fd| D             }t          | t                    rt          |          }|S dS )a$  
    Iterate through the state dict, applying the given functions to each tensor type
    and update the state dict in place.

    Args:
        iter_object (Any): the target state_dict.
        sharded_tensor_func (Callable): the function to apply to ShardedTensor
        dtensor_func (Callable): the function to apply to DTensor
        tensor_func (Callable): the function to apply to Tensor

    # TODO: let state_dict_util._iterate_state_dict() to support in place option
    so we don't need to have two versions of _iterate_state_dict.
    Nc                 4    g | ]}t          |          S r   )r   )r   vr   r   r   s     r)   r   z'_iterate_state_dict.<locals>.<listcomp>|  s8     
 
 
  <1DkRR
 
 
r+   )rx   r   r
   r%   rz   intfloatstrrR   ioBytesIOdictrw   r   listtuple)r   r   r   r   keyr   rets    ```   r)   r   r   V  sk   ( +w'' |K(((	K	/	/ "";///	K	.	. {;''';eS% DEE	K	&	& %++-- 	 	JC2|%8+   K 	K$	/	/ 
 
 
 
 
 
 
 
 
 k5)) 	**C
 r+   )<r   typingr   r   r   r   r   r%   torch.distributeddistributedr   torch._utilsr   !torch.distributed._shard.metadatar	   'torch.distributed._shard.sharded_tensorr
   torch.distributed.tensorr   torch.distributed.tensor._utilsr   r<   r   r   r   r   r   r   r   plannerr   r   r   r   r   r   
reshardingr   r   r   r   __annotations__rz   r*   r1   r?   rL   rN   rQ   rU   ra   rd   r   r}   r   r   r   r   r   r   r   r+   r)   <module>r      sV   					 2 2 2 2 2 2 2 2 2 2 2 2 2 2              + + + + + + ; ; ; ; ; ; A A A A A A , , , , , , Q Q Q Q Q Q                                        99c 9 9 9el 7K    } 1E    !-:   ( g )    (	+7D   
s 
EL 
Y 
 
 
 
 C    
 
 

 
 
4	4(4 +,4 
(^	4 4 4 4n? x    =S =# =$y/ = = = =w 3G    u| 5I0J    &
C 
] 
 
h 
 
 
 
.4c3h 4C 4 4 4 4n,,, ", 	, , , , , ,r+   