
    קgu$                     Z   d dl Z d dlmZmZmZmZmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ ddlmZmZ dZ G d d	e          Zd
ededefdZ G d d          Z ee          	 ddej        deeej        j                          dej        fd            Zdej        defdZdS )    N)	AnycastDictIterableListNoReturnOptionalSetTuple)_State)DistributedDataParallel   )_get_registrycontract c                   P    e Zd Zd fdZefdej        deej                 deej                 de	ddf
dZ
dd	Zdej        deej                 ddfd
ZddZddZdej        deedf         dee	ef         defdZdej        deej                 dej        dej        fdZ xZS )_ReplicateStatereturnNc                    t                                                       t          j                    | _        d| _        t          j                    | _        | j        | _        g | _        d| _	        d | _
        i | _        g | _        d S )NF)super__init__nnParameterListmodulehas_initialized_param_list_orig_module_param_names_no_sync
_init_args_init_kwargs_comm_hook_args)self	__class__s    c/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/_composable/replicate.pyr   z_ReplicateState.__init__   sz    !#!1!3!3%*-/-=-?-? !K')#59,.*,    r   ignored_modulesignored_paramsprefixc                    t          |          rd S ||v rd S |t          k    r| dnt          }|                    d          D ]A\  }}||vr8| j                            |           | j                            | |            B|                                D ]"\  }}	|                     |	||| |            #d S )N.F)recurse)r)   )_is_fully_sharded_ROOT_MODULE_PREFIXnamed_parametersr   appendr   named_children_collect_params)
r#   r   r'   r(   r)   recurse_prefixnpnamechild_modules
             r%   r2   z_ReplicateState._collect_params    s    V$$ 	F _$$F #&999vLLLL?R 	 ++E+:: 	A 	ADAq&& ''***!((N)?A)?)?@@@"("7"7"9"9 	 	D,  (0$00	 !    	 	r&   c                 \     t          j        d           fd            } |             d S )NT)	recursivec                       j         J   j         j         i  j                                           d _         i  _        d S )N )r    initr!   register_comm_hook)r#   s   r%   
_lazy_initz-_ReplicateState.lazy_init.<locals>._lazy_initA   sT    ?...DIt<$*;<<<##%%% DO "Dr&   )torch_disable_dynamo)r#   r>   s   ` r%   	lazy_initz_ReplicateState.lazy_init@   sF    			.	.	.	# 	# 	# 	# 
/	.	# 	
r&   c                    | j         rd S d| _         |                    dd           }|| _        d |D             }|                                D ]8}t	          |          r'|                    |                                           9ddlm}  |||           | 	                    |||           d|v r\|d         :|d         }t          |t          j                  r|j        dk    rd |d	<   n|g|d	<   nd |d	<   |                    d           t          | j        fi || _        t%          j        | j                  t(                              | j                  _        d S )
NTdevice_meshc                 @    h | ]}|                                 D ]}|S r;   )
parameters).0mr5   s      r%   	<setcomp>z'_ReplicateState.init.<locals>.<setcomp>X   s-    MMMallnnMM!MMMMr&   r   )_localize_dtensor)r(   	device_idcpu
device_ids)r   getr   modulesr-   updaterE   %torch.distributed.tensor.parallel.ddprI   r2   
isinstancer?   devicetypepopr   r   _ddpweakrefref	replicatestate_ddp_weakref)	r#   r   r'   kwargsrC   r(   	submodulerI   rJ   s	            r%   r<   z_ReplicateState.initK   s     	F#jj55MM_MMM)) 	> 	>I ++ >%%i&:&:&<&<===KKKKKK&@@@@V_nEEE&  
 k".";/	i66 79>U;R;R+/F<(( -6;F<(('+|$JJ{###+D,<GGGG	4;K	4J4J	$$111r&   c                 v    | j         D ]\  }} | j        j        |i | | j                                          d S N)r"   rU   r=   clear)r#   	comm_argscomm_kwargss      r%   r=   z"_ReplicateState.register_comm_hookw   sS    &*&: 	D 	D"I{(DI()C{CCCC""$$$$$r&   c                 "    || _         || _        d S r^   )r    r!   r#   argsr[   s      r%   record_init_argsz _ReplicateState.record_init_args|   s    "r&   rd   .r[   c                     | j         s| j        r|                                  | j         | j        _         | j        j        |i |S r^   )r    r!   rA   r   rU   require_backward_grad_sync_pre_forward)r#   r   rd   r[   s       r%   forward_pre_hookz _ReplicateState.forward_pre_hook   sQ     ? 	d/ 	NN37=/@	,%ty%t6v666r&   inputoutputc                 6    | j                             |          S r^   )rU   _post_forward)r#   r   rj   rk   s       r%   forward_post_hookz!_ReplicateState.forward_post_hook   s     y&&v...r&   r   N)__name__
__module____qualname__r   r.   r   Moduler
   	Parameterstrr2   rA   r<   r=   re   r   r   r   ri   r?   Tensorrn   __classcell__)r$   s   @r%   r   r      s       - - - - - -& * 	 RY BL)	
  
   @	 	 	 	*K	*K RY*K
 
*K *K *K *KX% % % %
# # # #7i7',S#X7@DS#X7	7 7 7 7/	/ U\"/ 	/
 
/ / / / / / / /r&   r   rd   r[   r   c                       t          d          )NzGDDP does not support deepcopy. Please use state dict for serialization.)AssertionError)rd   r[   s     r%   unimplemented_deepcopyrz      s    
Q  r&   c                   ,    e Zd Zd ZdeddfdZddZdS )DDPc                 >    | j         d         } |j        |g|R i |S )z
        Override ``__new__`` to remove the DDP class and directly construct
        the original class for cases like indexing into a container module.
           )__mro____new__)clsrd   r[   orig_clss       r%   r   zDDP.__new__   s2     ;q>x:4:::6:::r&   requires_gradient_syncr   Nc                 F    | t                               |           _        dS )a  
        Sets if the module should sync gradients. This can be used to implement
        gradient accumulation without communication.

        Args:
            requires_gradient_sync (bool): Whether to reduce gradients for the
                module's parameters.
        N)rX   rY   r   )r#   r   s     r%   set_requires_gradient_synczDDP.set_requires_gradient_sync   s"     .D)C	&&&r&   c                 n    t                               |           j                            ||f           d S r^   )rX   rY   r"   r0   rc   s      r%   r=   zDDP.register_comm_hook   s/    -44dF^DDDDDr&   ro   )rp   rq   rr   r   boolr   r=   r;   r&   r%   r|   r|      se        ; ; ;	D 	D$ 	D 	D 	D 	DE E E E E Er&   r|   )	state_clsr   r'   c                 p   t           j                            d           d|v rLt          |d         t          t           j        f          s%t          dt          |d                              t          |           rt          d          |i }nt          |          }t          t          t                              |                     }|                     |j        d           |                    dd          }|Sd	d
lm} |                    |          }||k    r2d	dlm}m} |                     |           |                     |           |                     |j                    |j        | |fi | | j        }	dt6          i}
t          d|	j         t:          |	f|
          }|| _        | S )zReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicaterJ   z6Expected device_id to be int or torch.device, but got zGCannot apply `replicate()` on a Module already managed by `fully_shard`NT)with_kwargsrC   r   )_mesh_resources)rI   _reconstruct_dtensor__deepcopy__r|   )r?   _C_log_api_usage_oncerQ   intrR   RuntimeErrorrS   r-   setr   r   rX   rY   register_forward_pre_hookri   rM   torch.distributed.device_meshr   get_root_meshrP   rI   r   register_forward_hookrn   re   r$   rz   rp   r|   )r   r'   r[   rY   rC   r   	root_meshrI   r   r   dctnew_clss               r%   rX   rX      s     
H  !>??? f&-U\/BCC 	7{ 3447 7  
    
U
 
 	
 o..)//&"9"9::E
$$U%;$NNN**]D11KAAAAAA#11+>>	 ##       
 ,,-ABBB(():;;;
  !8999E6?==f=== 
C1
2C'''#sS99GFMr&   c                 0    t          |           }|dS d|v S )z+Check if module is marked with fully_shard.NFfully_shard)r   )r   registrys     r%   r-   r-      s%    V$$HuH$$r&   r^   )rV   typingr   r   r   r   r   r   r	   r
   r   r?   torch.nnr   #torch.distributed._composable_stater   torch.nn.parallelr   r   r   r.   r   rz   r|   rs   rX   r   r-   r;   r&   r%   <module>r      s    R R R R R R R R R R R R R R R R R R R R R R        6 6 6 6 6 6 5 5 5 5 5 5 - - - - - - - -  }/ }/ }/ }/ }/f }/ }/ }/@#      E E E E E E E E4 
O$$$ <@E EIEhux78E Y	E E E %$EP%bi %D % % % % % %r&   