
    קg                     t   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlmZ d dlmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d d	lmZmZm Z m!Z! d d
l"m#Z# d dl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.  ej/        e0          Z1d Z2dej3        j4        dej5        fdZ6dej5        dej7        dej7        fdZ8 G d dej3        j9                  Z: G d dej3        j4                  Z; G d de;          Z<ej=        >                    dd           ej=        ?                    dd           d!             Z@ej=        A                    d          d"             Z@ejB        jC        j@        jD        ZEejF        jG        H                    eE           d# ZI G d$ d%e          ZJeeJeeKeJf         f         ZL G d& d'ejM                  ZN G d( d)          ZO G d* d+          ZPd, ZQd- ZRd.ej        jS        d/ejT        fd0ZU G d1 d2ej3        j4                  ZV G d3 d4e          ZW G d5 d6          ZXd7 ZYd8 ZZdej3        j4        d9eeKeWf         fd:Z[	 	 	 dCd;ej3        j4        d<eed=f         d>eeeKef                  d?eeeKeWf                  d@eeejS        gejS        f                  dAeVfdBZ\dS )D    N)defaultdict)Enum)	Parameter	Signature	signature)
MethodType)AnyCallableDictListOptionalSetTupleUnion)ProcessGroup)ExportedProgram)_assign_attr	_AttrKind_sink_paramsInterpreterModule)map_aggregate)split_module   )_null_coalesce_accumulatestage_backward)_outline_submodules)PipeInfo)_PipelineStagec                    |du rd S |du r.t          | t          j                  st          d|            | S t          |t          t
          f          rt          | t          t
          f          st          d|  d|           t          |           t          |          k    rt          d|  d|           t          | |          D ]\  }}t          ||          }||c S t          d|           t          |t                    rt          | t                    st          d|  d|           t          |                                           t          |                                          k    rt          d|  d|           |D ]$}t          | |         ||                   }||c S %t          d|           t          d	t          |           d
          )NFTz/Loss spec must specify a dynamic value but got zOutput value z' must match type of loss specification z) must match length of loss specification z)Did not find loss value in specification z' must match keys of loss specification zUnsupported type z in loss specification)
isinstancefxNodeRuntimeErrortuplelistlenzip_find_loss_from_output_and_specdictsetkeystype)
output_valspec_valoutspecloss_valks         \/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/pipelining/_IR.pyr(   r(   &   s|   5t4*bg.. 	N*NN   (UDM** S*udm44 	
      z??c(mm++
      Z22 	  	 IC6sDAAH# $QxQQRRR(D!! S*d++ 	
      z  !!S%9%999
       	  	 A6z!}hqkRRH# $QxQQRRR
Q4>>QQQ
R
RR    modgc                    d |j         D             }t          |          dk    sJ |d         }|j        d         }d }t          | t                    r)t          |j                  dk    sJ |}t          j        }nY|Et          |t                    r+d|                                v r|d         }d |D             }nd }d }nt          ||          }|}|||fS )Nc                 (    g | ]}|j         d k    |S )outputop).0ns     r3   
<listcomp>z%_find_loss_output.<locals>.<listcomp>V   s$    ;;;!!$(*:*:A*:*:*:r4   r   r   lossc                     i | ]	}||d k    
S )r?    )r<   r2   s     r3   
<dictcomp>z%_find_loss_output.<locals>.<dictcomp>f   s    AAAafAAAr4   )	nodesr&   argsr    TrivialLossWrapper	loss_specr)   r+   r(   )r5   r6   output_loss_value_specoutput_nodesoutput_noder-   generated_spec	loss_nodes           r3   _find_loss_outputrL   U   s   ;;qw;;;L|!!!!q/K!!$JN#)** 0 ;#$$))))	+5		'j$'' 	"Fjoo6G6G,G,G"6*IAAjAAANNI!NN3J@VWW	/k>11r4   rK   rI   c           
          i }t           j                  D ]}|j        dk    r|j        t          j        k    s
J d            t          |j                  dk    s
J d            t          |j                  \  }}||v rt          ||                   nd}t          |dz   |          }d t          |          D             }	||v r t          ||                   D ]
\  }
}||	|
<   ||	|<   t          |	          ||<   |d i|d i fd}                     |          5  t           j                  D ]}|vrfd	}t          j                            |j        |           t          j                            |j        |           |j        d
k    r4||v rK||         }t          fd||         D                       }fdt          ||                   D             }n|f}|         }dg}t#          |t                    s|fn|}                     t&          ||t)          |j                  |d          }t-          |j                  }||_        t          j        |          }|j        }t)          |j                  }t          j        |          }t          |          D ]\  }
} ||||
         j                   	 d d d            n# 1 swxY w Y    S )Ncall_functionzDFound non-getitem call in forward pass. Please report a bug to PiPPy   z:Found malformed getitem call. Please report a bug to PiPPyr   c                     g | ]}d S NrA   )r<   _s     r3   r>   z3_insert_stage_symbolic_backward.<locals>.<listcomp>   s    !E!E!E1$!E!E!Er4   c                 v    | v r.| j         dk    r#                    t          |          |f          }|| <   d S )Nplaceholder)r;   rN   r   )forward_node
grad_valuer6   val_to_grads     r3   assign_or_accumulate_gradzB_insert_stage_symbolic_backward.<locals>.assign_or_accumulate_grad   sO    ;&&<?m+K+K)\*J7 J %/L!!!r4   c                 4                         | d            d S rR   )
setdefault)r=   
live_nodess    r3   add_to_live_nodesz:_insert_stage_symbolic_backward.<locals>.add_to_live_nodes   s    %%a.....r4   call_modulec              3   D   K   | ]}                     |d           V  d S rR   )get)r<   r=   rX   s     r3   	<genexpr>z2_insert_stage_symbolic_backward.<locals>.<genexpr>   s1      (X(XaD)A)A(X(X(X(X(X(Xr4   c                 "    g | ]\  }}|v 	|S rA   rA   )r<   ir=   r\   s      r3   r>   z3_insert_stage_symbolic_backward.<locals>.<listcomp>   s)     / / /"ajr4   r   )stage_outputoutput_gradsinput_valuesoutputs_with_grads_idxs)kwargs)reversedrC   r;   targetoperatorgetitemr&   rD   r$   maxrange	enumerateinserting_beforer!   nodemap_argrh   r    rN   r   r%   all_input_nodesr)   Proxy)r6   rK   rI   tuplesrq   indexed_valuenode_idxexisting_list_sizenew_list_sizereconstructed_listrc   valrY   r]   rd   re   rg   	grad_callkwargs_copygrad_call_proxygradsinput_nodesgrads_proxy
input_noder\   rX   s   `                       @@r3   _insert_stage_symbolic_backwardr   q   s    $&F!!  >  >7o%% ;("2222/ 322
 DI!###K $##&+DI&6&6#M8 /<v.E.EF=)***2   1.@AAM!E!Em0D0D!E!E!E &&'}(=>> 0 0FAs,/&q)) ,0x($)*<$=$=F=!
 T"J5>4EK/ / / / / / 
		K	(	( 0O 0OQW%% /	O /	OD:%%/ / / / / GOODI'8999GOODK):;;;w-''6>>#)$<L#((X(X(X(X6RV<(X(X(X#X#XL/ / / /&/t&=&=/ / /++ %)7L#.t#4L/0c+ &lE::&\OO%  OO"(4(4(,T-A(B(B3J	  ,  	 #9#344#.	 "$(9"5"5',"4#788 huoo%.{%;%; O OMAz--j+a.:MNNNN_/	O0O 0O 0O 0O 0O 0O 0O 0O 0O 0O 0O 0O 0O 0O 0Od Hs   %F.K  K$'K$c                   D    e Zd Zedej        j        fd            Zd ZdS )PipeSequentialsequential_instancec                 (    t          d | D              S )Nc                 6    g | ]}t          j         |          S rA   )copy)r<   ms     r3   r>   z2PipeSequential.from_sequential.<locals>.<listcomp>   s     JJJ	!JJJr4   )r   )r   s    r3   from_sequentialzPipeSequential.from_sequential   s    JJ6IJJJKKr4   c                     t          |           D ]4\  }} ||          }|t          |           dz
  k    rt                       5|S )Nr   )ro   r&   
pipe_split)selfinputrc   modules       r3   forwardzPipeSequential.forward   sL    "4 	 	IAvF5MMECIIM!!r4   N)	__name__
__module____qualname__staticmethodtorchnn
Sequentialr   r   rA   r4   r3   r   r      sR        LUX-@ L L L \L    r4   r   c                   (     e Zd ZdZ fdZd Z xZS )LossWrapperaH  
    LossWrapper is a convenient abstract class that allows you to wrap up both
    your model as well as its loss function and specify the connectivity between
    the inputs, model, loss function, and output value. Example::

        class MyModelWrapper(LossWrapper):
            def forward(self, x, targets):
                model_out = self.module(x)
                loss_value = self.loss_fn(model_out, targets)
                return loss_value

    The above example defines a connectivity where we expect the forward/loss/backward
    training procedure to take two arguments (x and targets), pass x into the module
    to get the output of the feedforward computation, pass the model output and the
    targets value into the loss function, and get and return the loss value, which will
    be backpropagated by PiPPy. The above class would then be instantiated like::

        model = ... # instantiate the model
        loss_fn = torch.nn.MSELoss() # for the sake of demonstration

        wrapper = MyModelWrapper(model, loss_fn)
        pipe = Pipe.from_tracing(wrapper, ...)

    c                 d    t                                                       || _        || _        d S rR   )super__init__r   loss_fn)r   r   r   	__class__s      r3   r   zLossWrapper.__init__  s+    r4   c                      t          d          )NzThis instance of LossWrapper does not have an overriddenforward(). Please implement forward() to specify the arguments, connection between the module and loss, and loss output value.)NotImplementedErrorr   rD   rh   s      r3   r   zLossWrapper.forward	  s    !
 
 	
r4   )r   r   r   __doc__r   r   __classcell__r   s   @r3   r   r      sQ         2    

 
 
 
 
 
 
r4   r   c                       e Zd Zd ZdZdS )rE   c                 X    |                      |          }|                     ||          S rR   )r   r   )r   xtargets	model_outs       r3   r   zTrivialLossWrapper.forward  s%    KKNN	||Iw///r4   TN)r   r   r   r   rF   rA   r4   r3   rE   rE     s$        0 0 0 IIIr4   rE   zpippy::_pipe_splitz() -> ()BackendSelectc                      d S rR   rA   rA   r4   r3   _pipe_splitr   4      4r4   c                      d S rR   rA   rA   r4   r3   r   r   9  r   r4   c                  H    t           j        j                                        S )a  
    pipe_split is a special operator that is used to mark the boundary between
    stages in a module. It is used to split the module into stages. It is a
    no-op if your annotated module is run eagerly.

    Example:
        >>> # xdoctest: +SKIP
        >>> def forward(self, x):
        >>>     x = torch.mm(x, self.mm_param)
        >>>     x = torch.relu(x)
        >>>     pipe_split()
        >>>     x = self.lin(x)
        >>>     return x

    The above example will be split into two stages.
    )r   opspippyr   rA   r4   r3   r   r   G  s    " 9?&&(((r4   c                       e Zd ZdZdZdS )MultiUseParameterConfigr   rO   N)r   r   r   TRANSMIT	REPLICATErA   r4   r3   r   r   [  s        HIIIr4   r   c                   H     e Zd ZdZd	 fd	Zdd fd
Z fdZ fdZ xZS )
DetachExecutorz
    Special interpreter to run the split_gm in testing that detaches all inputs to
    a module invocation. This is needed so that the values at the boundary are
    leaf modules in autograd execution.
    Tc                 ^    d}t                                          ||           i | _        d S )NF)r   r   value_remap)r   r   garbage_collect_valuesr   s      r3   r   zDetachExecutor.__init__j  s2    !&!7888r4   N)initial_envc                D    i | _          t                      j        |d|iS )Nr   )r   r   run)r   r   rD   r   s      r3   r   zDetachExecutor.runo  s&    uww{D:k:::r4   c                       fd}	 t          ||          }t          ||          }t                                          |||          S )Nc                     t          | t          j                  rN| j        rG| j        vr1|                                                     d          }|j        | <   j        |          S | S )NT)r    r   Tensorrequires_gradr   detachrequires_grad_)anew_valr   s     r3   detach_tensorsz2DetachExecutor.call_module.<locals>.detach_tensorst  si    !U\** q D,,,hhjj77==G*1D$Q''**r4   )r   r   r^   )r   rj   rD   rh   r   r   s   `    r3   r^   zDetachExecutor.call_modules  sn    	 	 	 	 		
 
 
 
 

 ww""64888r4   c                      |t           k    r&t          |          } fd|d         D             |d<   t                                          |||          S )Nc                 F    g | ]}j                             ||          S rA   )r   r`   )r<   vr   s     r3   r>   z0DetachExecutor.call_function.<locals>.<listcomp>  s9     & & &/0 $$Q**& & &r4   rf   )r   r)   r   rN   )r   rj   rD   rh   r   s   `   r3   rN   zDetachExecutor.call_function  sj    ^##&\\F& & & &4:>4J& & &F>" ww$$VT6:::r4   )T)	r   r   r   r   r   r   r^   rN   r   r   s   @r3   r   r   c  s              
 &* ; ; ; ; ; ; ;9 9 9 9 94; ; ; ; ; ; ; ; ;r4   r   c                        e Zd ZU d Zeed<   dS )_NodeReferencec                     || _         d S rR   )name)r   r   s     r3   r   z_NodeReference.__init__  s    			r4   r   N)r   r   r   r   str__annotations__rA   r4   r3   r   r     s*            IIIIIr4   r   c                       e Zd Zd Zd ZdS )_LinearNodeListc           
         g | _         |D ]}t          j                            |j        d           }t          j                            |j        d           }t          j        d |j        |j        |j	        |||j
                  }t          j        |j                  |_        | j                             |           d S )Nc                 *    t          | j                  S rR   r   r   r=   s    r3   <lambda>z*_LinearNodeList.__init__.<locals>.<lambda>  s    ^AF=S=S r4   c                 *    t          | j                  S rR   r   r   s    r3   r   z*_LinearNodeList.__init__.<locals>.<lambda>  s    PQPVAWAW r4   )graphr   r;   rj   rD   rh   return_type)serialize_node_listr!   rq   rr   rD   rh   r"   r   r;   rj   r,   r   metaappend)r   	node_listrq   	node_argsnode_kwargsserialize_nodes         r3   r   z_LinearNodeList.__init__  s    #%  	< 	<D	3S3STTI'//$+7W7WXXKWY7{" I  N #')DI"6"6N$++N;;;;	< 	<r4   c           	         t          j                    }i fd}| j        D ]e}t          |j        |          }t          |j        |          }|                    |j        |j        |||j	        |j
                  }||j	        <   f|S )Nc                 L    t          | t                    r| j                 S | S rR   )r    r   r   )argref_str_to_nodes    r3   ref_to_nodez-_LinearNodeList.to_graph.<locals>.ref_to_node  s'    #~.. &sx00
r4   )r;   rj   rD   rh   r   	type_expr)r!   Graphr   r   rD   rh   create_noder;   rj   r   r,   )r   r   r   rq   r   r   
deser_noder   s          @r3   to_graphz_LinearNodeList.to_graph  s    

.0	 	 	 	 	 , 	4 	4D%di==I'[AAK**7{"Y) +  J *4ODI&&r4   N)r   r   r   r   r   rA   r4   r3   r   r     s2        < < <"    r4   r   c                      G d dt           j        j                  } ||           }t          j        ||                                          S )a  
    Custom `__reduce__` method for serialization.
    DO AS I SAY -- NOT AS I DO. This violates the principle that
    GraphModules serialize via code export & re-tracing. We allow
    for this here because **PIPE STAGES SHOULD NOT BE PERSISTED
    TO DISK -- THIS IS ONLY FOR TRANSMISSION VIA RPC**. Persisting
    these instances to disk will expose internal implementation
    details of `fx.Graph` and related data structures and is
    NOT advised.
    c                        e Zd Z fdZ xZS )6_direct_serialization_deserialize.<locals>.DummyModulec                 |    t                                                       | j                            |           d S rR   )r   r   __dict__update)r   bodyr   s     r3   r   z?_direct_serialization_deserialize.<locals>.DummyModule.__init__  s5    GGM  &&&&&r4   )r   r   r   r   r   r   s   @r3   DummyModuler     s8        	' 	' 	' 	' 	' 	' 	' 	' 	'r4   r   )r   r   Moduler!   GraphModuler   )r   rC   r   dummys       r3   !_direct_serialization_deserializer     s]    ' ' ' ' 'eho ' ' '
 KE>%!1!1222r4   c                     t          | j                  }|                    d           t          |t	          | j        j                  ffS )N_graph)r)   r   popr   r   r   rC   )r   serialization_dicts     r3   _direct_serialization_reducer     sH    dm,,8$$$)	_TZ-=>>? r4   gm
new_devicec           	         d}| j         j        D ]&}|j        dk    rfd|j        v r\|j        d         |k    rKt                              d|j         d|j        d          d|            |                    d|           d}t|j        dk    r|                     |j	                  }t          |t          j        j                  rt          ||           t          |t                    rt          |j        |           t                              d	|j	         d
t%          |                      (|r|                                  dS dS )z
    Modify the device argument of all "call_function" nodes in the graph.  This
    is useful for moving the graph to a different device. In particular for
    generator ops, like torch.ones.
    FrN   devicezChanging device of Node z from  to Tr^   z+Skipping device modification for submodule z because it is a N)r   rC   r;   rh   loggerdebugr   update_kwargget_submodulerj   r    r   r!   r   _modify_graph_op_devicer   graph_modulewarningr,   	recompile)r   r   modifiedrq   submods        r3   r  r    sp    H  7o%%4;&&4;x+@J+N+NgtyggH@Ugg[egg   !!(J777W%%%%dk22F&%("677 '
;;;;F$566 '(;ZHHHHn$+nn`dek`l`lnn    
 r4   c                   n   e Zd Zdej        dedefdZd Zdede	j
        j        fdZed	ej        fd
            Ze	 	 	 dde	j
        j        dedee         deee	j        j        ge	j        j        f                  fd            Zd Ze	 dde	j
        j        deedf         deeeef                  defd            Ze	 	 d de	j
        j        deedf         deeeef                  deeej        gej        f                  fd            Zd Zd ZdefdZ	 ddede	j        dee          de!fdZ"dS )!Pipesplit_gm
num_stageshas_loss_and_backwardc                    t           j        j                            |            || _        t          | j                  | _        || _        || _        || _	        |j
        j        D ]w}|j        dv sl|j        |j        fdt          j        fk    sN|j        |j        fdk    s<|j        |j        fdt           fk    s#|j        |j        fdt"          fk    s
J |            xi }| j                                        D ]@\  }}|                                D ]&\  }	}
|                    |
i            |	||
         |<   'Ad |                                D             | _        | j        D ]}|                                D ]\  }}t/          | j        |          }|                    d          }|d d         D ]}t/          ||          }t3          ||d         t5          j        t/          ||d                                        d }|| j        _        d}	 	 d
| }t/          | j        |          }t:          |j        _        |dz  }n# t@          $ r Y d S w xY wD)N>   r9   r^   rU   rN   )call_methodbackwardc                 >    g | ]\  }}t          |          d k    |S )r   )r&   )r<   rS   use_mappings      r3   r>   z!Pipe.__init__.<locals>.<listcomp>7  s9     8
 8
 8
;;!## ###r4   .rP   c                      t          d          )NzHTo run pipeline locally, invoke the Pipe object directly, not `split_gm`)r#   r   s      r3   throwzPipe.__init__.<locals>.throwJ  s    Z  r4   r   Tsubmod_r   )!r   r   r   r   r  r   executorr  r  rF   r   rC   r;   rj   rk   rl   r   r   named_childrennamed_parametersr[   itemsreplicated_paramsgetattrsplitsetattrr   deepcopyr   r   r   
__reduce__AttributeError)r   r  r  r  rF   rq   params_to_users
m_qualnamer5   
p_qualnameparamparam_mappingsubmod_nameparam_qualnamer  atomsatomr  rc   r   s                       r3   r   zPipe.__init__  s    	  &&&(0(6t}(E(E)%:""N( 	 	DCCCGT[)ox?O-PPPGT[)-HHHGT[)o~-NNNGT[)#%>?@ @ @@ @ @ EG#};;== 	@ 	@OJ%(%9%9%;%; @ @!
E**5"5555?&z22@8
 8
"1"7"7"9"98
 8
 8
 "3 	V 	VM/</B/B/D/D V V+^ <<&,,S11!#2#J 3 3D$VT22FFb	4=r9S9S+T+TUUUUV	 	 	
 !& 	$}} 55.J +Q!   	s   0I	 	
IIc           	         |}t          |          dk    rbg }| j        j        j        D ]}|j        dk    r|j        rXt          |j                  dk    r@|                    t          |j        t          j	        |j        d                              mt          j	        }|j        }|j        
                    d          rt          j        }|dd          }n0|j        
                    d          rt          j        }|dd          }|                    t          ||                     t          |          } |j        |i |}	|	                                 |	j                                        } | j        j        | }
|
S )Nr   rU   )defaultz**rO   *r   )r&   r  r   rC   r;   rD   r   r   rj   POSITIONAL_OR_KEYWORD
startswithVAR_KEYWORDVAR_POSITIONALr   bindapply_defaults	argumentsvaluesr  r   )r   rD   rh   executor_args
parametersrq   parameter_kind
param_namer   baress              r3   r   zPipe.forward\  s   v;;??J+1 Q Q7m++y QS^^a%7%7"))% $ ) ?(,	!      *3)H%)[
;11$77 8-6-BN)3ABBJJ![33C88 8-6-EN)3ABBJ")))J*O*OPPP!*--I000BL//11Mdm/
r4   	stage_idxreturnc                 z    |dk     s|| j         k    rt          d| d          t          | j        d|           S )zS
        Return a stage module corresponding to `stage_idx` of the `pipe`.
        r   zInvalid stage index !r  )r  
ValueErrorr  r  )r   r?  s     r3   get_stage_modulezPipe.get_stage_module}  sN     q==I88@I@@@AAAt}&;	&;&;<<<r4   r   c                 "   d}i }| j         j        D ]}}|j        dk    rp|j                            d          rVt          |j        t          d          d                    |j        d<   |                    |j        d                    |dz  }~|S )Nr   r^   r  r?  r   )	r   rC   r;   rj   r2  intr&   r   r[   )r   r  
found_idxsrq   s       r3    _number_and_count_forward_stagesz%Pipe._number_and_count_forward_stages  s    
&(
HN 	  	 Dw-''DK,B,B9,M,M'),T[Y9I9I-J)K)K	+&%%di&<===a
 r4   Nr5   exported_programmulti_use_param_specsplit_policyc                 4  ./01 |                                 }|%t                              d            ||          }t                              |                    d                     i }|j        j        D ]s}|j        dk    rf|                    |j	        |           ||j	                 |k    r:|
                    ||j	                            |j                            |           td}t                      }	t          |j        j                  D ]>\  }
}|j        |j	        fdt          fk    r ||
dz
  k    r|	                    |           |
}?|	D ]}|j                            |           |                                 d	/d
t"          j        f/fd}t'          || |          00j                                         0                                D ]r}t-          |t"          j                  rV|j        j        D ]5}|j        |j	        fdt0          fk    r|j                            |           6|                                 s0                                D ]I\  }}t-          |t"          j                  r*t5          |j                  }0                    ||           Jd }g 1d ..01fd}t9          t;          d 0j        j                            }|D ]s}t=          |j                  dk    r+t                              d|j	         d|j         d           |j        D ]&}|j        dk    sJ  |0|j	        |j	                   'ttA          t                    }| !                    d          "                                D ]-\  }}|tG          |                                       |           .| $                                D ]-\  }}|tG          |                                       |           .i }|D ]c} .| |j	                  \  }}t9          |tG          |                             }|r|||j%        <   E|j	        |j&        v r|j	        g||j%        <   dtA          t8                    }| !                    d          "                                D ]\  }}0                                D ]w\  }}t-          |t"          j                  rX .||          \  }}|rG|E||         '                    |           tQ          ||)                    d          d         |           x1D ]&\  }}	 tU          ||           # tV          $ r Y #w xY w0                                D ]]\  }}t-          |t"          j                  r>tY          ||g            |j        -                                 |                                 ^|"                                D ]9\  }} t]          0|          }t          |           }!d|fg}"|"r|"/                                \  }#}$t-          |$t"          j        t`          f          rI|$j        j        D ]<}|j        dk    r/|#r|#dz   |j	        z   n|j	        }||!v r|!1                    |           =|$                                D ]&\  }%}&|"'                    |#r|#dz   |%z   n|%|&f           '|"|!D ]L}||)                    d          }'}|'dd         D ]}(t]          ||(          }tU          ||'d                    M;|D ]Q}te          j2        |j                  D ]}|j        dk    sJ  |||           0j                            |           R03                                 0j        -                                 0                                 th          5                    0          })d}*|}+|utm          | 0j        |          \  },}-}+|,Gto          0j        |,|-           0                                 d}*t                              d           n,tq          d|          t                              d           t                              d0            ti          0|)|*|+          S )a  
        Additionally, the ``output_loss_value_spec`` value can be specified to disambiguate
        which value in the output of `forward` is the loss value on which PiPPy should apply
        backpropagation. For example, if your ``forward`` returns a tuple ``(loss, model_out)``,
        you can specify ``output_loss_value_spec=(True, False)``. Or, if your ``forward`` returns
        a dict ``{'loss': loss_value, 'model_out': model_out}``, you can specify
        ``output_loss_value_spec={'loss': True, 'model_out': False}``
        NzAuto-splitting modelF)print_outputget_attrrP   rN   r   r   r=   c                 ~    | j         | j        fdt          fk    r"t                              d            dz  S )NrN   zFound pipe_split r   )r;   rj   aten_pipe_split_aliasr  r  )r=   part_idxs    r3   split_callbackz)Pipe._from_traced.<locals>.split_callback  sO    ah%$   ;;;<<<AOr4   c           	      ~    t          |j                  dk    sJ  fdt          |j                  D             }t          |          dk    sJ t	          |j                  }|                    |d                    t          |          |_        t                              d  d| d|d                     dS )z
            Delete reference of `node` from `user`'s arg list.
            Args:
                - node: a `get_attr` node at root.
                - user: a submodule node that uses `node`.
            r   c                 &    g | ]\  }}|k    |S rA   rA   )r<   rc   r   rq   s      r3   r>   zDPipe._from_traced.<locals>.delete_user_reference.<locals>.<listcomp>  s"    LLLfatr4   r   zDeleted z from user z, arg index = N)	r&   rh   ro   rD   r%   r   r$   r  r  )rq   useruse_idxs	args_copys   `   r3   delete_user_referencez0Pipe._from_traced.<locals>.delete_user_reference  s     t{##q((((LLLL	$)(<(<LLLHx==A%%%%TYIMM(1+&&&i((DILLM4MMDMMMM    r4   c                     |                     d          }|d d         D ]%}t          | |          s dS t          | |          } &t          | |d                   s| d fS t          | |d                   }| |fS )Nr  rP   NN)r   hasattrr  )r5   fqnr,  r-  attrs        r3   _recursive_getattr_with_parentz9Pipe._from_traced.<locals>._recursive_getattr_with_parent  s    IIcNNEcrc
 ) )sD)) &%::c4((3b	** !Dy 3b	**D9r4   c           	         |                     d          } 	|          \  }}|d         |j        v }t          |t          j                  sVJ d| dt          j         dt          |           dt          |t          j        j                  r	d| d| dnd	z               |                     |          }t          ||          rJ d
| d|             |rt          |||t          j        d           nt          |||t          j                   t                              d| d|            
                    ||d         f           dS )a'  
            Move a parameter from the root module to a submodule.
            Args:
                root: The root module.
                callee_name: The name of the submodule to move the parameter to.
                param_fqn: The fully qualified name of the parameter to move.
            r  rP   z
Expected 'z' to be z	 but got z It might happen if module 'zu' was passed to some 'leaf function'(see https://pytorch.org/docs/stable/fx.html#fx.wrap). Please inspect usages of 'z' in the traced graph. zModule z already has a parameter named T)	attr_kind
persistent)ra  zMoved parameter r  N)r   _buffersr    r   r   r,   r   r   r  r[  r   r   BUFFER	PARAMETERr  r  r   )rootcallee_name	param_fqnr,  mod_itr	param_val	is_buffercalleer^  r   	to_deletes           r3   move_param_to_calleez/Pipe._from_traced.<locals>.move_param_to_callee  s    OOC((E!?!?y!Q!QGYb	W%55I i66 	 	YYYYYYtIYYY
 ")UX_== D9 D D"+D D D D 	 	 	 ''44F	  Q QPPPYPPQ Q Q
  '.#     '1	    LLHIHH;HHIII guRy122222r4   c                     | j         dk    S )NrN  r:   r   s    r3   r   z#Pipe._from_traced.<locals>.<lambda>U  s    14:+= r4   z
Parameter z used in multiple stages: r  r^   T)	keep_varsr`  z5Pipeline is in training mode, backward pass generatedz@Did not find any loss value according to output_loss_value_spec=z:Pipeline is in inference mode, backward pass not generatedzFull pipe model:
)9r   r  infor  print_readabler   rC   r;   r[   rj   replace_all_uses_with
erase_noder*   ro   r   addr	  r!   r"   r   eliminate_dead_codemodulesr    r   rP  r  r   register_moduler%   filterr&   usersr   
state_dictr  idnamed_buffersr   	constantsr   r!  r   delattrr$  r   lintr  r   r   remover   delete_all_unused_submodulesr  rH  rL   r   r#   )2r5   rI  rJ  rG   rK  tracedget_attr_nodesrq   prev_pipe_split_idxpipe_split_nodes_to_eraserc   rR  	submoduler   
new_submodrX  rn  
attr_nodesrU  
id_to_fqnsr\  tensorinputs_to_stater]  rS   fqnsadded_attributesr  parentchildri  	last_atom
attributesunused_attributesstackscope_mod_name_submodr,  r-  r  r  generated_loss_specrK   rI   r^  rQ  r   rm  s2                                                 @@@@r3   _from_tracedzPipe._from_traced  s	   & "((**#KK.///!\&))FV***>>???
 .0L& 	2 	2Dw*$$))$+t<<<!$+.$66..~dk/JKKKL++D111 !$'EE! !344 	( 	(GAt%/:)FFF&!a%//-11$777&'#- 	* 	*DL##D))))	bg 	 	 	 	 	 	 VS.99'')))  	& 	&I)R^44 &%O1 9 9D-'-2   "224888##%%%$3355 	8 	8OD))R^44 80AA
%%dJ777	 	 	& 	
	 
	 
	>	3 >	3 >	3 >	3 >	3 >	3 >	3B &!=!=u{?PQQRR
 	 	D4:""UUU
UUU   
  w-////$$KK    +6c*:*:
>>D>99??AA 	, 	,KCr&zz"&&s++++,,.. 	, 	,KCr&zz"&&s++++ 13 	; 	;D66sDKHHIAv
2f::.//D ;-1	** 0 :::.2k]	*
 2=T1B1B>>D>99??AA 	D 	DKC % 4 4 6 6 D Dffbn55 D$B$B63$O$OMFED#(=(.55c:::		#r(:FCCCD #, 	 	GY++++!   
 "0022 	# 	#LD&&".11 #V_b999!!###  """
 !1 6 6 8 8 	, 	,D*UD))F #J&\NE 
U#iikktdR^5F$GHH > $
 0 > >7j00?D"U%#+";";$+C"&777 1 8 8 = = =&*&9&9&;&; U UNE7LL"I%#+"5"5E7!STTTT  
U * , ,!'C!#2#J 5 5D%gt44GGr++++	,  	) 	)D	$*-- 2 2w-////%%dD1111K""4((((**,,,::5AA
 %4!-:KU["8; ;7I{$7 $/K  
 !!!(,%TUUUU"Y@VYY   LLUVVV4U44555!	
 
 	
s   U
U#"U#c                 8    | j                                          dS )z~
        Print the pipe in a human-readable format.
        This will print both the root pipe and each stage module.
        N)r  rr  r   s    r3   rr  zPipe.print_readable  s    
 	$$&&&&&r4   example_args.example_kwargsc                     t                               d           	 t          j                            | ||          }n"# t          $ r}t          d          |d }~ww xY w|S )NzTracing model ...a.  It seems that we cannot capture your model as a full graph. Typical reasons include graph breaks, data/shape-dependent control flow, or missing meta kernels for custom operators. You can use our manual pipeline interfaces, or try to fix the graph breaks, see https://pytorch.org/docs/stable/export.html)r  rq  r   export	Exceptionr#   )r5   r  r  epes        r3   _trace_with_exportzPipe._trace_with_export  s     	'(((	$$ BB
  	 	 	P  	 	s   !> 
AAAc                 <   t           j        }d }	 t                              | ||          }t                              | ||||          }|j        }|                                }	t          t          |	                                                    }
t          |
j                  }t          |	j                  }t          |j                  t          |j                  k    rFt                              dt          |j                   dt          |j                   d           npt!          j        |	j        j                  |
j        _        |
j        j        j                            d           |
j        j        _        |
                                 |S )N)rG   rK  zOriginal model takes z) args but the first pipeline stage takes z4. Please provide args to respective pipeline stages.)out_spec)r   r   r  r  r  r  r   nextiterchildrenr   r   r&   r:  r  rq  r   r"  r   _codegenpytree_info_replacer	  )r5   r  r  rK  rJ  rG   rI  piper   r  submod0submod0_sign
model_signs                r3   from_tracingzPipe.from_tracing  s     7@ '+	  22
 
    #9% ! 
 
 !((**tENN,,--.. 11v~..
z$%%\-D)E)EEE KKEJ,A(B(B E E.1,2I.J.JE E E    &*]6<3H%I%IGM"
 &2;;T;JJ M". r4   c                 4    | j                                         S rR   )r  __str__r  s    r3   r  zPipe.__str__B  s    }$$&&&r4   c                 4    | j                                         S rR   )r  __repr__r  s    r3   r  zPipe.__repr__E  s    }%%'''r4   c                 N    t          | j        j        | j        | j                  S )z
        Get information about the pipe.

        Returns
        -------
        PipeInfo
            A dataclass containing information about the pipe.
        )r   r  r  )r   r  r   r  r  r  s    r3   rq  z	Pipe.infoH  s.     -%"&"<
 
 
 	
r4   stage_indexr   groupc                 .   |                      |          }t          |t          j        j                  rt          ||           n*t                              dt          |                      | 	                                }t          |||||          S )z
        Create a `PipelineStage` given a stage index and distributed group.
        The `PipelineStage` can run with `PipelineSchedule`s.
        z*Expected a `torch.fx.GraphModule` but got )rD  r    r   r!   r   r  r  r  r,   rq  r   )r   r  r   r  stage_module	pipe_infos         r3   build_stagezPipe.build_stageW  s     ,,[99 lEH$899 	#L&9999NNQT,=O=OQQ   IIKK	lKFERRRr4   NNNrR   rZ  )#r   r   r   r!   r   rF  boolr   r   r   r   r   rD  r   rH  r   r   MultiUseParamSpecr
   r  rr  r   r	   r   r   r  r  r  r  r   rq  r   r   r   r  rA   r4   r3   r  r    s       L.L L  $	L L L L\  B=# =%(/ = = = = R^    \(  =A# C
 C
X_C
)C
 ''89C

 eh*+UX-AAB
C
 C
 C
 \C
J
' ' '  48 X_CHo !c3h0 
	   \.  48MQ	@ @X_@CHo@ !c3h0@ x(8".(HIJ	@ @ @ \@D' ' '( ( (
h 
 
 
 
& )-	 S  S S  S %	 S
 
 S  S  S  S  S  Sr4   r  c                       e Zd ZdZdZdS )
SplitPointr   rO   N)r   r   r   	BEGINNINGENDrA   r4   r3   r  r  z  s        I
CCCr4   r  c                       e Zd ZeZdS )PipeSplitWrapperN)r   r   r   r  rA   r4   r3   r  r    s        JJJr4   r  c                 8    t                        | j        |i |S rR   )r   _orig_forwardr   s      r3   _split_before_forwardr    s#    LLL4t.v...r4   c                 `    	  | j         |i |t                       S # t                       w xY wrR   )r  r   r   s      r3   _split_after_forwardr    s5    !t!42622
s    -r0   c                 `   |                                 D ]\  }}|                    d          }| }t          |d d                   D ]\\  }}	 t          ||          }# t          $ r9}t	          d| dd                    |d |dz                                 |d }~ww xY wt          ||d                   }	|	j        |	_        |t          j	        k    rt          t          |	          |	_        |t          j        k    rt          t          |	          |	_        
t          d          d S )Nr  rP   zSpecified target z referenced nonexistent module r   zUnknown split point type.)r  r   ro   r  r$  joinr   r  r  r  r   r  r  r  rC  )
r5   r0   qualname
split_typer,  predecessor_modulerc   r-  r  mod_to_wraps
             r3   annotate_split_pointsr    sc    $

 : :*s##  ss,, 	 	GAt%,-?%F%F""!   $E E E*-((51q5>*B*BE E   0%)<<$/$7!---",-BK"P"PK:>))",-A;"O"OK8999': :s   A  
B#*4BB#r   mb_args.	mb_kwargs
split_specrK  r@  c                     ||t          d          |-t          | |           t                              | ||          S t                              | |||          S )a9  
    Split a module based on a specification.

    See `Pipe` for more details.

    Arguments
    ---------
    module:
        The module to be splitted.
    mb_args:
        Example positional inputs, in micro-batch form.
    mb_kwargs:
        Example keyword inputs, in micro-batch form. (default: `None`)
    split_spec:
        A dictionary using submodule names as split marker. (default: `None`)
    split_policy:
        The policy to use for splitting the module. (default: `None`)

    Returns
    -------
    A pipeline representation of class `Pipe`.
    NzQCannot specify both `split_spec` and `split_policy`. Please use only one of them.)r5   r  r  )r5   r  r  rK  )rC  r  r  r  )r   r  r  r  rK  s        r3   pipeliner    s    : ,":_
 
 	
 fj111   $ ! 
 
 	
    $%	 ! 
 
 	
r4   r  )]r   loggingrk   collectionsr   enumr   inspectr   r   r   typesr   typingr	   r
   r   r   r   r   r   r   r   torch.fxr!   torch.distributedr   torch.exportr   torch.export.unflattenr   r   r   r   torch.fx.noder   torch.fx.passes.split_moduler   	_backwardr   r   
_unflattenr   _utilsr   stager   	getLoggerr   r  r(   r   r   r   rL   r"   r   r   r   r   rE   librarydefineimplr   register_faker   r   r/  rP  rq   _side_effectful_functionsru  r   r   r   r  Interpreterr   r   r   r   r   r   r   r  r  r  r  r  r  r  r  rA   r4   r3   <module>r     s      # # # # # #       3 3 3 3 3 3 3 3 3 3       I I I I I I I I I I I I I I I I I I I I        * * * * * * ( ( ( ( ( (            ( ' ' ' ' ' 5 5 5 5 5 5 @ @ @ @ @ @ @ @ + + + + + +       ! ! ! ! ! ! 
	8	$	$,S ,S ,S^258? 2rx 2 2 2 28i	xiwi i i i iX
 
 
 
 
UX( 
 
 
%
 %
 %
 %
 %
%(/ %
 %
 %
P       >   ): 6 6 6 (/::  ;: 122  32
 	3;   ! % %&; < < <) ) )(    d   
 14=T8T3UUV 1; 1; 1; 1; 1;R^ 1; 1; 1;h       * * * * * * * *Z3 3 3,           Fj	S j	S j	S j	S j	S58? j	S j	S j	SZ              
/ / /
  :ux :d3
?6K : : : :6 +/26IM1
 1
HO1
38_1
 S#X'1
 c:o./	1

 8R^$4bn$DEF1
 
1
 1
 1
 1
 1
 1
r4   