
    קg5              	       ~   d dl Z d dlZd dlZd dlmZmZmZmZmZm	Z	m
Z
mZmZmZ d dlZd dlmZmZ d dlmZ ddlmZ  ej        e          Zdej        deedf         fd	Zd
e	e         dee         deee         ee         f         fdZ G d d          Zd
e	e         de	e         fdZde	e         de	e         de	eeef                  fdZde	ej                 de
e	ej                          de	ej                 dee         fdZ dee         de	eeef                  fdZ!	 dde
e	e"                  fdZ#d Z$dS )    N)
AnycastDequeDictIteratorListOptionalSetTupleUnion)GradientEdgeNode)	Parameter   )map_debug_infotreturnc                     | j         rG| j        @|                     |           }|j        }||j        d         d         S t	          d          | j        S )z
    Get the grad function or grad accumulator for a tensor.

    Accumulate grad nodes are lazily created, so we need to a
    dummy view in order to trigger its creation.
    Nr   zRAttempted to get grad_fn, but got None.Is this being created in a no-grad context?)requires_gradgrad_fnview_asnext_functionsRuntimeError)r   viewed_tr   s      b/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/pipelining/_backward.py_get_grad_fn_or_grad_accr      sf     	 19,99Q<<")!,Q//>  
 y    rootstarget_nodesc                 T   t                      }t                      }t          j                    }| D ]2}|.||vr*|                    |           |                    |           3|r|                                }t          t          t          t          f         |j
                  }|                    dg           }|D ]d\  }}	 |            }
|
|
j        }||v s| ||v r|                    |           :|                    |           |                    |           e|||fS )a   
    This function returns the reverse closure of the given roots,
    i.e. the set of nodes that can be reached from the roots by following the
    reverse edges of the graph. The target_nodes are the nodes that we want to
    include in the closure.
    Nreverse_edges)setcollectionsdequeaddappendpopleftr   r   strr   metadatagetnode)r   r   closurevisited_target_nodesqr+   r)   r!   
holder_refidxreffns               r   reverse_closurer3   (   s@    G55 &((A  G 3 3KKHHTNNN
 yy{{S$Y77 _b99, 	 	OJ*,,C{ BW}}
\!!$((,,,KKOOOHHRLLLL#  $ (((r   c                       e Zd ZdefdZdS )Holderr+   c                     || _         d S N)r+   )selfr+   s     r   __init__zHolder.__init__P   s    			r   N)__name__
__module____qualname__r   r9    r   r   r5   r5   O   s/        T      r   r5   c                    t          j                    }t                      }g }| D ]2}|.||vr*|                    |           |                    |           3|r|                                }|j        D ]\  }}|t          t          t          t          f         |j                  }|                    dg           }t          |          dk    r|                    |           t          |          }	t          j        |	          }
|                    |	           |                    |
|f           ||d<   ||S )Nr!   r   )r#   r$   r"   r&   r%   r'   r   r   r   r(   r   r)   r*   lenr5   weakrefr1   )r   r.   	root_seenreverse_graph_refsr+   r2   r0   r)   r!   holderr/   s              r   construct_reverse_graphrD   T   sB    &((A55I')    I 5 5HHTNNNMM$
 :yy{{* 	: 	:GB~S$Y== (_b A A}%%**HHRLLL$[00
"))&111$$j#%6777,9)  : r   inputsparamsc                    t          | t                                \  }}t                      }t          |          D ]\  }}t          |g|          \  }}|h|d}	|D ]j}
|                    |
d          }|K|d                             |	d                   |d<   |d                             |	d                   |d<   |}	e|	||
<   kt                      }t                      }g }|                                D ]e}	t          |	          |vrR|                    t          |	                     |	                    |	           |                    |	d                   }f|S )a  
    Given a list of inputs and a list of parameters, return a list of parameter
    groups, where each group contains the parameters and the intermediates that
    are connected to the parameters.

    The returned list of parameter groups is a list of dictionaries, where each
    dictionary contains the following keys:
    - "params": a set of parameters
    - "intermediates": a set of intermediates

    The returned list of parameter groups is a list of dictionaries,
    )rF   intermediatesNrF   rH   )
r3   r"   dict	enumerater*   unionvaluesidr%   r&   )rE   rF   inputs_closure_param_groupsiparamr,   intersectedparam_group
input_nodeexistingunion_paramsseen_idsunique_param_groupss                  r   get_param_groupsrZ   m   s    (66NA/3vvLf%% 7 75.wGGg('
 '
 & 		7 		7J#''
D99H#%-h%7%=%=k(>S%T%T",4_,E,K,K0- -) '+6Z((		7 "eeLH#**,, E Ek??(**LLK)))&&{333'--k(.CDDL
 r   stage_outputsoutput_gradsinput_valuesweightsc           	         t          t          dt          t          |                               }t          t          dt          t          |                              }t          t          dt          t          |                              }t	          |          }t          ||          }~|D ]?}	t          |	d                   D ]'\  }
}d }|                     ||	|
                     (@t          d |D                       rr|d | D             }t          j
                            | ||d          }t          |          D ]0\  }
}|j        ||
         |_        |xj        ||
         z  c_        1nd}||fS )z[
    compute the gradients for only the stage inputs with respect to the stage outputs
    NrH   c                       fd}|S )Nc                                          dd           d gt          d                   z  d<   | d         <   d S )NgradsrH   )r*   r?   )grad_inputsrQ   rT   s    r   hookz4stage_backward_input.<locals>.get_hook.<locals>.hook   sS    "w55=04v'89 9 0G, /:K(+++r   r=   )rT   rQ   rd   s   `` r   get_hookz&stage_backward_input.<locals>.get_hook   s)    : : : : : : r   c              3   $   K   | ]}|j         V  d S r7   )r   ).0tensors     r   	<genexpr>z'stage_backward_input.<locals>.<genexpr>   s%      
;
;F6
;
;
;
;
;
;r   c                 6    g | ]}t          j        |          S r=   )torch	ones_like)rg   stage_outputs     r   
<listcomp>z(stage_backward_input.<locals>.<listcomp>   s/       2>--  r   T)rE   grad_outputsretain_graph)listfiltermapr   rD   rZ   rJ   register_prehookallrk   autogradgrad)r[   r\   r]   r^   stage_output_grad_fnsstage_input_grad_fnsweight_grad_fnsrB   rP   rT   rQ   intermediatere   dinputsinps                  r   stage_backward_inputr~      s    )-tS1=AABB) ) (,tS1<@@AA( ( #'tS17;;<<# #O 11FGG#$8/JJL# D D(_)EFF 	D 	DOA|   ))((;*B*BCCCC	D" 
;
;l
;
;
;;;  BO  L .%%%	 & 
 
  -- 	' 	'FAsx"1:GAJ&		' L  r   rP   c           
      p   i }g }t          |           D ]5\  }}t          |          }||f||<   |                    |j                   6|D ]}t	          d |d         D                       }t	          d |d         D                       }	t          d |d         D                       sJ t          j                            ||	t          |d         t	                                          }
t          |d         |
          D ]/\  }}||         \  }}|j        ||_        |xj        |z  c_        0|S )Nc              3   6   K   | ]}t          |d           V  dS r   Nr   )rg   rQ   s     r   ri   z(stage_backward_weight.<locals>.<genexpr>   s=       #
 #
#$LA#
 #
 #
 #
 #
 #
r   rH   c              3   6   K   | ]}t          |d           V  dS r   r   )rg   ws     r   ri   z(stage_backward_weight.<locals>.<genexpr>   s,      PPQl1a00PPPPPPr   rF   c              3   <   K   | ]}t          |          d k    V  dS )r   N)r?   )rg   gs     r   ri   z(stage_backward_weight.<locals>.<genexpr>   s,      ==13q66Q;======r   rb   )ro   )
rJ   r   r&   rw   tupleru   rk   rv   sumzip)r^   rP   grad_acc_to_weightweight_gradsindexweightgrad_accrT   intermediate_edgesweights_edgesdweightsdws               r   stage_backward_weightr      s    L"7++ ) )v+F33'-u}8$FK((((# " "" #
 #
(3O(D#
 #
 #
 
 
 PP+h:OPPPPP==G(<======== >&&[1577;; ' 
 

  H 5x@@ 	" 	"LHb.x8MFE{" r!	" r   outputs_with_grads_idxsc           	         	
 | fd|D              fd|D             	 g 
g 		
fd             t           j                            
	           g }|D ]L}t          |t           j                  r|                    |j                   7|                    d           MnU# t          $ rH}dt                      dt                     dt          |           d	}t          |          |d}~ww xY w|S )
a  
    This is a helper function to:
    1. compute the gradients for the stage inputs, and
    2. accumulate gradients for the stage module's parameters.

    Given the input value(s) and the corresponding gradient for the output
    value(s), compute and accumulate gradients for all parameter values (leaves
    in the autograd trace) as well as return a list of the gradients for the
    input values
    Nc                      g | ]
}|         S r=   r=   )rg   rQ   rm   s     r   rn   z"stage_backward.<locals>.<listcomp>      IIIAQIIIr   c                      g | ]
}|         S r=   r=   )rg   rQ   r\   s     r   rn   z"stage_backward.<locals>.<listcomp>  r   r   c                    t          | t          j                  r| j        s	| j        d S t          |t          j        t          d           f          sJ dt          |                                           |                                |           d S t          | t          t          f          r|d S t          |t          t          f          s*J dt          |            dt          |                       t          |           t          |          k    sJ t          | |          D ]\  }} ||           d S t          | t                    r|d S t          |t                    sJ t          |                                           t          |                                          k    sJ |                                 D ]} | |         ||                    d S d S )Nz)Expected Tensor or None gradient but got z!grad_value expected to have type z	 but got )
isinstancerk   Tensorr   r   typer&   r   rq   r?   r   rI   r"   keys)
output_valgrad_valovgvkextract_tensors_with_gradsoutput_grad_tensorsstage_output_tensorss        r   r   z2stage_backward.<locals>.extract_tensors_with_grads!  s5   *el33 !/ J4F4NF!u|T$ZZ8  P POtH~~OOP P P %++J777#**844444J66 #F!udm  c cbtJ7G7GbbRVW_R`R`bbc c c :#h--7777!*h77 7 7FB..r266667 7J-- 	#F!(D11111:??,,--X]]__1E1EEEEE#** K KA..z!}hqkJJJJK K r   )grad_tensorsz=
        Failed to run stage backward:
        Stage output: z
        Output gradient: z
        Input: z	
        )
rk   rv   backwardr   r   r&   rw   	Exceptionr   r   )rm   r\   r]   r   rc   valeexc_msgr   r   r   s   ``      @@@r   stage_backwardr     s     *IIII1HIIIIIII1HIIIF+  " 	 	 	 	 	 	 	: 	#"<>>> /B 	  	
 	
 	

  	) 	)C#u|,, )""38,,,,""4((((
		  + + +%l33  )66  |,,	   7##*+ s   B	B/ /
D9AC<<Dc                 <    | |S || S t          j        | |          S )z]
    Coalesce two values, even if one of them is null, returning the non-null
    value.
    )rk   r%   )lhsrhss     r   _null_coalesce_accumulater   h  s*    
 {
	
yc"""r   r7   )%r#   loggingr@   typingr   r   r   r   r   r   r	   r
   r   r   rk   torch.autograd.graphr   r   torch.nnr   _debugr   	getLoggerr:   loggerr   r   r3   r5   rD   r(   rZ   r~   r   intr   r   r=   r   r   <module>r      s         V V V V V V V V V V V V V V V V V V V V V V V V  3 3 3 3 3 3 3 3       " " " " " " 
	8	$	$ tTz1B    ,#):#)%(Y#)
3t9c$i #) #) #) #)N       
4: $v,    2/T$Z /d /T#s(^@T / / / /d?!%?!4-.?! u|$?! i 	?! ?! ?! ?!D"i "04T#s(^0D" " " "R 48	] ] &d3i0	] ] ] ]D
# 
# 
# 
# 
#r   