
    Χg9                     P   d dl Z d dlZd dlmZmZmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlZd dlmZmZ d dlZd dlZ ej        e          Zd Zd Z e            Zd Zeee ed          ee fZ!dZ"d	 Z# G d
 de          Z$ddddZ%d Z&d Z'd Z(d Z)dZ*dZ+d Z,d Z-d Z.dS )    N)AnyDictList
NamedTupleOptionalTuple)NamedTemporaryFile)_frames_fmt_block_extrac                      dfd}t          j        |            fdt          j                                       fd}|S )NTc                      d d S )NF )enableds   S/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/utils/viz/_cycles.pydisablez observe_garbage.<locals>.disable   s         c                     sd S | dk    r t          j        t           j                   d S | dk    r5t          j                    dgfd}t          j        |           d S d S )NstartstopFc                  ,   d         sdd<   nt          j                   d	 d         dk    rt          j                      t          j                   t          j                                         t          j        d           t          j        	                                }t          j                     t          j        	                                }||k    rt                              d||z
             dn# dw xY w | i |S d S )Nr   TF
generation   z.CUDA Memory changed during GC, %d bytes freed.)sys
setprofilegccollectgarbageclear	set_debugtorchcudamemory_allocatedloggerwarning)	argskwargsbeforeafterr   infoobserver
orig_traceself_returns	       r   
do_collectz8observe_garbage.<locals>.gc_callback.<locals>.do_collect#   s   "1~ '%)KNNN:...#G'  -22JLLL ,,,
((*** Q!&!<!<!>!>
 %
 ; ; = =!U??"NN+[]cfk]klll"&$)%:t6v666 *)s   CD D)r   r   DEBUG_SAVEALLr   
getprofiler   )phaser)   r-   r+   r,   r   r*   s    ` @@r   gc_callbackz$observe_garbage.<locals>.gc_callback   s     	FGL)*****f__))J 'K7 7 7 7 7 7 7 7 7: N:&&&&&C _r   c                  F    t           j                                        d S N)r   	callbacksremove)r1   s   r   r5   zobserve_garbage.<locals>.removeE   s    
K(((((r   )atexitregisterr   r4   append)r*   r   r5   r   r1   s   `  @@r   observe_garbager9      s    G    
 OG'' '' '' '' '' ''R L$$$) ) ) ) )Mr   c                  N    dd} t           |             j        d                   S )Nc                       fdS )Nc                       S r3   r   xs   r   <lambda>z+_get_cell_type.<locals>.f.<locals>.<lambda>Z   s    q r   r   r=   s   `r   fz_get_cell_type.<locals>.fY   s    yyyr   r   r3   )type__closure__)r@   s    r   _get_cell_typerC   X   s1       "###r   c                     i fd fdfd}fd} fd} fd} fd}fd} fd	} fd
}fd}	t           |t          |t          |t          |t          |t
          j        |t
          j        |t          |t
          j	        |t          j        |t
          j        |	i}
t                     j        D ]}||
v r |
|                       dd           t           t                    r d           S )a%  
    Return known information about references held by the given object.

    Returns a mapping from referents to lists of descriptions.  Note that there
    may be more than one edge leading to any particular referent; hence the
    need for a list.  Descriptions are currently strings.

    c                 t                         t          |          g                               |            d S r3   )
setdefaultidr8   )nameobj
referencess     r   add_referencez+annotated_references.<locals>.add_referencej   s3    bggr**11$77777r   c                  f    | D ],}t          |          r |t          |                     -d S r3   )hasattrgetattr)attrsattrrK   rI   s     r   	add_attrsz'annotated_references.<locals>.add_attrsm   sK     	8 	8DsD!! 8dGC$6$6777	8 	8r   c                  B    	   d           d S # t           $ r Y d S w xY w)Ncell_contents)
ValueErrorrQ   s   r   add_cell_referencesz1annotated_references.<locals>.add_cell_referencesr   sB    	Io&&&&& 	 	 	
 DD	s    
c                  .      ddddddddd		  	         d S )
N__defaults__rB   __globals____code____name__
__module____doc____qualname____annotations____kwdefaults__r   rU   s   r   add_function_referencesz5annotated_references.<locals>.add_function_references|   s=    	.!#"		$ 		$ 		$ 		$ 		$r   c                  R    t                    D ]\  } } d|  d|           d S )N[])	enumerate)positionitemrK   rI   s     r   add_sequence_referencesz5annotated_references.<locals>.add_sequence_references   sC    'nn 	1 	1NHdM/h///40000	1 	1r   c                                                       D ].\  } } d|             dt          |            d|           /d S )Nkeyrb   rc   )itemsrepr)ri   valuerK   rI   s     r   add_dict_referencesz1annotated_references.<locals>.add_dict_references   sa    ))++ 	3 	3JCM%%%%M*d3ii***E2222	3 	3r   c                  *    D ]}  d|            d S )Nelementr   )eltrK   rI   s    r   add_set_referencesz0annotated_references.<locals>.add_set_references   s1     	* 	*CM)S))))	* 	*r   c                  "      ddd           d S )N__self____func__im_classr   rU   s   r   add_bound_method_referencesz9annotated_references.<locals>.add_bound_method_references   s    	*j*55555r   c                      t                    t          j        u r=t          j                  } t          |           dk    r| d         } d|           d S d S d S )N   r   __callback__)rA   weakrefrefr   get_referentslen)	referentstargetrK   rI   s     r   add_weakref_referencesz4annotated_references.<locals>.add_weakref_references   sh     99##(--I9~~"""1nf55555	 $#""r   c                      j         }  dddddd           t          |           t          u r.j                                         D ]\  }} d| |           d S d S )Nf_backf_code
f_builtins	f_globalsf_tracef_localszlocal )r   rA   dictrj   )r   rH   localrQ   rK   rI   s      r   add_frame_referencesz2annotated_references.<locals>.add_frame_references   s    <	(HlKJWWW >>T!!"|1133 6 6eotoou5555 "!6 6r   c                  "      ddd           d S )N__objclass__r[   __doc__r   rU   s   r    add_getset_descriptor_referencesz>annotated_references.<locals>.add_getset_descriptor_references   s    	.*i88888r   __dict__	__class____mro__)tuplelistr   set	frozensettypesFunctionType	FrameTypeCellType
MethodTyperz   r{   GetSetDescriptorTyperA   r   
isinstance)rI   rV   r`   rg   rm   rq   rv   r   r   r   type_based_referencestype_rQ   rK   rJ   s   `           @@@r   annotated_referencesr   _   s    (*J8 8 8 8 88 8 8 8 8 8
    
$ 
$ 
$ 
$ 
$1 1 1 1 1 13 3 3 3 3 3
* * * * * *6 6 6 6 66 6 6 6 6 66 6 6 6 6 6 69 9 9 9 9 	&%!%3-%5+"$D c" + +)))(!%(***Ij+&&&#t 	)r       c                    d }t          | t                    rt          |           S t          |           j        dk    r
d| j         S t          | t
          j                  r%	 | j        j        }n# t          $ r d}Y nw xY wd| S t          | t                    rd ||            dS t          | t                    rd ||            d	S t          | t                    rd
t          |            dS t          | t
          j                  r
d| j         S t          | t                    r
d| j         S t          | t          j                  r! |             }|dS dt#          |          dS t          | t
          j                  rG| j        j        }t          |          t*          k    rd|t*          dz
   d         z   }d| d| j         S dt          |           j         dt          |           j         S )zx
    Return a string to be used for Graphviz nodes.

    The string should be short but as informative as possible.
    c                     d                     d t          t          d          |           D                       }t          |           dk    r| dt          |           dz
   }|S )N,c              3      K   | ]?\  }}t          |t                    rt          |          nt          |          j        V  @d S r3   )r   
BASE_TYPESrk   rA   r[   ).0ir>   s      r   	<genexpr>z=object_annotation.<locals>.format_sequence.<locals>.<genexpr>   sK      rrW[WXZ[:a#<#<RQ$q''BRrrrrrrr      z, ...)joinzipranger}   )rI   bodys     r   format_sequencez*object_annotation.<locals>.format_sequence   si    xxrr_bchijckckmp_q_qrrrrrs88a<<//SA//Dr   functionz	function
z<anonymous>zinstancemethod
rb   rc   ()zdict[zmodule
ztype
Nzweakref (dead referent)zweakref to id 0xr>   z...   zframe
:zobject
.)r   r   rk   rA   r[   r   r   rt   __qualname__AttributeErrorr   r   r   r}   
ModuleTyperz   r{   rG   r   r   co_filenameFRAME_FILENAME_LIMITf_linenor\   )rI   r   	func_namereferentfilenames        r   object_annotationr      sO      #z"" CyyCyyZ''*CL***	C)	*	* F	&1II 	& 	& 	&%III	&-)---	C		 F*??3''****	C		 F*??3''****	C		 F"s3xx""""	C)	*	* F(#,(((	C		 F&&&&	C	%	% F355,,6bll6666	C	)	) F:)x==///x*>*B(C(D(DEEH222CL222E$s)).EEc1CEEEs   %A2 2B Bc                   `    e Zd ZU eed<   ee         ed<   eed<   eeee	f                  ed<   dS )Nodelabelcontextroot
referrentsN)
r[   r\   r   strr^   r   boolr   r   intr   r   r   r   r     sM         JJJc]
JJJU38_%%%%%%r   r   )r   filterc                   t                      t          fd| D             }d | D             }d t          |           D             }| D ]}|t          |                   }||         }t	          |          }	t          j        |          D ]}
t          |
          }|                    |d           }|*||         }|	                    |dg          }||                             |           |D ]}|j	                            ||f           d t          |          D             }t                      }|rM|                                }||v r|                    |           ||         }|                    |           |Mi g }t          |          D ]0\  }}||v r't                    |<   |                    |           1|D ]}fd|j	        D             |j	        d d <    |S )Nc           	      r    g | ]3}t          t          |           |           |          g           4S r   )r   r   )r   rI   r   r   s     r   
<listcomp>z create_graph.<locals>.<listcomp>  s@    \\\ST#C((''#,,sRHH\\\r   c                     g | ]}g S r   r   r   rI   s     r   r   z create_graph.<locals>.<listcomp>  s    &=&=&=cr&=&=&=r   c                 4    i | ]\  }}t          |          |S r   )rG   )r   r   rI   s      r   
<dictcomp>z create_graph.<locals>.<dictcomp>  s$    >>>C"S''1>>>r   ?c                 &    g | ]\  }}|j         |S r   )r   )r   r   ns      r   r   z create_graph.<locals>.<listcomp>   s#    :::tq!16::::r   c                 2    g | ]\  }}|v 	||         fS r   r   )r   r   idxid_to_filtered_ids      r   r   z create_graph.<locals>.<listcomp>0  s>     8 8 8+s!%666 "#4S#9:666r   )cuda_allocation_contextis_cuda_tensorrd   rG   r   r   r|   getr8   r   r   popaddextendr}   )objectsr   r   nodesnode_referrers
id_to_noderI   fidxr@   rJ   	referrentridtidxtlabelsr   	to_searchto_keepr   	referrersfilteredr   r   r   s    ``                    @r   create_graphr     sq   )++~\\\\\T[\\\E&=&=W&=&=&=N>>9W+=+=>>>J 3 3"S''"$K)#..
)#.. 		3 		3IY--C>>#t,,D|dA^^C#//F4 ''--- 3 3##UDM22223		3 ;:y//:::IeeG
 $mmoo'>>C"3'	###  $ )+H%    1<<#&'8#9#9a OOA 8 88 8 8 8/0|8 8 8QQQ Or   c                 *    t          j        |           S r3   )jsondumps)r   s    r   escaper   5  s    :a==r   c                     t          | t          j                  o&| j        ot          | t          j        j                   S r3   )r   r    Tensoris_cuda_subclasses
FakeTensor)rI   s    r   r   r   9  s4    c5<((nS[nCQVQbQmAnAn=nnr   c                      t           j        j                                        } i | d         D ]C}|d         }|d         D ]0}|d         dk    rt	          |          \  }}||<   ||d         z  }1Dfd}|S )Nsegmentsaddressblocksstateactive_allocatedsizec                     t          |           ra|                                                                 }                    |          }|$d                    t          |d                    S d S )N
T)full_filename)r   untyped_storagedata_ptrr   r   r
   )rI   addrframesaddr_to_frames      r   object_contextz/cuda_allocation_context.<locals>.object_contextG  sn    # 	J&&((1133D"&&t,,F!yyV4!H!H!HIIItr   )r    r!   memory	_snapshotr   )snapshotsegr   blkr  	real_sizer  r  s          @r   r   r   <  s    z **,,HM
#    9~x= 	  	 C7|111$0$5$5!	&,d#CKDD		      r   c                    g d}t          |           D ]>\  }}|                    | dt          |j                   d|j        rdnd d           ?t          |           D ]=\  }}|j        D ]0\  }}|                    | d| dt          |           d	           1>|                    d
           d                    |          S )N)zdigraph GraphName {znode [shape=rect];zrankdir=LR;z [label=z, color=redblackz];z -> z
 [label = rc   z}
r   )rd   r8   r   r   r   r   r   )r   linesr   r   r@   r   js          r   to_dotr  P  s    HHHE%   _ _1]]6!'??]]af=YUURY]]]^^^^%   B B1 	B 	BHE1LLA@@1@@u@@@AAAA	B	LL99Ur   az  
<!DOCTYPE html>
<html>
<head>
  <style>
    body {
      margin: 0;
      padding: 0;
      overflow: hidden;
    }

    #container {
      display: flex;
      flex-direction: column;
      height: 100vh;
    }

    #main {
      flex: 2;
      overflow: auto;
    }

    #preContainer {
      flex: 1;
      overflow: auto;
    }

    svg {
        overflow: scroll;
    }

    pre {
      margin: 0;
      padding: 10px;
    }
  </style>
</head>
<body>
  <div id="container">
    <div id="main">
    </div>
    <div id="preContainer">
      <pre id="stacktrace">Mouse over tensor objects to see where they were allocated.</pre>
    </div>
  </div>
<script src='https://cdnjs.cloudflare.com/ajax/libs/viz.js/1.8.0/viz-lite.js'></script>
<script>
let dot = $DOT
let image = Viz(dot, {format: 'svg'});
document.getElementById('main').innerHTML = image
$LISTENERS
</script>
</body>
</html>
z
document.getElementById('node{id}').addEventListener('mouseover', function(event) {{
  document.getElementById("stacktrace").textContent = {stack}
}})
c           	         g }t          |           D ]j\  }}|j        t                              t	          |dz             t          |j         d|j                             }|                    |           kt          |           }t          
                    dt          |                    
                    dd                    |                    S )Nrx   z:
)rG   stackz$DOTz
$LISTENERSr   )rd   r   _listener_templateformatr   r   r   r8   r  	_templatereplacerk   r   )r   	listenersr   r   sdots         r   to_htmlr    s    I%    19%%QU6QWB\B\QRQZB\B\;];]%^^
--CVT#YY//77diiPYFZFZ[[[r   c                 v     t           j        j                            d            fd}t	          |          S )Ni )max_entriesc                     | r\t          d | D                       st                              d           d S  t          t	          |                                d S d S )Nc              3   4   K   | ]}t          |          V  d S r3   )r   r   s     r   r   z:observe_tensor_cycles.<locals>.observer.<locals>.<genexpr>  s*      >>s~c**>>>>>>r   z No CUDA Tensors found in garbage)anyr#   r)   r  r   )r   callbacks    r   r*   z'observe_tensor_cycles.<locals>.observer  sr     	5>>g>>>>> >???HW\'223344444		5 	5r   )r    r!   r  _record_memory_historyr9   )r  r*   s   ` r   observe_tensor_cyclesr!    sH    	J,,,@@@5 5 5 5 5 8$$$r   c                  Z    t                               d           d } t          |           S )a  
    Install a warning that reports whenever a cycle that is holding CUDA memory is observed.

    The warning produces an .html file that visualizes the cycle,
    and links it to the stack frame that allocted the CUDA tensor.

    Reference cycles are freed by the cycle collector rather than being cleaned up
    when the objects in the cycle first become unreachable. If a cycle points to a tensor,
    the CUDA memory for that tensor will not be freed until garbage collection runs.
    Accumulation of CUDA allocations can lead to out of memory errors (OOMs), as well as
    non-deterministic allocation behavior which is harder to debug.
    z2Watching Python reference cycles for CUDA Tensors.c                     t          ddd          5 }|                    |            t                              d|j                   d d d            d S # 1 swxY w Y   d S )Nwz.htmlF)suffixdeletezDReference cycle includes a CUDA Tensor see visualization of cycle %s)r	   writer#   r$   rH   )htmlr@   s     r   write_and_logz)warn_tensor_cycles.<locals>.write_and_log  s    GEBBB 	kaGGDMMMNNacdcijjj	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	ks   6AAA)r#   r)   r!  )r)  s    r   warn_tensor_cyclesr*    s9     KKDEEEk k k !///r   )/r   r   typingr   r   r   r   r   r   r   rz   r   tempfiler	   r    torch.cuda._memory_vizr
   r   r6   logging	getLoggerr[   r#   r9   rC   r   r   r   floatcomplexrA   r   bytesr   r   r   r   r   r   r   r   r  r  r  r  r!  r*  r   r   r   <module>r3     s   				 



 ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?    ' ' ' ' ' '  < < < < < < < <  		8	$	$8 8 8R$ $ $
 >g g gZ 5'44::sE:
 .F .F .Fd& & & & &: & & & &*$ , , , , ,\  o o o  (	 	 	6	n 
\ \ \	% 	% 	%0 0 0 0 0r   