
    Χg(                        U d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d d	l"m#Z# er&d dl$Z$d dl%Z%d d
l&m'Z( d dl)Zd dl*Zd dl+Zd dl,Zd dl-Zd dl.Zda/ee0         e1d<   g dZ2de0fdZ3i Z4ee5e6f         e1d<   	 dOde7deej8        j9                 de5fdZ:dee5         fdZ;de5fdZ<d Z=de5fdZ> ej?        e@          ZA G d de          ZBdej8        j9        ddfdZCdee5df         fdZDdej8        j9        deedf         fdZEdej8        j9        defdZFdej8        j9        dee5df         fd ZGd!ee5df         dee5df         fd"ZHd#eeejI        ejJ        e6ejK        eLejM        e0f         df         ded$         fd%ZNd&eejI        df         d'ed$         deejI        df         fd(ZOd)edejI        fd*ZPd+eejI        e6eLe0f         d,d-dejI        fd.ZQd)ejI        d/eejI        ejJ        e6ejK        eLejM        e0f         deejI        e6eLe0f         fd0ZRd1d2d3ee5df         d4eejI        df         d5ed$         d6ee5df         d7eejI        df         d8ed$         d9e0d:ed;         d<eeejI        ejJ        e6ejK        eLejM        e0f         df         deeejI        e6eLe0f         df         fd=ZSd1d2d3ee5df         d4eejI        df         d5ed$         d6ee5df         d7eejI        df         d8ed$         d9e0d:ed;         d<eeejI        ejJ        e6ejK        eLejM        e0f         df         deeejI        e6eLe0f         df         fd>ZT G d? d@          ZUe jV         G dA dB                      ZWee5ee5e
e5ef         f         f         ZXee1dC<   	  e jV        dDE           edFG           G dH dI                                  ZY edFG           G dJ dK                      ZZ edFG          ddLdej8        j9        dMeeeYe
e5ef         f                  fdN            Z[dS )P    N)AnyCallableDictFinalListMappingOptionalSequenceSetTupleTYPE_CHECKINGUnion)	TypeAlias)
FakeTensor)compatibility)FakeTensorProp)OperatorSupport)CALLABLE_NODE_OPS)_pytree_pybind_state_SUPPORT_ONNXRT)is_onnxrt_backend_supportedtorch_compile_backendOrtExecutionProviderOrtBackendOptions
OrtBackendreturnc                      t           n	 t          j        d           t          j        d           t          j        d           ddl} ddl} ddl} ddl} ddlm}m	}m
}m} da n# t          $ r da Y nw xY wt           S )	a!  Returns ``True`` if ONNX Runtime dependencies are installed and usable
    to support TorchDynamo backend integration; ``False`` otherwise.

    Example::

        # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ONNX)
        >>> import torch
        >>> if torch.onnx.is_onnxrt_backend_supported():
        ...     @torch.compile(backend="onnxrt")
        ...     def f(x):
        ...             return x * x
        ...     print(f(torch.randn(10)))
        ... else:
        ...     print("pip install onnx onnxscript onnxruntime")
        ...
    Nonnxruntimezonnxruntime.capi._pybind_state
onnxscriptr   )decomposition_tablefx_onnx_interpreterpasses
type_utilsTF)r   	importlibimport_module
torch.onnxtorch.onnx._internal%torch.onnx._internal._exporter_legacy torch.onnx._internal.diagnosticstorch.onnx._internal.fxr"   r#   r$   r%   ImportError)torchr"   r#   r$   r%   s        \/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/onnx/_internal/onnxruntime.pyr   r   ;   s    & 	$#M222#$DEEE #L111''''88883333            #OO 	$ 	$ 	$#OOO	$ s   AA$ $A32A3_dumped_onnx_modelmodel_stringgraph_modulec                    t           j                            dd          }|sdS t                              |d          dz   }| | d}t	          |d          5 }|                    |            ddd           n# 1 swxY w Y   |t          |<   |X| | d}t	          |d	d
          5 }|                    t          |j                             ddd           n# 1 swxY w Y   |S )a  Stores the onnx model into a file.
    The name is "{ONNXRT_DUMP_PATH}{N}.onnx"
    where *N* is the number of files already stored with
    this prefix.
    If graph_module is not None, the graph is stored as a string with
    the same filename except the extension (.txt).
    ONNXRT_DUMP_PATHN    z.onnxwbz.txtwzutf-8)encoding)osenvirongetr0   openwritestrgraph)r1   r2   prefixnfilenameffilename_txts          r/   _dump_onnx_modelrG   o   su    Z^^.55F rvr**Q.A"!"""H	h		 	              !"v )!))),g666 	-!GGC*++,,,	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	-Os$   A<<B B ,(C  C$'C$c                      dgS )NCPUExecutionProvider rJ       r/   _infer_default_epsrL      s     ###rK   namec                     t           j                                        r&t           j        j                            |            dS dS )zIf PyTorch is installed with CUDA support, this starts NVTX range.

    Check torch.cuda.nvtx.range_push's document for more details.
    N)r.   cudais_availablenvtx
range_pushrM   s    r/   _nvtx_range_pushrT      sB    
 z   )
""4((((() )rK   c                      t           j                                        r%t           j        j                                         dS dS )zIf PyTorch is installed with CUDA support, this terminates NVTX range.

    Check torch.cuda.nvtx.range_pop's document for more details.
    N)r.   rO   rP   rQ   	range_poprJ   rK   r/   _nvtx_range_poprW      s@    
 z   $
!!#####$ $rK   device_typec                     ddl m} | dk    r|j                                        S | dk    r|j                                        S | dk    r|j                                        S t          d| z             )Nr   r   rO   cpumaiazUnsupported device type: )onnxruntime.capir   	OrtDevicerO   rZ   npu
ValueError)rX   ORTCs     r/   _get_ort_device_typera      s    666666f~""$$$e~!!###f~!!###
0;>
?
??rK   c                        e Zd ZdZdee         deeef         f fdZde	ee
j        j        f         de
j        j        def fdZ xZS )	OrtOperatorSupporta0  Operator support for ONNXRuntime backend.

    It has two-level of support decision. One is via support_dict and the other one
    is via extra_support_dict. The logic of using support_dict is implemented in
    OrtOperatorSupport and extra_support_dict is used by OperatorSupport.is_node_supported.
    support_dictextra_support_dictc                 X    t                                          |           || _        d S N)super__init___onnx_support_dict)selfrd   re   	__class__s      r/   ri   zOrtOperatorSupport.__init__   s,    
 	+,,,".rK   
submodulesnoder   c                    |j         t          vrdS |j         dk    rC|j        | j        v r5t                              d|j        t          |j                             dS t                                          ||          r5t                              d|j        t          |j                             dS t          	                    d|j        t          |j                             dS )NFcall_functionz0support_dict supports node.target: %s (type: %s)Tz6extra_support_dict supports node.target: %s (type: %s)zLsupport_dict and extra_support_dict don't support node.target: %s (type: %s))
opr   targetrj   loggerinfotyperh   is_node_supportedwarning)rk   rm   rn   rl   s      r/   rv   z$OrtOperatorSupport.is_node_supported   s     7+++57o%%$+9P*P*PKKBT[!!  
 4 77$$Z66 	KKHT[!!  
 4ZK	
 	
 	

 urK   )__name__
__module____qualname____doc__r   r   r   r@   ri   r   r.   nnModulefxNodeboolrv   __classcell__)rl   s   @r/   rc   rc      s         /SX /4S> / / / / / /!#ux"67?Dx}	         rK   rc   c                     | j         }g }d}|j        D ]1}|j        dk    r|                    |           ||j        dk    r|}2|dS |D ]}|                    |           dS )z
    In torch.fx.Graph, placeholder is a special assignment node. If it's not
    executed in the beginning, it could overwrite values computed by upstream
    nodes.
    Nplaceholder)rA   nodesrq   appendprepend)r2   rA   placeholdersfirst_not_placeholderrn   r   s         r/   _move_placeholder_to_frontr      s     EL  ) )7m##%%% (TW-E-E$(!$# 3 3%%k22223 3rK   .c                      g }| D ]Z}t          |d          rH|j        }|j        dk    r|                    d           :|j        dk    r|                    d           [t	          |          S )zBReturn the first valid device (i.e., GPU or CPU) in argument list.devicerO   CUDAExecutionProviderrZ   rI   )hasattrr   ru   r   tuple)argsepsargr   s       r/   _infer_ep_from_devicer      s    
C 3 33!! 	3ZF{f$$

23333%%

1222::rK   c                    g }| j         j        D ]b}|j        dk    rUt          |d          r0d|j        v r't          |j        d         t          j                  sJ |                    |           ct          |          S )Nr   metaval)
rA   r   rq   r   r   
isinstancer.   Tensorr   r   )r2   r   rn   s      r/   _extract_graph_module_inputsr     s    L"( & &7m##tV$$ B$)););!$)E"2ELAAAAA%%%rK   c                 r    | j         j        D ]}|j        dk    r|j        d         c S t	          d          )zHCollect "val" fields from outputs metadata in this torch.fx.GraphModule.outputr   z2No output node found in this torch.fx.GraphModule.)rA   r   rq   r   r_   )r2   rn   s     r/   _extract_graph_module_outputsr     sN    "(    7h 9Q<  I
J
JJrK   c                 t    t          j        t          |                     \  }}d |D             }t          | S )z[Return the all valid devices (i.e., GPU or CPU) among outputs of this torch.fx.GraphModule.c                 Z    g | ](}t          |d           rd|j        v |j        d         )S )r   r   r   r   ).0
output_args     r/   
<listcomp>z/_infer_ep_from_graph_module.<locals>.<listcomp>!  sO        J''
 -2Z_,D,D	 	 -E,D,DrK   )r   tree_flattenr   r   )r2   flattened_output_args_selected_output_argss       r/   _infer_ep_from_graph_moduler     sR    &3%l33   1 /   !"677rK   r   c                     dt           dt          fd}t          |           }t          t	          ||d                    S )z:Sort execution providers in eps based on pre-set priority.epr   c                 &    | dk    rdS | dk    rdS dS )NrI      r   r7   r   rJ   )r   s    r/   get_execution_provider_priorityz2_sort_eps.<locals>.get_execution_provider_priority.  s+    '''1((( 1qrK   T)keyreverse)r@   intsetr   sorted)r   r   
unique_epss      r/   	_sort_epsr   +  sP    	C 	C 	 	 	 	 SJ
(GQUVVVWWWrK   valueszORTC.OrtDevice.c           	      b   ddl m dt          dt          fddt          t          j        t          j        t          t          j        t          t          j	        t          f         dt          ffdt          |           dk    rt          fd| D                       }|S  d	          fS )
Nr   r   	device_idr   c                 
    | pdS )Nr   rJ   )r   s    r/   _device_id_or_zeroz-_get_onnx_devices.<locals>._device_id_or_zeroG  s    ~ArK   valuec           	      (   t          | t          j                  rS j        t	          | j        j                  j                                         | j        j                            S t          | t          j	        t          t          j        t          t          j        t          f          r6 j        t	          d          j                                        d          S t          dt!          t          |                     z             )NrZ   r   zUnsupported value type: )r   r.   r   r]   ra   r   ru   default_memoryindexSymIntr   SymFloatfloatSymBoolr   r_   r@   )r   r`   r   s    r/   _map_tensor_or_sym_to_devicez7_get_onnx_devices.<locals>._map_tensor_or_sym_to_deviceJ  s    
 eU\** 	L!4>$U\%677--//""5<#566  
 EL#u~uemTR
 
 	L "4>$U++T^-J-J-L-La   7#d5kk:J:JJKKKrK   c              3   .   K   | ]} |          V  d S rg   rJ   )r   r   r   s     r/   	<genexpr>z$_get_onnx_devices.<locals>.<genexpr>_  s/      TTE88??TTTTTTrK   r7   )r\   r   r   r   r.   r   r   r   r   r   r   lenr   )r   ort_devicesr`   r   r   s     @@@r/   _get_onnx_devicesr   =  s     766666c c    LL%,U^UEMSWW
L 
	L L L L L L L( 6{{QTTTTVTTTTT,,Q//11rK   tensorsdevicesc                    ddl m} ddlm}  |j                    }|                    t          |                      g }g }g }| D ]p}|                    ||j                            |                    |	                                           |                    |
                                           q|                    | ||||           |S )Nr   r   )_TORCH_DTYPE_TO_NUMPY_DTYPE)r\   r   "torch.onnx._internal.fx.type_utilsr   OrtValueVectorreserver   r   dtypesizedata_ptrpush_back_batch)	r   r   r`   r   	ortvaluesdtypesshapes	data_ptrstensors	            r/   !_get_ortvalues_from_torch_tensorsr   e  s     766666NNNNNN##%%Ic'll###FFI , ,1&,?@@@fkkmm$$$**++++gy&&'JJJrK   r   c                     | j         rt          d          t          j        |                                 | j        | j                  }|S )Nz#sparse tensor is not yet supported.)r   r   )	is_sparser_   r.   emptyr   r   r   )r   outs     r/   _to_real_tensorr   z  sD     @>???
+fkkmm6<
N
N
NCJrK   dynamo_value
value_infoonnx.ValueInfoProtoc                 `   t          | t          j                  rFt          |j        j        j        j                  dk    r| j        dk    rt          j        |           S t          | t                    r t          j
        | t          j                  S t          | t                    r t          j
        | t          j                  S t          | t                    r t          j
        | t          j                  S t          | t          j                  sJ |                                 S )z9Helper function to wrap PyTorch variables as torch.Tensorr   )r7   )r   )r   r.   r   r   ru   tensor_typeshapedimsqueezer   r   int64r   float32r   
contiguous)r   r   s     r/   _adjust_scalar_from_fx_to_onnxr     s     	<..)
+1566!;;$&& }\***	L#	&	& )|L<<<<	L%	(	( )|L>>>>	L$	'	' )|L
;;;;,55555&&(((rK   
prim_valuec           	          t          | t          j                  s
J d            t          |t          j        t          t          j        t          t          j        t          f          r| 	                                S | S )zFHelper function to wrap ORT-produced torch.Tensor as PyTorch variableszORT's output must be tensor.)
r   r.   r   r   r   r   r   r   r   item)r   r   s     r/   _adjust_scalar_from_onnx_to_fxr     se    $ fel++KK-KKKK	sENE5=$G  
 {{}}MrK   sessonnxruntime.InferenceSessioninput_namesinputsinput_devicesoutput_namesoutputsoutput_devicespreallocate_outputinput_value_infosr   .normalized_prim_outputsc
                    dd l }
ddlm} t          d           t	          d t          ||          D                       }t                       t          d           t          ||          }|r*t	          d |D                       }t          ||          }n |j                    }t                       t          d            |
j	                    }|
                    dd	           |                     ||||||           t                       |rFt          d
           t	          d t          ||	          D                       }t                       |S t          d
           |
j        j        j                            |          }t	          d t          ||	          D                       }t                       |S )Nr   r   r   c              3   <   K   | ]\  }}t          ||          V  d S rg   r   r   r   r   s      r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  D        C 	'sJ77     rK   r   c              3   b   K   | ]*}t          |t                    rt          |          n|V  +d S rg   )r   r   r   )r   ts     r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  sO       
 
GH*Q
";";BOA
 
 
 
 
 
rK   run_with_ortvaluevector'disable_synchronize_execution_providers1zafter run_with_ortvaluevectorc              3   <   K   | ]\  }}t          ||          V  d S rg   r   r   onnx_outputprim_outputs      r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  D       
 
([ +;DD
 
 
 
 
 
rK   c              3   <   K   | ]\  }}t          ||          V  d S rg   r  r  s      r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  r  rK   )r    r\   r   rT   r   ziprW   r   r   
RunOptionsadd_run_config_entryr   training	ortmodule_utils_ortvalues_to_torch_tensor)r   r   r   r   r   r   r   r   r   r   r    r`   
ort_inputspth_outputsort_outputsrun_optionss                   r/   %_run_onnx_session_with_ortvaluevectorr    sC   " 666666\"""  "6+<==    F &'''26=IIJ
  , 
 
LS
 
 
 
 
 8^TT)d)++.///(+(**K$$%NPSTTT  [*lK     8999  
 
,/=T,U,U
 
 
 
 
 	 	8999!*4;VV
 
  
 
,/=T,U,U
 
 
 
 
 	rK   c
                    dd l t          d t          ||          D                       }fdt          ||          D             }
|                     ||
          }t          d t          ||	          D                       }|S )Nr   c              3   <   K   | ]\  }}t          ||          V  d S rg   r   r   s      r/   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>!  r   rK   c                     i | ]D\  }}|j                             |                                                                          ES rJ   )OrtValueortvalue_from_numpyrZ   numpy)r   rM   r   r    s      r/   
<dictcomp>z0_run_onnx_session_with_fetch.<locals>.<dictcomp>%  sT       D& 	k"66vzz||7I7I7K7KLL  rK   c              3   `   K   | ])\  }}t          t          j        |          |          V  *d S rg   )r   r.   
from_numpy)r   r   r  s      r/   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>*  sW        
 E;	 	'U##	
 	
     rK   )r    r   r
  run)r   r   r   r   r   r   r   r   r   r   feedr  r  r    s                @r/   _run_onnx_session_with_fetchr!    s    "   "6+<==    F   V44  D ((<..K  
 #&k3J"K"K    K rK   c                       e Zd ZdZdddeedf         ded         deedf         d	ed         d
ed         ded         deeej        df         ej        f         fdZ	d Z
dS )OrtExecutionInfoPerSessionzWInformation required to execute torch.fx.GraphModule using onnxruntime.InferenceSessionsessionr   r   .r   r   r   output_value_infosr   r   r   example_outputsc	                 v    || _         || _        || _        || _        || _        || _        || _        || _        d S rg   r$  r   r   r   r%  r   r   r&  )	rk   r$  r   r   r   r%  r   r   r&  s	            r/   ri   z#OrtExecutionInfoPerSession.__init__7  sS     6= -8BS-9CU :G:H  	rK   c                 .   ddl m}m} t          |          t          | j                  k    rdS t          || j                  D ]N\  }}t          |t          j        t          t          f          s dS t          |t          t          t          f          r[ |t          |                    }||j        j        j        k    r dS t          |j        j        j        j                  dk    r dS ||j                 }||j        j        j        k    r dS t          |j        |j        j        j        j                  D ]R\  }}t          |t                    r|j        |k    s|j        r-t          |t          j                  r|j        rO  dS PdS )Nr   )(_TORCH_DTYPE_TO_ONNX_TENSOR_ELEMENT_TYPE,from_python_type_to_onnx_tensor_element_typeFT)r   r*  r+  r   r   r
  r   r.   r   r   r   r   ru   r   	elem_typer   r   r   	dim_value	dim_paramr   )	rk   r   r*  r+  r   r   
onnx_dtyper   onnx_dims	            r/   is_supportedz'OrtExecutionInfoPerSession.is_supportedX  s   	
 	
 	
 	
 	
 	
 	
 	
 t99D233335"4)?@@ 	! 	!OCcEL%#=>> uu #UD122 II$s))TT
!<!FFF 55z28<==BB 55 B#)LJZ_8BBBuu!$SY
0K0Q0U!V!V ! !Xc3'' !&#--1C-U\22 !x7I ! 555! trK   N)rx   ry   rz   r{   r   r@   r   r.   r   ri   r1  rJ   rK   r/   r#  r#  4  s        aa
/
 38_
 !!;<	

 CHo
 ""<=
 23
 34
 uU\3%67EF
 
 
 
B% % % % %rK   r#  c                   Z    e Zd ZddZdej        j        fdZdej        j        defdZ	dS )	"OrtExecutionInfoForAllGraphModulesr   Nc                     i | _         d S rg   )execution_info_per_graph_module)rk   s    r/   ri   z+OrtExecutionInfoForAllGraphModules.__init__  s    
  	,,,rK   r2   c                 \    || j         vrd S | j         |         }|D ]} |j        | r|c S d S rg   )r5  r1  )rk   r2   r   
candidates	candidates        r/   &search_reusable_session_execution_infozIOrtExecutionInfoForAllGraphModules.search_reusable_session_execution_info  s_     tCCC4 9,G
# 	! 	!I%y%t, !    ! trK   rt   c                 r    || j         vr|g| j         |<   d S | j         |                             |           d S rg   )r5  r   )rk   r2   rt   s      r/   cache_session_execution_infoz?OrtExecutionInfoForAllGraphModules.cache_session_execution_info  sI     tCCCBFD0>>>0>EEdKKKKKrK   )r   N)
rx   ry   rz   ri   r.   r~   GraphModuler9  r#  r;  rJ   rK   r/   r3  r3    sz           !H0    L!H0L8RL L L L L LrK   r3  r   T)frozenF)is_backward_compatiblec                      e Zd ZU dZdZeee                  ed<   	 dZ	e
ed<   	 dZeee                  ed<   	 dZe
ed<   	 dZe
ed	<   	 dZed
         ed<   	 dZed         ed<   	 dZeeedgdf                           ed<   dS )r   aJ  Options for constructing an ``OrtBackend``, the ONNX Runtime
    backend (``"onnxrt"``) for ``torch.compile``.

    Example::

        >>> @torch.compile(
        ...     backend="onnxrt",
        ...     options=torch.onnx._OrtBackendOptions(...),
        ... )
        ... def ort_function(x):
        ...     return x ** x
    Npreferred_execution_providersTinfer_execution_providersdefault_execution_providersFr   use_aot_autogradztorch.onnx.ExportOptionsexport_optionszonnxruntime.SessionOptionsort_session_optionszonnx.ModelProtopre_ort_model_transforms)rx   ry   rz   r{   r@  r	   r
   r   __annotations__rA  r   rB  r   rC  rD  rE  rF  r   rJ   rK   r/   r   r     s          OS!8H5I,J#KRRR '+t***mLP(3G*H!IPPP  %$$$V!d!!! <@NH78???UBF">?FFFV 	 h,-t345   1 1rK   r   c            	          e Zd ZU dZddee         fdZdej        j	        de
eeeeef         f                  fdZdej        j	        fdZdej        j	        dej        j	        fd	Zdej        j	        dej        j	        fd
ZdZeed<   g Zeed                   ed<   e	 ddeeeeeef         f                  dd fd            Zed             Zed             ZdS )r   a	  A backend compiles (sub-)graphs in torch.fx.GraphModule to onnxruntime.InferenceSession calls.

    The compiler entry point is OrtBackend.compile, which
        1. partitions the original graph into supported sub-graphs (type: torch.fx.GraphModule) and unsupported
           sub-graphs.
        2. For each supported sub-graph, it replaces its _wrapped_call function with _ort_accelerated_call.
        3. Inside _ort_accelerated_call, it creates onnxruntime.InferenceSession and calls it to execute the sub-graph.
    Noptionsc                 H   ddl m} dd l}dd l}dd l}|t                      n|| _        |j        j        j	        
                    | j        j        |j                                        n| j        j                  | _        |j        j        j        j                            | j        j                  }d d d d d d}t%          ||          | _        i | _        t+                      | _        d| _        d| _        t3          |j        d          rt6          nt8          | _        d S )Nr   r   )getattrz_operator.getitemz_operator.mulz_operator.addz_operator.subFr   )r\   r   r(   r*   +torch.onnx._internal.fx.decomposition_tabler   _optionsonnx	_internal_exporter_legacyResolvedExportOptionsrD  ExportOptions_resolved_onnx_exporter_optionsr~   r"   '_create_onnx_supports_op_overload_tableonnx_registryrc   _supported_ops_partitioner_cacher3  _all_ort_execution_info_assert_allclose_to_baselineexecution_countr   r   r  r!  r  )rk   rI  r`   r.   rd   re   s         r/   ri   zOrtBackend.__init__  sX   ::::::4444::::6=o02227  J 1GG=/7 
((***]1  	,& z+.Bjj0>
 

  "& "!!
.
 
.
 1?QRR UW (J'K'K$,1) 
 t*,=>>.11- 	rK   r2   r   c                    d}| j         j        r!t          | x}r|}nt          |          x}r|}g }g | j         j        pg t          |          | j         j        pt                      R D ]^}t          |t                    r|i f}n't          |t                    r|d         
|d         i f}|||vr|                    |           _|S )NrJ   r7   r   )rM  rA  r   r   r@  r   rB  rL   r   r@   r   r   )rk   r2   r   inferred_epseps_from_argseps_from_graph_moduleselected_epsr   s           r/   _select_epszOrtBackend._select_epsS  s    )+=2 	5 5t <<} 
5
  -*El*S*SS& 5  5
m9?R
|$$
 m7O;M;O;O
 
 
	( 
	(B
 "c"" !"XB&& !2a5=eR[~"L"8"8##B'''rK   c                 z	   ddl }ddlm}m}  | j        j        |g|R  }|r:|j        }|j        }	|j        }
|j	        }|j
        }|j        }|j        }|j        }nJ|                    | j        j        |                                          }| j        j        r/d| _        t)          |          }d }t+          j        ||          }nM	  t/          |          j        |i |}n1# t2          $ r$ t4                              d|           d| _         w xY w|                    | j        j                  }|                    | j        j        |                                          }|                    || j        j                  }|                    | j        j         j!        	          }	 dd
l"m#} ddl$m } |%                    |          }|&                    |          }n*# tN          $ r t4                              d           Y nw xY w| j(        j)        r| j(        j)        D ]} ||           |*                                }tV          j,        -                    dd          rt]          ||            |j/        || j(        j0         | j1        |g|R            }te          d |j3        j4        D                       }	te          d |j3        j5        D                       }
tm          |          }to          |td                    rtm          |          }ntm          |f          }te          d |j3        j4        D                       }te          d |j3        j5        D                       }tq          ||	||
||||          }| j        9                    ||           | xj:        dz  c_:        to          |tv          j<                  }|r|fn|}to          |td                    sJ t{          d |D                       sJ t}          d           |                     ||	|||
||| j(        j        ||
  
        }t                       | j@        r\tw          jA        jB        jC        |g|R ddi}|r|fn|}t          ||          D ]%\  }} tv          jE        F                    ||            &|r|d         n|S )a  This function replaces GraphModule._wrapped_call in compiled model.

        The _wrapped_call is the underlying implementation of forward method. Replacing
        it means we delegate the computation to _ort_acclerated_call and therefore
        onnxruntime.InferenceSession.
        r   N)r#   r$   Fc                 R    t          | d          rd| j        v r| j        d         S | S )Nr   r   r   )r   s    r/   maybe_map_to_meta_valz>OrtBackend._ort_acclerated_call.<locals>.maybe_map_to_meta_val  s4    uf-- %%5:2E2E  %z%00$rK   zFakeTensorProb failed for %s)diagnostic_context)fx_graph_moduleonnxfunction_dispatcher)opset_version)	optimizer)r    zONNXScript optimizer is not available. Skipping optimization. Please `pip install onnxscript -U` to enable post-export optimization.r4   )r2   )path_or_bytessess_options	providersc              3   $   K   | ]}|j         V  d S rg   rS   r   inputs     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s$      OOu
OOOOOOrK   c              3   $   K   | ]}|j         V  d S rg   rS   r   r   s     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s$       S S S S S S S SrK   c              3      K   | ]}|V  d S rg   rJ   rm  s     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s"      %P%Pe%P%P%P%P%P%PrK   c              3      K   | ]}|V  d S rg   rJ   rp  s     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s"      &T&T&v&T&T&T&T&T&TrK   r(  r7   c              3   n   K   | ]0}t          |t          j        t          j        t          f          V  1d S rg   )r   r.   r   r   r   )r   elems     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>(  sK       
 
 telEL#>??
 
 
 
 
 
rK   $run_onnx_session_with_ortvaluevectorexecutoraten)Gr    r,   r#   r$   rX  r9  r$  r   r   r   r%  r   r   r&  MovePlaceholderToFrontrS  rd  r  dynamic_shapesr   r   r   tree_mapr   	propagate	Exceptionrs   rw   FxOnnxInterpreterInsertTypePromotionrf  to_model_protorU  rg  r!   rh  onnxscript.rewriteroptimizerewriter-   rM  rF  SerializeToStringr;   r<   r=   rG   InferenceSessionrE  r`  r   rA   rn  r   r   r   r#  r;  rZ  r.   r   allrT   rW   rY  _primsrv  executer
  testingassert_close)!rk   r2   r   kwargsr    r#   r$   !cached_execution_info_per_sessiononnx_sessionr   r   r   r%  r   r   prim_outputsextracted_outputsrc  fx_interpreterexported
onnx_modelrh  ort_rewriter	transformonnx_model_bytesexecution_info_per_sessionis_single_tensor_outputr   onnx_outputsbaseline_outputsnormalized_baseline_ouptutsr  baseline_outputs!                                    r/   _ort_acclerated_callzOrtBackend._ort_acclerated_callt  s@    	GGGGGGGG PD(O#   	*
 - X	<DL;GK<IL A S!B!U=KM>MN<LLL
 "884G  cee  3B */'$A,$O$O!% % %  '/)+<   #I>,#?#?#I$!'$ $LL !   NN#A<PPP /4D+  1BB#'#G#Z C  N "554G cee 
 &)) ,(,(L(d *  H
 "00"BP^ 1  J000000      '//
;;
)11*==

   ]     }5 *!%!G * *IIj)))));;==z~~0$77 	N !!1MMMM 8;7.!]>*$*<?$???  L  OO
8H8NOOOOOK  S S:;K;R S S SSSL-d33M ,.. D!2<!@!@!2L?!C!C %%P%P9I9O%P%P%P P P!&&T&TJ<L<S&T&T&T!T!T)C$'"3)#5+- ,	* 	* 	*& (EE8   	!
 #-\5<"H"H6H\OOL 	  1599999 
 
/
 
 
 
 
 	
 	
 	

 	?@@@xx#M,#
 
 	, 	I$|4< #     .4    (?T!##DT ( 1491 1 I I,_ **;HHHH"9K|A|Ks   
C% %.D/6G& &$HHc                 .   ddl m} || j        v r| j        |         }nw|} ||| j        d          }|                                }|| j        |<   |j        j        D ]7}|j        dk    r*d|j        v r!t          ||j                  }| j
        |_        8|S )Nr   )CapabilityBasedPartitionerT)allows_single_node_partitioncall_modulefused_)!torch.fx.passes.infra.partitionerr  rW  rV  partition_and_fuserA   r   rq   rM   rK  r  _wrapped_call)	rk   r2   r   r  partitioned_prim_graph_moduleprim_graph_modulepartitionerrn   fused_modules	            r/   compilezOrtBackend.compileK  s    
 	QPPPPP, 4222,0,CL,Q)) ,44!#-1  K
 -8,J,J,L,L)4QD#L1 6;A K K 7m++DI0E0E#*+H$)#T#TL 261JL.,,rK   c                     | j         j        r3ddlm} ddlm}   || j        || j        j                  ||          S |                     ||          S )zIf ``OrtBackendOptions.use_aot_autograd`` is ``True``, the `auto_autograd` compiler
        will be invoked, wrapping this ``OrtBackend`` instance's ``compile`` method. Otherwise,
        the ``compile`` method is invoked directly.r   )#min_cut_rematerialization_partition)aot_autograd)fw_compilerpartition_fndecompositions)	rM  rC  functorch.compiler  torch._dynamo.backends.commonr  r  rS  r"   )rk   r2   r   r  r  s        r/   __call__zOrtBackend.__call__  s     =) 	"MMMMMMBBBBBB<< L@#CW   D	" " " ||L$///rK      %_OrtBackend__instance_cache_max_count_OrtBackend__instance_cachec                     dt           dt           fdt           t                     st          d
i  pi  t           fdt          j        D             d          }|}t          t          j                  t          j        k     s(J dt          j         dt           dt           d	            t          j                            t                     x}           |S )a  Returns a possibly cached instance of an ``OrtBackend``. If an existing
        backend was created previously through this function with the same options,
        it will be returned. Otherwise a new backend will be created, cached, and
        returned.

        Note: if ``options`` sets ``ort_session_options``, a new ``OrtBackend``
        will always be returned, since ``onnxruntime.SessionOptions`` cannot
        participate in caching.abc                    | j         |j         k    sP| j        |j        k    s@| j        |j        k    s0| j        |j        k    s | j        |j        k    s| j        |j        k    rdS | j        |j        dS | j        |j        u rdS | j        k|j        d| j        j        |j        j        k    oI| j        j	        |j        j	        k    o/| j        j
        |j        j
        u o| j        j        |j        j        u S dS )NFT)r@  rA  rB  r   rC  rF  rE  rD  ry  diagnostic_optionsrU  fake_context)r  r  s     r/   reusablez<OrtBackend.get_cached_instance_for_options.<locals>.reusable  s#   /13RRR.!2MMM0A4QQQ'1+???%);;;-1KKKu $0A4I4Uu1#333t +0@0L$3q7G7VV W(;':;W (6!:J:XXW (59I9VV 5rK   c              3   >   K   | ]} |j                   |V  d S rg   )rM  )r   r  rI  r  s     r/   r   z=OrtBackend.get_cached_instance_for_options.<locals>.<genexpr>  s6      UU1xx
G7T7TUQUUUUUUrK   NzNo more than z instances of z allowed. Please instantiate `z` explicitly to pass to `torch.compile`. See https://github.com/pytorch/pytorch/pull/107973#discussion_r1306144795 for discussion.rJ   )r   r   nextr   r  r   r  r   )rI  backendr  s   ` @r/   get_cached_instance_for_optionsz*OrtBackend.get_cached_instance_for_options  s   !	) !	.? !	 !	 !	 !	F '#455 	;'::7=b::GUUUUU
3UUU
 

 ?J/00:3XXXX"
 E " "" "=G" " " YXX '..*W:M:M/MwNNNrK   c                  B    t           j                                         d S rg   )r   r  clearrJ   rK   r/   clear_cached_instancesz!OrtBackend.clear_cached_instances  s    #))+++++rK   c                  4    t          t          j                  S rg   )r   r   r  rJ   rK   r/   get_cached_instanceszOrtBackend.get_cached_instances  s    Z0111rK   rg   )rx   ry   rz   r{   r	   r   ri   r.   r~   r<  r
   r   r@   r   r   r`  r  r  r  r  r   rG  r  r   staticmethodr   r  r  r  rJ   rK   r/   r   r     s         V
 V
): ; V
 V
 V
 V
p!H0	%WS#X../	0   BUL1E UL UL UL ULn4-EH$8 4-58CW 4- 4- 4- 4-l0!H00		0 0 0 0$ )*)))24eD./444IMC C% 17383D DEFC	C C C \CJ , , \, 2 2 \2 2 2rK   r   )rI  rI  c                H    t                              |          | |          S rg   )r   r  )r2   r   rI  s      r/   r   r     s"     55g>>|TRRRrK   rg   )\dataclassesr&   loggingr;   typingr   r   r   r   r   r   r	   r
   r   r   r   r   typing_extensionsr   r.   torch._C
torch._opstorch._prims.executortorch.fxtorch._subclasses.fake_tensorr   torch.fx._compatibilityr    torch.fx.passes.fake_tensor_propr    torch.fx.passes.operator_supportr   torch.fx.passes.tools_commonr   torch.utilsr   rN  r    r\   r   r`   r(   r)   r*   r+   rL  torch.onnx._internal.fx.passesr   r   rG  __all__r   r0   r@   r   bytesr~   r<  rG   rL   rT   rW   ra   	getLoggerrx   rs   rc   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r!  r#  	dataclassr3  r   r   r   r   rJ   rK   r/   <module>r     s	             				                            ( ' ' ' ' '            4 4 4 4 4 4 1 1 1 1 1 1 ; ; ; ; ; ; < < < < < < : : : : : :        
*KKK6666660000++++6666)))) #'$ & & &  .T . . . .b &( DcN ' ' ' IM '/0D'E   2$HSM $ $ $ $)3 ) ) ) )$ $ $
@c 
@ 
@ 
@ 
@ 
	8	$	$/ / / / / / / /d3UX-A 3d 3 3 3 3*
E#s(O 
 
 
 
ux/C cSVh    K0D K K K K K8eh.B 8uSRUX 8 8 8 8 X5c? XuS#X X X X X$%2L%,U^UEMSWW	
 		%2  !%2 %2 %2 %2P5<$%056K0L
5<   *J 5<    )	) &) \) ) ) )@L	 	L	
   8N
(NsCxN %,#$N ./	N
 S/N 5<$%N /0N N 78N #L%,U^UEMSWW	
 		N  5sE4/0#56!N N N Nb#
(#sCx# %,#$# ./	#
 S/# 5<$%# /0# # 78# #L%,U^UEMSWW	
 		#  5sE4/0#56!# # # #LI I I I I I I IX L L L L L L L LB #(U3S8I3I-J(J"K i K K K	 d###e,,,<1 <1 <1 <1 <1 <1 <1 -, $#<1~ e,,,q2 q2 q2 q2 q2 q2 q2 -,q2h e,,,
 FJ	S S S(&S e-wsCx/@@AB	S S S -,S S SrK   