
    קg!3                    0f   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
mZmZmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlmc mZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z-m.Z. d d	l/m0Z0m1Z1m2Z2m3Z3m4Z4 d
dl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z> ddl:m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJ ddlmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZS ddlTmUZUmVZV  ejW        eX          ZYi ZZeej[        j\        ede
f         f         e]d<   i Z^eej[        j\        eede
f                  f         e]d<    e_            Z`eej[        j\                 e]d<   ejU        ja        ZaejU        jb        ZbejU        jc        Zc e_            Zdeej[        j\                 e]d<    e_            Zeeej[        j\                 e]d<    e_            Zfeej[        j\                 e]d<   i Zgeej[        j\        ej[        j\        f         e]d<   ejU        jh        Zhdede
f         deede
f                  fdZid Zjd Zkd Zld  Zm eleajn        eajo        eajp        eajq        eajr        eajs        eajt        eaju        eajv        eajw        eajx        eajy        eajz        g           ej{        ej|        ej}        ej~        ej        ej        ej        ej        ej        ej        ej        ej        ej        d!Zd"efd#Zd$ Zd% Zd&e&fd'Zd( Zd) Zd* Zd+ Zd, Zd-e&j        d-fd.Zd/ Zdvd0Z	 	 	 	 	 	 dwd1Zdxd2Zdxd3eHd"ej        fd4Z eecj        d5          d3eHd"ej        fd6            Zd-d7d3eHd"ej        fd8Z eeaj        j        d5          d3eHd"ej        fd9            Zd-d7d3eHd:ej        fd;Z eecj        d5          d3eHd:ej        fd<            Zdd=e&j        d-ddd-d-df	d>Zd? Z e             	 dxd@Z eeaj        d-dA          dB             Z eeaj        d-dA          dC             Z eeaj        eaj        eaj        eaj        ecj        g          dD             Z eeadE          r  eeaj                  e            eeaj        d5          dydF            Z eeaj        d5          dydG            Z eeaj        g          dydH            Z eeaj                  dI             Z eeaj                  dJ             Z eeaj                  dK             Z eeaj                  dL             Z eeaj        j                  dM             Z eeaj                  dN             Z eeaj        d5          dO             Z eecj        d5          dP             Z eeaj        d5          dQ             Z eeaj                  dR             Z eeaj        d5           eeaj        d5           eeaj        d5          dS                                     Z eeaj        d5          dT             Z eeaj        d5          dzdV            Z eeajn        d5          dydW            Zn eeaj        d5          dydX            Z eeajo        d5          dydY            Zod{dZZ eehj        d5          d[eHd\eHd]eHd^ed_ed`ed"ej        deHfda            Z eehj        d5          d[eHd\eHd]eHd^ed_ed`ed"ej        deHfdb            Z eehj        j        d5          d[eHdcedded_ed`ed"ej        deHfde            Z eehj        j        d5          d[eHdcedded_ed`ed"ej        deHfdf            Z eehj        j        d5          d[eHdceHddeHd_ed`ed"ej        deHfdg            Z eehj        j        d5          d[eHdceHddeHd_ed`ed"ej        deHfdh            Z eeaj                  d{di            Z eeaj        d5          d|djedkedlefdm            Z eeaj        d5          d|djedkedlefdn            Z eeaj        d5          d|djedkedlefdo            Z eeaj        d5          dp             Z eeaj        d5          d}dq            Z eeaj        d5          d{dr            Z eeaj        d5          d{ds            Z eeaj        d5          dt             Z eeaj        d5          du             Z eeaj        d5          dv             Zېd{dwZ eeaj                  d~dy            ZݐddzZ e j        d          d{             Zdyd|ej        fd}Zdyd|ej        fd~Zddej        j        fdZddZd Z eejU        j        j        d5          d             Z eeaj        d5          d             Z eeaj        d5          d             Z eeaj        j        d5          d             Z eeaj                  d             Z e j        d          d             Zd Z eeaj        j                  Z eeaj        j                  Z eeaj        j                  Z eeaj        j                  Z eeaj                    eeaj                  d             Z eeaj                  d             Z ee9j        d5          d             Z ee9j        d5          d:ej        fd            Z ee9j        d5          d             Z  ee9j        d5          d             Z ee9j        d5          d ddee         deHdedjefd            Z ee9j        d5          d ddededee         deHdjef
d            Z eeaj        d5          d-d-dd[eHdeHdedefd            Zd Zd Z	d Z
d ZdhZd Z eeaj                    eeaj                    eeaj                    eeaj                    eeaj        d-            eeaj        j        d-            eeaj                    eeaj        d-            eeaj                    eeaj                    eeaj                    eeaj                    eeaj                    eeaj                    eeaj        j                    eeaj        j                    eeaj                    eeaj        j                     eeaj!        j                    eeaj"        j                    eeaj#                    eeaj$        d-            eeajt        e            eeaj%        e            eeaj&        e	            eeaj'        e	            eeaj(        e	            eeaj)                    eeaj*                    eeaj*                    eeaj+                    eeaj,                    eeaj-                    eeaj.        e            eeaj/                    eeaj0                    eeaj1                    eeaj2                    eeaj3                    eeaj4                    eeaj5                    eeaj6                    eeaj7        e	            eeaj8                    eeaj9        e            eeaj:                    eeaj;                    eeaj;        j<                    eeaj=                    eeaj>                    eeaj?                    eeaj@                    eeajA                    eeajB                    eeajC                    eeajD                    eeajE                    eeajF                    eeajG                    eeajH                    eeajI                    eeajJ                    eeajK                    eeajL                    eeajM                    eeajN                    eeajO                    eeajP                    eeajQ                    eeajR                    eeajS                    eeajT                    eeajU                    eeajV        jW                    eeajX                    eeajY                    eeajZ                    eeaj[                    eeaj\                    eeaj]                    eeaj^                    eeaj_        j                    eeaj`        j        d-            eeaja        e            eejb        jc        jd                    eejb        jc        je                    eeajf                    eeajg                    eeajh        e	            eeaji                    eeajj                    eeajk                    eeajl                    eeajm                    eeajn        jo                    eeajp        j        ed-            eeajq        j        ed-            eeajr        j        ed-            eeajs        j        ed-            eeajt        j        ed-            eeaju        j        ed-            eeajv        j        ed-            eeajw        j        ed-            eeajx        j        e            eeajy        j        e            eeajz        j        e            eeaj{        j        e            eeaj|                    eeaj}        d5          dxd            Z} eeaj~                  ddd            Z~d Z eead          r  eeaj                  e~            eecj                  d             Z eeaj        d5          dedefd            Z eeaj        d5          dd            Zd Z eej        eaj        g          dddd-dd            Z eej                  dvd            Z eej                  d             Z eeaj                  d             Z eeaj                  d             Zd Z eeaj        d5          d             Zd Z eej        eaj        g          dddddddd            Zd Zd Z  eeaj                   ee                    Z e ed                    Z e ed                     Zd Z eeaj                  dddddd            Z eeaj                  dddddd            Z eeaj                  dddddd            Z eecj        j                  d             Z eej        eaj        g          d             Z eeaj        d5          dxdÄ            Z eeaj        d5          ddĄ            Zdń ZdƄ ZdǄ ZdȄ Z eeaj        d5          dɄ             Z eeaj        d5          dʄ             Z eeaj                  dxd˄            Z eeaj                  dxd̄            Zd̈́ Zd΄ Z eeaj        d5          dxdτ            Z ee9j        d5          dxdЄ            Zdф Z eeaj        j        d-Ҧ          Z eeaj        j        d-Ҧ          Z eeaj        d5          dӄ             Z eeaj        d5          dԄ             ZedՄ             Z eeaj        d5          dydք            Z eeaj        d5          defdׄ            Zdd=d؜dej[        j\        dedee         defd܄Z eeaj        d5          ddݜdedee         fdބ            Z eeaj        d5          defd߄            Z eeaj        d5          defd            Z eeaj        d5          defd            Z eeaj        d5          d=ddedefd            Z	 	 ddeee         df         dedefdZ eeaj        j                  dyd\ee         fd            Z eeaj        j                  dyd\ee         fd            Z eeajx        j                  	 dvdee         dee         fd            Zx eeajy        j                  	 dvdee         dee         fd            Zy eeaj        j                  	 	 	 ddee         dee         dee         fd            Z eeaj        j                  	 	 	 ddee         dee         dee         fd            Zd Z eecj        j                  d             Z eeaj        d5          d{d            Zdej        deej        ef         fdZdej        dej        fdZdej        dej        dej        fdZ	 ddZd Zd ZdddZ eecj        d5          	 dxd            Z eecj        d5          d             Z eeaju                    eeajv        j        d-Ҧ          Z eeajv        d5          d              ZvddZɐd Zʐd Zːd Z eeaj        j        d-Ҧ          Z eeaj                  d             Z eeaj        j        d-Ҧ          Z eeaj                  d             Z eeaj        j        d-Ҧ          ZҐd Z eeaj                  d	             Z eeaj        j                  	 dd
            Z eeajp        j        d-Ҧ          Z eeaj        j        d-Ҧ          Z eeajp        d5          	 	 	 	 	 dd            Zp eeaj        d5          	 	 	 	 	 dd            Z֐d Z eeajq        j        d-Ҧ          Z eeajq        d5          	 dyd            Zq eeaj        j        d-Ҧ          Z eeaj        d5          	 dyd            Zڐd Zܐd ZݐdydefdZސd Z eeaj                  dddd            Zd Zd Zd Zd Z eeaj        ecj        g          dydd-dd            Z eeaj                  dydd-dd            Zd Zed              Z eeaj        j        d-Ҧ          Z eeaj        j        d-Ҧ          Z eeaj        j        d-Ҧ          Z eeaj        d=!          d"             Zdxd#Z eeaj                  d$             Z eeaj        d5          dxd%            Zed&             Zed'             Z eeaj        d=!          dyd(            Z eeaj        gd=!          d)             Zd3e:j        dee:j                 fd*Z eecj        gd=!          d+             Z eeaj        eaj        j        gd=e&j        A          d,             Z eeaj        ecj        gd=!          d-             Z eeaj                   d.             Z  eeaj        ecj        g          dddd/            Z eeaj        j                  Z eeaj        j                  Z eeaj        j                  Z eeaj	        j                  Z
 eeaj        j                  Z eeaj                  dvd0            Z eeaj                  dvd1            Z eeaj                  d2             Z eeaj	        d5          dyd3            Z	 eeaj        d5          dyd4            Z eeaj                  dddd5            Z eeaj                  dd6            Z eeaj        d5          dd7            Z eeaj        d5          dd8            Z  eecj                   eސd9                       eeaj                   eސd:                    Z  eeaj                   eސd;                    Z  eeaj                   eސd<ej        =                    Z  eeaj                   eސd>ej        =                    Z eeaj        d=d?@          Z eeaj;        j<        d-Ҧ          Z eeaj;        j<        d5          ddxd-dAdB            Z eeaj;        j        d5          ddC            Z;dvdDZ dE Z! e!eaj"                  Z" e eaj#                  Z# e eaj$                  Z$ eeaj%                  Z% e!eaj&                  Z& e!eaj'                  Z' eeaj(                  Z( eeaj)        d=F          Z) e!eaj*                    e!eaj+                    eeaj,                  Z, eeaj-                  Z- eeaj.                  Z. eeaj/        dGH          Z/ eeaj0                  Z0 eeaj1                  Z1 eeaj2                  Z2 e eaj3                    e eaj4                  Z4  eeaj5        e&j        5          e4            e eaj6                    e eaj7                    e eaj8                    e!eajY                    eeaj9        dd=ej        I          Z9 eeaj:        dd=ej        I          Z: eeaj;        dd=ej        I          Z; eeaj<        dd=ej        I          Z< eeaj=                  Z= eeaj>                  Z>  eeaj?                  e=             eeaj@                  e>            eeajA                  ZA eeaj,                  Z, e eajB                  ZB eeajC                    eeajD        dJH          ZD eeaj                    eeajE        ej        =             eeajF                  eA            eeajG        ej        =            eeajH        ej        =            eeajI        ej        =            eeajJ        ej        =          ZJ eeajK        ej        =            eeajL        ej        =            e eajM                    e eajN                    e eajO                    e eajP                    e eajQ                    e eajR                    e eajS                    e eajT                    e eajU                    e eajV                    e eajW                    e eajX                    e eajY                    e eajZ                    e eaj[                    e eaj\                   ddKl]m^Z^m_Z_ dL Z`e_D ][Za e`eaea          D ]\  ZbZcZd eebeaecedM            e`ecea          D ]\  ZbZcZd eebeaecedM           \ eeaje        j        ed=F          Zf eeaje        j        ed=F          Zg eeaje        j        ed=F            eeajh        j        e          Zi eeajh        j        e          Zj eeajk        j        e)            eeajk        j        e)            eeajl        j        eA            eeajm        j        e,            eeajn        j        e            eeajn        jo        e            eeajp        j        e          Zq eeajp        j        e          Zr eeajs        e'            eeajt        j        e=            eeajt        j        e=            eeaju        j        e>            eeaju        j        e>            eeajv        j        e=            eeajv        j        e=            eeajw        j        e>            eeajw        j        e>            eeajx        eB            eeajy        eD            eeajz        e}           dN Z{ e{eaj|        j        eaje        j        ef            e{eaj|        j        eaje        j        eg            e{eaj}        j        eajh        j        ei            e{eaj}        j        eajh        j        ej            e{eaj~        j        eajp        j        eq            e{eaj~        j        eajp        j        er           dO Z eeaj        e            eeaj        e-            eeaj        e.            eeaj        e/            eeaj        e0            eeaj        e1            eeaj        e2            eeaj        e            eeaj        j        e            eeaj        j        e            eeaj        e9            eeaj        e:            eeaj        e;            eeaj        e<            eeaj        e)            eeaj        e%            eeaj        e&             eeaj                  e-             eeaj                  e.             eeaj                  e0             eeaj                  e1             eeaj                  e2            eeaj        eaj                    eeaj        eaj                    eeaj        eaj                    eeaj        eaj                    eeaj        eaj                    eeaj                  dvdP            Z eeaj        j                  dQ             Z eeaj        j                  dR             Z eeaj                  dS             Z e-j                    D ]&\  ZZ  e e.e                    e           ' eeaj                  dT             Z eejU        j        j                  dU             Z eejU        j        j                  dV             Z eejU        ja        j        j                  dW             Z eejU        j        dX          r, eejU        j        j        j                  dY             Z eejU        ja        j                  dddZ            Zd d[lmZ  ee            ee          d\             Z eejU        j        j                  d]             Z eejU        j        j                  d^             Z eed5          d_e:j        defd`            Z eejU        jc        j        j                  da             Z eejU        j        j                  db             Z	 d dlZejU        j        Z eej                  dc             Z eej                  dd             Z eej                  de             Z eej                  df             Z eej                  dg             Z eej                  dh             Z eej                  di             Z eej                  dj             Z eej                  dk             Z eej                  dl             Z eej                  dm             Z eej                  dn             Z eej                  do             Z eejU        j        j                  dp             Zn&# eאef$ r eY                    dq           Y nw xY wddrl7mڐZ  e6eڦ           ddsl7mېZ  eېj                      eېj                     ddtl7mސZ  eސj                     ddul7mZ  ej                     dS (      N)defaultdict)AnyCallableDictListOptionalSetTupleUnion)patch)associative_scan_op)triton_kernel_wrapper_mutation)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)CeilDivFloorDivIdentity
IntTrueDivModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)	DtypeView
ExpandViewIndexingConstant	is_tritonops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)	ceildivdecode_device
is_dynamicis_gpuis_pointwise_use,needs_fallback_due_to_atomic_add_limitationspad_listlikesympy_productuse_scatter_fallback)opsV.	lowerings_maybe_layout_constraints	fallbacksneeds_realized_inputsforeach_opsinplace_foreach_opsinplaceable_foreach_opsfnreturnc                      t           t          j        j                  sdS  t          v rt                    S  t
          v rdt           <   dS  fd}t                     } ||          S )zHGet layout constraints. Returns None if there are no layout constraints.Nc                     | t           j        j        j        u rt          t
          <   t
                   S | t           j        j        j        u rd t
          <   d S t          d|            )NzUnknown layout constraint tag: )torch_CTagneeds_fixed_stride_orderconstrain_to_fx_stridesrB   flexible_layoutAssertionError)tagrH   s    T/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/_inductor/lowering.pyhandle_layout_constraint_tagz>maybe_layout_constraints.<locals>.handle_layout_constraint_tagi   sd    %(,777,C%b),R00EHL000,0%b)4 !H3!H!HIII    )
isinstancerL   _ops
OpOverloadrB   rA   get_layout_constraint_tag)rH   rU   rS   s   `  rT   maybe_layout_constraintsr[   \   s    b%*/00 t	&&&(,,	Y(,!"%tJ J J J J $B
'
'C'',,,rV   c                     t           j        j        j        t           j        j        j        g}|D ]}|| j        v r|c S t          t           j        j        t          j                  S N)	rL   rM   rN   rO   rQ   tagsgetattrr$   #custom_op_default_layout_constraint)rH   tags_by_priorityrS   s      rT   rZ   rZ   w   s_    -$    "'>>JJJ 58<!KLLLrV   c                 .    | st          d|           d S )Nzinductor does not support NotImplementedError)condmsgs     rT   
assert_nyirg      s.     F!"Ds"D"DEEEF FrV   c                 J    t           t          t          t          f          rd  D             S t                                          t           t          j        j                  r:t          	                     fd 
                                D                        d S d S )Nc                 ,    g | ]}t          |          S  )add_needs_realized_inputs.0xs     rT   
<listcomp>z-add_needs_realized_inputs.<locals>.<listcomp>   s!    999)!,,999rV   c              3   8   K   | ]}t          |          V  d S r]   )r_   )rm   overloadrH   s     rT   	<genexpr>z,add_needs_realized_inputs.<locals>.<genexpr>   s>       %
 %
&.GB!!%
 %
 %
 %
 %
 %
rV   )rW   listtuplesetrD   addrL   rX   OpOverloadPacketupdate	overloads)rH   s   `rT   rk   rk      s    "tUC()) :99b9999b!!!"ej122 
$$ %
 %
 %
 %
24,,..%
 %
 %
 	
 	
 	
 	
 	

 
rV   c                     t          | t          j        j                  r1|                                 D ]}|t
          t          | |          <   d S |t
          | <   d S r]   )rW   rL   rX   rw   ry   rB   r_   )rH   
constraintrq   s      rT   add_layout_constraintr|      si    "ej122 3 	J 	JH?I%gb(&;&;<<	J 	J )3!"%%%rV   )r   r#   r!                     	   
         dtypec                 |    t          | t                    s| S | t          v sJ d|  d            t          |          } | S )Nzid z missing from DTYPE_ID_LOOKUP)rW   intDTYPE_ID_LOOKUPr   s    rT   decode_dtyper      sM    eS!! O###%O5%O%O%O###E"ELrV   c                     t          | t                    rBt          |                                           p t	          |                                           S t          | t
          j                  r	| j        du S t          | t                    S NT)	rW   r3   r   	get_dtyper   sympyExpr
is_integerr   rn   s    rT   is_integer_typer      sq    !Y "..Q2B1;;==2Q2QQ	Auz	"	" "|t##!S!!!rV   c                     t          | t                    r!t          |                                           S t          | t                    S r]   )rW   r3   r   r   boolr   s    rT   is_boolean_typer      s:    !Y #...!T"""rV   type_promotion_kindc                 H    d fd|D             }t          |d| i\  }}|S )Nc                    t          | t          t          j        f          r| S t	          | d          sJ t          |                                           }t          j        dg|z  | 	                                          S )Nr   r#   r   )
rW   r   r   Basichasattrlenget_sizerL   zerosr   )inpdims     rT   construct_inputz+get_promoted_dtype.<locals>.construct_input   so    cFEK011 	AJ3,,,,,cllnn%%C;sSy@@@@rV   c                 &    g | ]} |          S rj   rj   )rm   argr   s     rT   ro   z&get_promoted_dtype.<locals>.<listcomp>   s#    111SOOC  111rV   r   )r   )r   argsinps_r   r   s        @rT   get_promoted_dtyper      sL    A A A 2111D111D!4Q=PQQHAuLrV   c                 P   t          | t          t          f          s| g} nt          |           } t          |           D ]f}t          |t          j        j                  rE|                                D ]0}t          ||          }|t          vr| 	                    |           1g| S r]   )
rW   rs   rt   rL   rX   rw   ry   r_   rA   append)aten_fnrH   rq   other_fns       rT   get_overloadsr      s    ge}--  )w--7mm - -b%*566 	-LLNN - -"2x009,,NN8,,,NrV   c                     t          | t          j        j                  r	|| j        v S t          | t          j        j                  r||                                 v S dS NF)rW   rL   rX   rw   _qualified_op_namerY   name)op	namespaces     rT   in_namespacer      sV    "ej122 &B111	B
-	.	. &BGGII%%5rV   c           	      8   	 d t                     D             |s|r>r<|rt          j        nd  D             }t          |d|i fd		fd D              |rrt	          t           fdD                        D ]
\  }}| |<   t          t                               D ]k}t           |         t          j
                  rIt          j         |         t           d                                                                       |<   l S )Nc                 B    g | ]\  }}t          |t                    |S rj   rW   r3   rm   irn   s      rT   ro   z"transform_args.<locals>.<listcomp>  +    IIITQ
1i0H0HIqIIIrV   c                 v    g | ]6}t          |t          t          j        f          st	          |d d          4|7S )r   N)rW   r   r   r   r_   )rm   as     rT   ro   z"transform_args.<locals>.<listcomp>
  sQ       a&%+!677 1gt,,8  988rV   r   c                     t          | t                    rt          |           S t          | t          j                  r9t          j        | j        d                                                            S | S Nr   )rW   r3   to_dtyper&   Constantvalue
get_device)r   r   r   indicess    rT   promoteztransform_args.<locals>.promote  si    #y)) U+++C-- {39eT'!*5E5P5P5R5RSSS
rV   c                 &    g | ]} |          S rj   rj   )rm   r   r   s     rT   ro   z"transform_args.<locals>.<listcomp>  s!    )))q

)))rV   c                      g | ]
}|         S rj   rj   rm   r   r   s     rT   ro   z"transform_args.<locals>.<listcomp>  s    4N4N4NT!W4N4N4NrV   r   )	enumeraterL   r   r   zipbroadcast_tensorsranger   rW   r&   r   r+   creaters   r   )
r   	broadcastr   convert_input_to_boolpromoting_argsr   rn   r   r   r   s
   `      @@@rT   transform_argsr     s   IIYt__IIIG *4 *' *  	JEE   N '5H E
	 	 	 	 	 	 	 *)))D))) XW X!24N4N4N4Ng4N4N4N!OPP 	 	DAqDGGs4yy!! 	X 	XA$q'2;// X$+DGT$wqz:J:S:S:U:U5V5VWWQKrV   c                     t          j                  fd            }t          |           }t                              |           t
                              t                              ||                     |S )a  
    Add a foreach lowering to lowerings dict.

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                  `    t          |           dk    sJ  | i |}t          |           |S )Nr!   )r   r4   )r   kwargsout	decomp_fns      rT   wrappedz+_register_foreach_lowering.<locals>.wrapped4  s=    4yyA~~~~i(((C
rV   )	functoolswrapsr   rE   rx   rA   dictfromkeys)r   r   r   aten_fnss    `  rT   _register_foreach_loweringr   (  s}     _Y      W%%Hx   T]]8W55666NrV   c                      t          j                   fd            }t                      t                              t
                               |                     |S )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                     t          |           } d}t          |           dk    r,t          | d         t           t          f          r
d}| d         } t	          d D                       sbt          d |                                D                       rJ t          d |                                D                       r
J d            t          |           } |r| g}  | i |}t          |           |S )	NFr#   r   Tc              3   H   K   | ]}|t           v pt          |d           V  dS )_c10d_functionalN)rC   r   )rm   rH   s     rT   rr   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>Y  sG       
 
JLR9_DR1C D D
 
 
 
 
 
rV   c              3   @   K   | ]}t          |t                    V  d S r]   r   rl   s     rT   rr   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>\  s,      MM:a33MMMMMMrV   c              3   "   K   | ]
}|d k    V  dS )r   Nrj   rl   s     rT   rr   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>^  s7         U
     rV   zout= ops aren't yet supported)
rs   r   rW   rt   allanyvalueskeysr   r4   )	r   r   unpackedr   r   r   r   r   r   s	       rT   r   z#_register_lowering.<locals>.wrappedO  sD   BFt**t99>>ja4-@@>H7D  
 
PW
 
 
 
 
 	/ MMV]]__MMMMMMMM  $*KKMM     / /./ / / )02G
 
  	6Di(((C
rV   )r   r   r   rA   rx   r   r   )r   r   r   r   r   r   s   ````` rT   _register_loweringr   A  s     _Y         < G$$GT]]7G44555NrV   Fc                 >    t          j        t          | |||          S )z+
    Shim to support decorator syntax.
    r   r   r   )r   partialr   )r   r   r   r   s       rT   register_loweringr   t  s-     /3   rV   c                 |   g }t          j        t          |           t          |          t          j        d                    D ]\  }}|dk    r|                    |           !|dk    r|                    |           =t          j        j        	                    ||           t          t          j        |          j                  t          t          j        |          j                  k     r|                    |           |                    |           t          t          |                    S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    r#   )	fillvalue)	itertoolszip_longestreversedr   Integerr   r@   graphsizevarsguard_equalsr   expandfree_symbolsrt   )r   boutputrn   ys        rT   broadcast_symbolic_shapesr     s    F%Xa[[EM!,<,<   ! !1 66MM!!VVMM!G))!Q///5<??/003u|A7S3T3TTTa    a    &!!"""rV   c                    ||
J d            ||t           j        }t          d | D                       s| S t          d | D                       r!|pt	          | d|ifdfd| D             S t          d | D                       }g }| D ]J}t          |t          t          f          r|	                    t          j        t          j        ||                                |                                          t!          |                                                               t          |t$          j                  r||	                    t          j        t)          ||                                |                                          t!          |                                                               5|	                    |           L|S )NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3   d   K   | ]+}t          |t          j        t          t          f          V  ,d S r]   )rW   r   r   r   floatrl   s     rT   rr   z$promote_constants.<locals>.<genexpr>  s5      HHAz!ek3677HHHHHHrV   c              3   d   K   | ]+}t          |t          t          t          j        f          V  ,d S r]   )rW   r   r   r   r   rl   s     rT   rr   z$promote_constants.<locals>.<genexpr>  s5      
D
D:a#uek233
D
D
D
D
D
DrV   r   c                     t          | t          j                  r#t          j        | t          d                     S t          j        | t          d                     S r]   )rW   r   r   r&   r,   r7   r   rn   r   s    rT   
const_funcz%promote_constants.<locals>.const_func  sQ    !U[)) B*1e]45H5HIII{1e]4-@-@AAArV   c                 &    g | ]} |          S rj   rj   )rm   rn   r  s     rT   ro   z%promote_constants.<locals>.<listcomp>  s!    ...!

1...rV   c              3   h   K   | ]-}t          |t          t          t          j        f          )|V  .d S r]   )rW   r3   r+   r&   r   rl   s     rT   rr   z$promote_constants.<locals>.<genexpr>  s:      WWA:a)Z1U#V#VWaWWWWWWrV   )r   DEFAULTr   r   r   nextrW   r   r   r   r+   r   r&   r   r   r   rs   r   r   r   r,   )inputsoverride_return_dtyper   exr   rn   r  r   s         @@rT   promote_constantsr
    s   %)<)D)DN *E)D)D $)<)D=EHHHHHHH 

D
DV
D
D
DDD /% 
);*
)<*
 *
	B 	B 	B 	B 	B /...v....	WWWWW	W	WB
C  a#u&& 	JJ!K2<<>>2==??CCT"++--EXEX    
 5;'' 	JJ!$QHH''     JJqMMMMJrV   c           	      V     d ddt           t                   f fd}|S )Nalphar  c           	        	
 +t          t          t          |                    r	rJ  | S t          |          }r1| .| dk    r(t	          |          }t          |d         |           |d<   n| J d |D             	|d                                         
p|d                                         t          t          |d         
                                          j                  |dd          D ]s}t          |t          j                  sWt          
          t          |                                          k    s%J d d
 d|                                             tt           j        d uoxt%          t           j        dd           d uo[t           j        j        j        d uoCt           j        j        j                            dd	          ot,          j        t,          j        fv 	
fd
}s_d }|D ]>}t          |
                                j                  r|
                                } n?|s|d         
                                }p|}t3          j        ||
          S )Nr#   c                 6    g | ]}|                                 S rj   make_loaderrl   s     rT   ro   z1make_pointwise.<locals>.inner.<locals>.<listcomp>  s     333q1==??333rV   r   zndim mismatch  current_nodelow_precision_pointwise_barrierFc                 
    t                     t                    k    sJ d  d             t          j        k    r  fd	D              S 
r#r!t          j        k    r 
 fd	D              S g }	D ]P} |           }r,t	          j        |d          }t	          j        |          }|                    |           Q | }r,t	          j        |d          }t	          j        |          S |S )Nzwrong ndim r  c                 &    g | ]} |          S rj   rj   rm   loadindexs     rT   ro   zCmake_pointwise.<locals>.inner.<locals>.inner_fn.<locals>.<listcomp>  s!    4U4U4UTTT%[[4U4U4UrV   c                 &    g | ]} |          S rj   rj   r  s     rT   ro   zCmake_pointwise.<locals>.inner.<locals>.inner_fn.<locals>.<listcomp>  s!    5V5V5Vddd5kk5V5V5VrV   F)use_compute_types)r   rL   r   float64r?   r   r   )r  inputs_loadedr  r   downcastr   emulate_precision_castsrH   is_gpu_deviceloadersoverride_fn_when_gpu_float64override_fn_when_input_boolrangess   `    rT   inner_fnz/make_pointwise.<locals>.inner.<locals>.inner_fn  s\   u::V,,,.LE.L.LF.L.L,,,
""'B'N224U4U4U4UW4U4U4UVV,! U]**335V5V5V5Vg5V5V5VWW "# . .D$u++C. <#&<Ue#T#T#T!l8U;;!((----b-(* 9  #|C%PPPH<%888
rV   devicer   r&  r%  )r   mapr-   r
  rs   mulr   r   r9   r7   r   typerW   r&   BaseConstantr   r@   r   r_   r  metagetrL   bfloat16float16r0   r   )r  r  otherr&  r(  r   r   r   r!  r"  r%  allow_alpharH   override_devicer#  r$  r  triton_fallbacks         @@@@@rT   innerzmake_pointwise.<locals>.inner  s   &3s9f/E/E+F+F&"""""?F++"6+@AA 	! UaZZf U33r
===33F333##%%%>)<)<)>)>}VAY-A-A-C-CDDIJJABBZ 	A 	AEeR_55 AV  I I : : :@@@V@@enn.>.>@@: : : G4 966dB9$)59 $)--.OQVWW9 %.%-88 	 	 	 	 	 	 	 	 	 	 	 	 	6  	0F  !,,..-.. \\^^FE  0--// *F	
 
 
 	
rV   r   r3   )rH   r  r3  r$  r#  r2  r4  r5  s   ``````` rT   make_pointwiser7    sv     /3 L
 L
 L
tI L
 L
 L
 L
 L
 L
 L
 L
 L
 L
 L
 L
\ LrV   c                 X     dddt           t           t                            f fd}|S )Nr#   r  r  c                 B   d }t          t          j        j        j                  dk    pt          j        j        j        t          v }t          j        j        j        D ]'}|j        D ]}|j        dk    r|j        t          v sd}(d }|D ]"}t          |t          t          f          r|} n#|
J d            g }|D ]Z}t          |t          t          f          s'|                    |gt          |          z             E|                    |           [ |t          |           }	d gt          |          z  }
|	                                D ]\  \  }}}g }|D ]\  }}r	 |d| i}n | }||
|<   t          j                            |t           j                  r?|r=|r;|                                 |                    |                                           |rt          j                            |           t+          d |
D                       sJ |
S )Nc                 H   t          t                    }t          |           D ]}\  }}t          |  pt          j        }d }|D ]2}t          |t                    r|j        	                                } n3|
J d            |||f         
                    ||f           ~|S )Nz.foreach op should have at least one tensor arg)r   rs   r   r8   r$   #combo_kernel_foreach_dynamic_shapesrW   r3   datar   r   )	arg_pairsr   r   r   use_foreachr(  ts          rT   
group_argsz9make_foreach_pointwise.<locals>.inner.<locals>.group_args(  s    d##C$Y// = =4"D))WV-W    A!!Y// !"!2!2!4!4 &&C '&&V[)*111d)<<<<JrV   r   call_functionTz1at least one input must be a list to a foreach opr  c              3      K   | ]}|d uV  	d S r]   rj   rl   s     rT   rr   z8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>n  s&      22Q1D=222222rV   )r   r@   r   r  userstargetrF   r   rE   rW   rs   rt   r   r   itemshas_featureBackendFeatureFOREACHrealizeget_operation_nameregister_operation_listr   )r  r  r@  realize_outputsnodeusera_list_inputinputbroadcast_inputsgroupsoutputsr(  r>  groupoperation_list
output_indr   r   r2  pw_fns                     rT   r5  z%make_foreach_pointwise.<locals>.inner%  s   	 	 	$ $*++q0 Bw#*.AA 	 G(. 	+ 	+D
 + +?22{8R8R&*O+  	 	E%$// $ $$> %$$  	/ 	/EedE]33 / ''#l2C2C(CDDDD ''....C!1233&3|,,,,2LLNN 	@ 	@(!V[5(*N G G  *"UD666FF"UD\F&,
# G''0FGGG#G (G
 NN$$$"))&*C*C*E*EFFF @//???22'22222222rV   r6  )rW  r2  r5  s   `` rT   make_foreach_pointwiserX  $  sV    45 J J JtDO, J J J J J J JX LrV   rn   c                     |                                  k    r|rt          |           n| S fd} t          |          |           S )Nc                 2    t          j        |           S )N)	src_dtype)r?   r   )rn   r   r[  s    rT   	_to_dtypezto_dtype.<locals>._to_dtypey  s    |Au	::::rV   r  )r   cloner7  )rn   r   copyr\  r[  s    `  @rT   r   r   t  so    IE&uQxxxQ&; ; ; ; ; ; B>)5AAA!DDDrV   r   c                 H   |j         s|                                 j         rq|                                 r3t          | |          }t          j                            ||            |S  t          t          j	        j
        d          | |          S t          | |d          S )Nr   Fadd_to_fallback_setTr_  )
is_complexr   r   
empty_liker&   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr   )rn   r   dsts      rT   _convert_element_typerm    s     
1;;==3 
::<< 		 Qe,,,C"))#q111J#*2     Au4((((rV   rd  c                .   |                                  }||k    r|rt          |           n| S d } ||          } ||          }||k    r( t          t          j        j                  | |          S t          t          j        | |                    S )Nc                 t    | j         rt          j        |           j        S t          j        |           j        S r]   )is_floating_pointrL   finfobitsiinfor   s    rT   _get_primitive_bitwidthz1to_dtype_bitcast.<locals>._get_primitive_bitwidth  s3    " 	+;u%%**;u%%**rV   )	r   r^  rh  atenviewr   r3   r*   r   )rn   r   r_  x_dtypert  src_bitsdst_bitss          rT   to_dtype_bitcastrz    s    kkmmG%&uQxxxQ&+ + + '&w//H&&u--H80	00E:::)!U33444rV   c                     |j         s|                                 j         rLt          j        t          j                            t          j        j        j	        j
        | |                    S t          | |          S r]   )re  r   r3   r   r&   ComplexViewrL   r?   ru  rv  r   rz  r  s     rT   _view_dtyper}    sg     
1;;==3 
N!!%)."5";QFF
 
 	
 Au%%%rV   r(  c                    t          |          }|                                 |k    r|rt          |           n| S t          j        t
          j                            | |                    S r]   )r7   r   r^  r3   r   r&   
DeviceCopy)rn   r(  r_  s      rT   	to_devicer    sZ    6""F||~~&uQxxxQ&BM00F;;<<<rV   c                 &    t          | |d          S )NTrd  )r  )rn   r(  s     rT   _device_putr    s    QT****rV   Tc
                 p   |p| j         }t          |          }
|rt          d|z             }|t          |          }t          |
|||r|nd||	          }
 t          | |||          |
          }
t	          t
          |          r. t          t          t
          |          d|          |
           |
S )z3A pointwise function that maps ops.{name} to inputs
libdevice_N)r  r$  r#  r2  r4  r   )r   r   )__name__r.   r7  r   r   ri  r_   )r   r   r   r   r   r  r$  r2  use_libdevice_for_f64r4  rH   fn_libdevices               rT   register_pointwiser    s
    #7#D	T		B 8"<$#677".&12M&N&N#	
3$?5J%T\\PT'
 
 
B
	/3	
 
 

 	

 

B ud 	
E4   $"7	
 	
 	
 		 	 	
 IrV   c                  f   d} t          d          fd}fd}t          |          t          |t          j                  gfd} t	          t
          j                  |          }t          t          |           r- t	          t          t          |           d          |           |S )z2A pointwise function that maps ops.frexp to inputsfrexpc                        | i |d         S r   rj   r   r   r  s     rT   frexp0zregister_frexp.<locals>.frexp0      ud%f%%a((rV   c                        | i |d         S Nr#   rj   r  s     rT   frexp1zregister_frexp.<locals>.frexp1  r  rV   r]  c                  <     d         | i | d         | i |fS Nr   r#   rj   )r   r   pw_fnss     rT   rH   zregister_frexp.<locals>.fn  s4    vay$)&))96!9d+Ef+E+EEErV   Nr`  )
r.   r7  rL   int32r   ru  r  r   ri  r_   )r   r  r  rH   r  r  s       @@rT   register_frexpr    s   D  E) ) ) ) )) ) ) ) ) 	vvU[AAAF
F F F F F
	

 


 

B ud 	
E4   $	
 	
 	
 	 	 	 IrV   c                 H    t          ||          }t          | |          }|S )Nr2  )rX  r   )r   pointwise_lowering_fnr2  rH   s       rT   register_foreach_pointwiser    s+    
 
  5;	O	O	OB	#GR	0	0BIrV   )r   r   c           	      f   d }t          |t          t          f          r t          |          |          }t          |t          t          f          r t          |          |          }| ||gt	          d         d         t
          j                  }d t                    D             }t          |t          fd|D                        D ]
\  }}||<   t          t                              D ]k}t          |         t          j                  rIt          j        |         t!          |d                                                                      |<   l t%          ||          d         t'          d         |          t'          d         |                    S )	Nc                      t          j        |  S r]   )r?   wherer   s    rT   rH   zwhere.<locals>.fn  s    y$rV   r#   r!   r`  c                 B    g | ]\  }}t          |t                    |S rj   r   r   s      rT   ro   zwhere.<locals>.<listcomp>  r   rV   c                      g | ]
}|         S rj   rj   r   s     rT   ro   zwhere.<locals>.<listcomp>  s    0J0J0JQa0J0J0JrV   r   r]  )rW   r   r   constant_liker   r   r  r   r   r   r   r   r&   r   r+   r   rs   r   r7  r   )	re   r   r   rH   r   r   r   rn   r   s	           @rT   r  r    s         !eS\""  M!Q!eS\""  M!Q!Q<DQa.M.U  E JIYt__IIIGG.0J0J0J0J'0J0J0JKLL  1Q3t99 T Td1gr{++ 	T 'Qd71:6F6O6O6Q6Q1R1RSSDG:>"E:::Q$q'5))8DGU+C+C  rV   c                     t          |           dk    r1t          | d         t          t          f          rt	          | d          S t          j        t          d | D             g           }g }| D ]}|                                }t          |          t          |          k    s't          d t          ||          D                       rt          ||          }|                    |           |S )Nr#   r   c                 6    g | ]}|                                 S rj   )r   rl   s     rT   ro   z%broadcast_tensors.<locals>.<listcomp>.  s     #A#A#AQAJJLL#A#A#ArV   c              3   L   K   | ]\  }}|d k    r|d k    p|d k    o|d k    V   dS r#   Nrj   rm   r   r   s      rT   rr   z$broadcast_tensors.<locals>.<genexpr>3  sV       ,
 ,
=AQa1fa7Q!V%6Q,
 ,
 ,
 ,
 ,
 ,
rV   )r   rW   rs   rt   r   r   reducer   r   r   r   r   r   )r  rD  rS  rn   sizess        rT   r   r   )  s    
6{{aJvay4-@@ &),,(/!#A#A&#A#A#A2   F G  

u::V$$ ,
 ,
EHPVEWEW,
 ,
 ,
 )
 )
$ q&!!AqNrV   c                     | S r]   rj   r   s    rT   nopr  ;  s    HrV   
lift_freshc                 .   t          | t                    sJ |&t          t          j        | j                            S t          |t
          t          j        f          r$t          j	        j
                            |          nt          d |D                       }t          t          |                                           |          }t!          t          |t                    s|fn|          }g }t#          |                                           D ]U\  }}||v r7t          j	        j
                            t          j        |d                    s|                    |           V||                                 k    rt+          | |          n| S )Nc              3   ^   K   | ](}t           j        j                            |          V  )d S r]   r@   r   r   evaluate_static_shaperm   ds     rT   rr   zsqueeze.<locals>.<genexpr>M  s5      JJ17#99!<<JJJJJJrV   r#   )rW   r3   r2   r   r<  r   r   r   r@   r   r   r  rt   r   r   r   ru   r   evaluate_exprEqr   rv  )rn   r   dims	new_shaper  ss         rT   squeezer  D  s^   a#####
{+AF33444 cC,--	K..s333JJcJJJJJ 
 C

--s
3
3CZU33<vv==DI!**,,''    1T		ag.<<UXa^^LL	Q "+ajjll!:!:49ArV   c                 <    t          t          | |                    S r]   )r^  r  )rn   r   s     rT   squeeze_copyr  [  s    C!!!rV   c                     t          | |          }t          | t                    sJ t          |t                    sJ |j        | _        | S r]   )r  rW   r3   r<  rn   r   vals      rT   squeeze_r  `  sI    
!S//Ca#####c9%%%%%XAFHrV   c                     t          |           rt          | dt          j                  S t	          d          } t          |t          j                  |           S )NFr   isinfr]  r   	full_likerL   r   r.   r7  rn   rH   s     rT   r  r  i  U    q 5E4444	W		B?>"EJ???BBBrV   c                     t          |           rt          | dt          j                  S t	          d          } t          |t          j                  |           S )NFr   isnanr]  r  r  s     rT   r  r  q  r  rV   c                     t          |           rt          |           S t          d          } t          |          |           S )Nceilr   r^  r.   r7  r  s     rT   r  r  y  sB    q Qxx	V		B>"a   rV   c                     t          |           rt          |           S t          d          } t          |          |           S )Nfloorr  r  s     rT   r  r    B    q Qxx	W		B>"a   rV   c                     t          |           rt          |           S t          d          } t          |          |           S )Nroundr  r  s     rT   r  r    sB    q %Qxx!!!~b!!!$$$rV   c                     t          |           rt          |           S t          d          } t          |          |           S )Ntruncr  r  s     rT   r  r    r  rV   c                 T   ddl m} t          | g          \  } t          | t          j                  r"t          j        | t          |                    S t          | t                    sJ t          |t          t          f          sJ t          |                                           t          |          k    r| S  ||                                           st          j        j                            t!          |                                                     }|dk    rR ||          sG|                     t          j        j                            t!          |                    |z             t          t          j        | j        t          |                              S )Nr   )free_unbacked_symbols)%torch.fx.experimental.symbolic_shapesr  r
  rW   r&   r,  r+   r   rt   r3   rs   r   r@   r   r   	size_hintr=   
mark_reuser<  )rn   r  r  x_size_products       rT   r   r     sg   KKKKKKaS!!DQ!R_%% 2 E%LL111a#####edE]+++++QZZ\\eEll**  .. 
)33M!**,,4O4OPP
 A&;&;E&B&BLL **=+?+?@@NR   Z&qvuU||<<===rV   c                     t          |          }|D ]}d||<   | }t          |          D ]\  }}|dk    rt          ||          }t          ||          S Nr  )rs   r   	unsqueezer   )r   shapebroadcast_dimensionsr  broadcast_dimensionvidxrn   s           rT   broadcast_in_dimr    sr    UA3 $ $!#
	AA,, " "Q77!S!!A!UrV   c                 F    t          | |                                          S r]   )r   r   )rn   r   s     rT   	expand_asr    s    !QZZ\\"""rV   c                     t          |                                           t                    t                    k    rUt          j        d          gt                    t                    z
  z  z   t          | t                              } t                    t          |                                           k    sJ t          |                                           }d}t          t                              D ]$}|         dk    rd}||         |         z  ||<   %|r6t          ||                                 | 	                                          S t          d t                    D                       rt          t          | |                    S fd}t          j        j                            t%                              }|dk    rG|                     t          j        j                            t%          |                    |z             |                                 t+          j        | 	                                |                                 |t          |                    S )	Nr#   Fr   Tr   r(  c              3   4   K   | ]\  }}|d k    p|d k    V  dS r  rj   r  s      rT   rr   zrepeat.<locals>.<genexpr>  s3      
A
A$!QAFa1f
A
A
A
A
A
ArV   c                 Z   t          |           t                    k    sJ t          |           } t          t                              D ]R}|         dk    rD|         dk    rt          j        d          | |<   2t          | |         d|                   | |<   S |           S )Nr#   r   )r   rs   r   r   r   r    )r  r   old_sizerepeatsx_loaders     rT   r&  zrepeat.<locals>.inner_fn  s    5zzS\\))))Us7||$$ 	I 	IAqzQA;!##$}Q//E!HH.uQxHQKHHE!HxrV   r'  )rs   r   r   r   r   rv  r   emptyr   r   r   r   r^  r   r@   r   r   r  r=   r  r  r0   r   )	rn   r  new_sizezero_tensorr   r&  old_size_productr  r  s	    `     @@rT   repeatr    s@   AJJLL!!H
7||c(mm##M!$$%WH)EFQDNN##w<<3qzz||,,,,,,AJJLL!!HK3w<<   / /1:??KqkGAJ. KXQ[[]]1<<>>JJJJ

A
A#gx*@*@
A
A
AAA *VAx(()))	 	 	 	 	 	 	 w'11-2I2IJJ!	G&&}X'>'>??CSS	
 	
 	
 }}H||~~kkmmH~~	   rV   c                     t          | t                    sJ t          |t          t          f          sJ t          t	          j        | j        |                    S r]   )rW   r3   rs   rt   r5   r   r<  )rn   r  s     rT   rv  rv    sP     a#####edE]+++++T[//000rV   c                     t          | t                    sJ t          |t          t          f          sJ t          t	          j        | j        t          |                              S r]   )rW   r3   rs   rt   r/   r   r<  )rn   r  s     rT   permuter    sU    a#####dT5M*****['d<<===rV               c           
          t          | t                    sJ t          | |d          }t          t          j                            | j        |||||                    S )Nr   clamp)rW   r3   _validate_dimr&   	SliceViewr   r<  )rn   r   startendstepr  s         rT   slice_r  	  sV    a#####
3
"
"CR\((eS$e(TTUUUrV   c           	         t          | t                    r8t          | j        t          j                  r| j                                        } |                                  t          j        |           st          d|  d          t          j	        |           \  }}t          j
        |j        |j        d |D             d |D             t          j        |pd                    }t          t          j        ||                    S )Nzunrealized as_strided(z, ...)c                 6    g | ]}t          j        |          S rj   r   r   rm   r  s     rT   ro   zas_strided.<locals>.<listcomp>  s     '''Qa'''rV   c                 6    g | ]}t          j        |          S rj   r  r  s     rT   ro   zas_strided.<locals>.<listcomp>       )))Qa)))rV   r   )rW   r3   r<  r&   BaseViewunwrap_viewrI  is_storage_and_layoutrd   as_storage_and_layoutFixedLayoutr(  r   r   r   ReinterpretView)rn   sizestridestorage_offsetstorage
old_layout
new_layouts          rT   
as_stridedr    s    !Y !Jqvr{$C$C !F  IIKKK#A&& F!"D1"D"D"DEEE2155GZ''$'''))&)))^(q)) J R'<<===rV   c                 l    t          | t                    sJ t          | |||          j        | _        | S r]   )rW   r3   r  r<  )rn   r  r  r  s       rT   as_strided_r  #  s5    a#####488=AFHrV   c                 D    t          | |||          }t          |          S r]   )r  r^  )rn   r  r  r  results        rT   as_strided_copyr  *  s!    488F==rV   c                     g d} D ]B}                     |||                                         z   f           d         d         }Cd  D              fd}t           d                                                   }d         d         |<   t          j         d                                          d                                         ||          S )Nr   r  c                 6    g | ]}|                                 S rj   r  rm   r   s     rT   ro   z!pointwise_cat.<locals>.<listcomp>8  s"    :::Ccoo'':::rV   c           	        
 t          j        |          t          j                  }g }g }t	          t                              D ]X

dk    rt          j        dt          j                  n*t          j        
         d         t          j                  }t          j        
         d         t          j                  }t          j        ||          }t          j        ||          }
dk    r|}n.
t                    dz
  k    r|}nt          j	        ||          }|
                    |           t          |           t                   
         d         z
            <   |
                    t          j        |
fdd                     Z|d         }	t	          t                    dz
  dd          D ]$
t          j        |
         |
         |	          }	%|	S )Nr   r#   c                  &                         S r]   rj   )r   idx_loadinputs_loaderss   rT   <lambda>z1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>\  s    -N1-h77 rV           r  r!   )r?   
index_exprrL   int64r   r   constantgeltand_r   rs   r   maskedr  )r  idx_dimmasksmasked_loadsr  r  
start_condend_condmasknext_valr   r  r   r  r  inputs_rangess             @@rT   r&  zpointwise_cat.<locals>.inner_fn:  s   .S5;77s6{{##  	  	A 66 Q,,,^M!$4Q$7EE 
 .q!1!!4ekBBC//Jvgs++HAvvc&kkAo%%!x
H55LLCyyH %Xc]]15Ea5H%HIIHSM
777777      #Fq("b11 	 	AyaQ HH
 rV   r'  )r   r   rs   r0   r   r   r   )r  r   prev_endr   r&  r  r  r)  s   ``    @@rT   pointwise_catr+  0  s   9;MH ) )h3<<>>#3F(FGHHH $R(::6:::N. . . . . . . .` F1I&&(())H!"%b)HSMay##%%Qi!!##	   rV   rP  scaleszero_pointsaxis	quant_min	quant_maxc           	      v  	
 t                                                    dk    s
J d            t                                                    dk    s
J d            |                                 t          j        k    rt          | t          j                  } |                                 t          j        k    sJ d|                                              t          |                                           k     s,J dt          |                                                        |                                                                 	                                
	
f	d}t          j	        | 
                                ||                                           S )Nr#   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                 <  	 | 
         f} |           } |          } |          }t          t          j                  \  }}j        t          j        k    rt	          j        |t          j                  }j        t          j        k    rt	          j        |t          j                  }t	          j        |          }t	          j        ||z            |z   }t	          j	        |t	          j
        ||                    }	t	          j        |	          S Nr   )_create_constantsrL   float32r   r?   r   r  
reciprocalr  maximumminimum)r  channel_idxrP  scale
zero_pointqminqmax	inv_scaler  clampedr.  r   input_loaderr0  r/  r,  scales_loaderr-  zero_points_loaders             rT   r&  z;quantized_decomposed_quantize_per_channel.<locals>.inner_fn  s    4ylS!!k**''44
&y)5=QQQ
d<5=((L66E++j%+>>JN5))	i	)**Z7+dCKc$:$:;;|GU+++rV   r'  )r   r   r   rL   r/  r   r9  r  r0   r   r   rP  r,  r-  r.  r/  r0  r   r&  rD  rE  rF  s    `````` @@@rT   )quantized_decomposed_quantize_per_channelrH  u  s    v  !!Q&&&(=&&&{##%%&&!+++-G+++EN**..U]***YeooFWFWYY 	+**#    8U^^%5%5!6!688   $$&&L&&((M$0022, , , , , , , , , , , , ," !!~~	   rV   c                   	
 t                                                    dk    s
J d            t                                                    dk    s
J d            |                                 |k    s"J d| d|                                              t          |                                           k     s,J dt          |                                                        |                                                                 	                                
	
fd}t	          j        |                                 t          j        ||                                           S )	Nr#   r2  r3  Expecting input to have dtype , but got dtype: r5  c                    |          f} |           } 	|          } |          }j         t          j        k    rt          j        |t          j                  }
j         t          j        k    rt          j        |t          j                  }t          j        t          j        |t          j                  |          |z  }|S r]   )r   rL   r9  r?   r   sub)r  r=  rP  r>  r?  r  r.  rD  r,  rE  r-  rF  s         rT   r&  z=quantized_decomposed_dequantize_per_channel.<locals>.inner_fn  s    4ylS!!k**''44
<5=((L66E--j%-@@Jgcl5%-88*EEM
rV   r'  	r   r   r   r  r0   r   r   rL   r9  rG  s    ```    @@@rT   +quantized_decomposed_dequantize_per_channelrO    s    v  !!Q&&&(=&&&{##%%&&!+++-G+++U"""SSS@Q@QSS 	#""#    8U^^%5%5!6!688   $$&&L&&((M$0022          !!m~~	   rV   r>  r?  c                    |                                  t          j        k    rt          | t          j                  } |                                  t          j        k    sJ d|                                               |                                 fd}t          j        |                                 t          j
        |t          |          t          |                    |                                           S )Nr4  c                 J    
|           }t          d|z  |t          j                  \  }}t          j        ||z            |z   }t          t          j                  \  }}t          j        t          j        ||          |          }t          j        |	          S )N      ?r   )r8  rL   r9  r?   r  r<  r;  r   )r  r>  r?  rP  rB  r  r@  rA  rC  r   rD  r0  r/  s            rT   r&  zBquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn  s    S!! 1%K5=!
 !
 !
	: i	)**Z7&y)5=QQQ
d+ck#t44d;;|GU+++rV   r>  r?  r'  )r   rL   r/  r   r9  r  r0   r   r   r   r   r   r   r   rP  r>  r?  r/  r0  r   r&  rD  s      ``` @rT   0quantized_decomposed_quantize_per_tensor_defaultrU    s
    EN**..U]***YeooFWFWYY 	+** $$&&L, , , , , , , , !!"E%LLS__
 
 
 ~~   rV   c                    |                                  |k    s"J d| d|                                               |                                 fd}t          j        |                                 t
          j        t          j        |t          |          t          |                    |                                           S )NrJ  rK  c                      |           }t          ||t          j                  \  }}t          j        t          j        |t          j                  |          |z  }|S r7  )r8  rL   r9  r?   rM  r   )r  r>  r?  rP  r  rD  s        rT   r&  zDquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fn  sV    S!!-eZu}UUUzgcl5%-88*EEM
rV   rS  r'  )r   r  r0   r   r   rL   r9  r   r   r   r   r   rT  s          @rT   2quantized_decomposed_dequantize_per_tensor_defaultrX    s     	U"""SSS@Q@QSS 	#"" $$&&L     !!m"E%LLS__
 
 
 ~~   rV   c                   	 |                                  t          j        k    rt          | t          j                  } |                                  t          j        k    sJ d|                                               t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            |                                                                                                 		fd}t          j	        | 
                                ||                                           S )Nr4  r   r#   expect scale as scalar tensor"expect zero_point as scalar tensorc                     	|           } t                                                    dk    rdnd          } t                                                    dk    rdnd          }j        t          j        k    rt          j        |t          j                  }j        t          j        k    rt          j        |t          j                  }t          j        |t          j        |          z            |z   }t          
t          j                  \  }}t          j
        t          j        ||          |          }t          j        |          S )Nr#   r   rj   r   )r   r   r   rL   r9  r?   r   r  r:  r8  r<  r;  )r  rP  _scale_zero_pointr  r@  rA  rC  r   rD  r0  r/  r>  scale_loaderr?  zero_point_loaders           rT   r&  zAquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn:  s#   S!!c%..*:*:&;&;q&@&@ddbII''ENN4D4D0E0E0J0JPRSS;%-''\&%-88Fu},,,{EMBBKiv 6 6677+E&y)5=QQQ
d+ck#t44d;;|GU+++rV   r'  )r   rL   r/  r   r9  r   r   r  r0   r   r   
rP  r>  r?  r/  r0  r   r&  rD  r`  ra  s
    ````` @@@rT   /quantized_decomposed_quantize_per_tensor_tensorrc    s    EN**..U]***YeooFWFWYY 	+**u~~  A%%ENN""u~~'7'7':a'?'?'?& (@'?'?z""$$%%**J!!""a''J,?,?,A,A!,D,I,I,I+ -J,I,I $$&&L$$&&L"..00, , , , , , , , , , , , !!~~	   rV   c                 ^  	 t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            |                                 |k    s"J d| d|                                              |                                                                                                 		fd}t	          j        |                                 t          j        ||                                           S )	Nr   r#   rZ  r[  rJ  rK  c                      |           } t                                                    dk    rdnd          } 	t                                                    dk    rdnd          }j        t          j        k    rt          j        |t          j                  }j        t          j        k    rt          j        |t          j                  }t          j        t          j        |t          j                  |          |z  }|S )Nr#   r]  rj   )r   r   r   rL   r9  r?   r   rM  )
r  rP  r^  r_  r  rD  r>  r`  r?  ra  s
        rT   r&  zCquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fnh  s    S!!c%..*:*:&;&;q&@&@ddbII''ENN4D4D0E0E0J0JPRSS;%-''\&%-88Fu},,,{EMBBKgcl5%-88+FFO
rV   r'  rN  rb  s
    ``    @@@rT   1quantized_decomposed_dequantize_per_tensor_tensorrf  O  s    u~~  A%%ENN""u~~'7'7':a'?'?'?& (@'?'?z""$$%%**J!!""a''J,?,?,A,A!,D,I,I,I+ -J,I,I 	U"""SSS@Q@QSS 	#"" $$&&L$$&&L"..00	 	 	 	 	 	 	 	 	 !!m~~	   rV   c                   
 | d                                          j        dk    }|rt          d | D                       rt| D ]}|                                 t          d | D                       rt	          t
          j        g| R  \  } } t          t
          j        j                  | |          S t          |           dk    rt          | d                   S t          | d         |d          }t          | dt          j        ifd| D             } dt          t           t"          j        f         d	t"          j        fd
d fdt)          fd| D                       }d	t*          ffd|r-t!          t"          j                            | |                    S fdd}d
dt0          j        j        fdt          |           |k    s9t          |           t6          j        k    rt          
fd| D                       rt          fdt:          j        j        D                       }t)          fd| D                       o|}t          fd| D                       ot)          fd| D                        }	|s|	r|stA          | |          S t!          t"          j                            | |                    S )Nr   cpuc              3   n   K   | ]0}|                                 t          j        t          j        fv V  1d S r]   )r   rL   int8uint8rm   rP  s     rT   rr   zcat.<locals>.<genexpr>~  sK        ;@ej%+66     rV   c              3   `   K   | ])}t          |                                          d k    V  *dS )r~   N)r   r   rl  s     rT   rr   zcat.<locals>.<genexpr>  s9      >>es5>>##$$)>>>>>>rV   r#   r   c                 0    g | ]}t          |          S rj   r   )rm   r   r   s     rT   ro   zcat.<locals>.<listcomp>  s#    555shsE""555rV   rn   rI   c                     t          | t                    r?t          | j        t          j                  r| j                                        S | j        S t          | t          j                  r| j        S | S r]   )rW   r3   r<  r&   r   r  
StorageBoxr   s    rT   unwrap_tensorzcat.<locals>.unwrap_tensor  se    a## 	!&"+.. v))+++va'' 	6MrV   c                 t    t          | t          j                  ot          | j        t          j                  S r]   )rW   r&   ComputedBufferr<  r1   )r?  s    rT   is_reductionzcat.<locals>.is_reduction  s(    !R.//TJqvr|4T4TTrV   c                    t          | t          t          j        f          r  |                     S  |           pFt          | t          j                  o,t          fd|                                 D                       S )Nc              3   h   K   | ],} t           j                            |                    V  -d S r]   )r@   r   
get_buffer)rm   readcan_fuse_reductions     rT   rr   z2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>  sR         #"17#5#5d#;#;<<     rV   )rW   r3   r&   rq  r0   r   get_read_names)r?  rz  ru  rr  s    rT   rz  zcat.<locals>.can_fuse_reduction  s    a)R]344 	8%%mmA&6&6777LOO !R\**     ,,..    	
rV   c              3   .   K   | ]} |          V  d S r]   rj   rm   r?  rz  s     rT   rr   zcat.<locals>.<genexpr>  s/      BBa..q11BBBBBBrV   c                 D   t          j        |           r9t          j        | d          \  }}t           j                            |           S t          | t          t           j        f          r  |                     S t          | t           j                  rdS dS )NF)freezeT)	r&   r  r  ConcatKernelcan_realize_into_without_copyrW   r3   rq  r0   )rn   r	  r   should_lower_cat_inputrr  s      rT   r  z#cat.<locals>.should_lower_cat_input  s     #A&& 	N1!EBBBJGQDDWMMMMa)R]344 	<))--*:*:;;;a&& 	4urV   c                 `   t          | t          t          j        f          r  |                     S t          | t          j                  sdS |                                 j        }|                                 D ]-}| t          j	        
                    |                    z  }.|S r   )rW   r3   r&   rq  r0   inner_fn_opcountnum_opsr{  r@   r   rx  )rn   country  op_countrr  s      rT   r  zcat.<locals>.op_count  s    a)R]344 	.8MM!,,--- !R\** 	1""$$,$$&& 	8 	8DXXag0066777EErV   r   r!   r   c                 J    | t           j        j        t           j        j        fv S r]   )ru  catrk  constant_pad_ndr   s    rT   additional_pointwise_opsz%cat.<locals>.additional_pointwise_ops  s    dh&(<(DEEErV   c              3   6   K   | ]} |          k    V  d S r]   rj   )rm   r?  MAX_SIMPLE_OP_COUNTr  s     rT   rr   zcat.<locals>.<genexpr>  s1      CCq22CCCCCCrV   c              3   8   K   | ]}t          |          V  d S r]   )r:   )rm   user  s     rT   rr   zcat.<locals>.<genexpr>  sB       
 
 S":;;
 
 
 
 
 
rV   c              3   .   K   | ]} |          V  d S r]   rj   rm   r   r  s     rT   rr   zcat.<locals>.<genexpr>  s/      >>&&s++>>>>>>rV   c              3   .   K   | ]} |          V  d S r]   rj   r  s     rT   rr   zcat.<locals>.<genexpr>  s@       "
 "
,/""3''"
 "
 "
 "
 "
 "
rV   c              3   .   K   | ]} |          V  d S r]   rj   r}  s     rT   rr   zcat.<locals>.<genexpr>  s/      <<((++<<<<<<rV   )!r   r+  r   rI  require_channels_lastru  r  rh  rk  r   r^  r  r   r   r  r   r3   r&   rq  IRNoder   r   r  r   rL   rX   rY   r$   max_pointwise_cat_inputsr@   r  rC  r+  )r  r   
cpu_devicerP  r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_usehorizontal_fuse_catr  r  rz  r   ru  r  r  rr  s             @@@@@@@@rT   r  r  {  s   %%'',5J 	?c  DJ     	?
  	 	EMMOOOO>>v>>>>> 	A-dh@@@@IFA1 011&#>>>
6{{aVAY
q	3
*
*C	%D%L E 6555f555F
y"-78 
RY 
 
 
 
U U U

 

 

 

 

 

 

 BBBB6BBBBBT       "  >//<<===     $ !"FUZ%: F F F F 6{{///	V7	7	7CCCCCFCCCCC 
8  
 
 
 
~+
 
 
 
 
 >>>>v>>>>>Q> 	 " "
 "
 "
 "
39"
 "
 "
 
 
 =<<<<V<<<<<< 	  	."5 	.>O 	. ---R_++FC88999rV   offsetdim1dim2c                   	
 |                                  
t          
          	t          	          t          	          t          k    fd           t          j        j                            t          j	        |d                    }|rXt          j        j        
                    t          j        j                            
         |z   
                   d          }nWt          j        j        
                    t          j        j                            
         
         |z
            d          }d|r| dfnd|ffdt          
          D             }|                    |           	
fd}t          t          j                            | ||                    S )N)r  rankc                      d  d S )Nz(diagonal dimensions cannot be identical z, rj   )r  r  s   rT   r  zdiagonal.<locals>.<lambda>  s    WWWQUWW rV   r   )r   r   c                 &    g | ]\  }}|fv|S rj   rj   )rm   r   r  r  r  s      rT   ro   zdiagonal.<locals>.<listcomp>  s+    NNN41a$8M8MQ8M8M8MrV   c                 
   | d         }dgt          	          z  }d}t                    D ]<}|k    r|d         z   ||<   |k    r|d         z   ||<   ,| |         ||<   |dz  }=|t          	          dz
  k    sJ |S )Nr  r   r#   r!   )r   r   )
r  diag_idxoriginal_idxcur_dimr  base_idxr  r  num_dimsoriginal_shapes
        rT   	reindexerzdiagonal.<locals>.reindexer  s    r7sS000x 	 	ADyy"*Xa["8Qd"*Xa["8Q"%g,Q1#n--11111rV   )r   r   r   r   r@   r   r   r  r   Ltevaluate_maxevaluate_minr   r   r3   r&   GenericViewr   )rP  r  r  r  offset_negative	diag_sizer  r  r  r  r  s     ``    @@@rT   diagonalr    s   ^^%%N>""H8444D8444D	WWWWW   g&44UXfa5H5HIIO 
G$11G))t$v-~d/C  	
 
		 G$11G))t$nT&:V&C  	
 
	 H GQ<v;NNNNN9^44NNNE	LL          R^**5%CCDDDrV   c                 @    t          t          | |||                    S r]   )r^  r  )rP  r  r  r  s       rT   diagonal_copyr  ,  s    %t44555rV   c                 h    t          |           }t          ||||          }t          ||           |S r]   )r^  r  	mutate_to)rP  srcr  r  r  r   rD  s          rT   diagonal_scatterr  1  s5    5\\FffdD11FfcMrV   c           	          t          j        ||                                 |                   }t          t	          | |||dz             |          S r  )r5   handle_negative_indexr   r  r  )rn   r   r  s      rT   selectr  9  sC    

$S!**,,s*;
<
<C6!S#sQw//555rV   c           
         t          | |d          }t          |t          j                  r$t          j        j                            |          }t          |t          t          j	        f          rKt          j        j                            | 
                                |                   }|g||z   dz
  |z  z  }g }d}|D ]0}||z   }|                    t          | ||||                     |}1|S )Nr   r#   r  )r  rW   r   r   r@   r   r   r  r   r   r   r   r  )	rn   r  r   r  x_sizer  r  r  r  s	            rT   splitr  ?  s    
3
"
"C%$$ >  66u==%#u}-.. :!77

S8IJJFUNQ.589FE  dlfQUCu===>>>MrV   c                 (    t          | ||d          S )NFr  )r  )rn   r  r   s      rT   split_with_sizesr  R  s    E3e,,,,rV   c                    t          | |d          }t          j        j                            |                                 |                   }g }t          |          D ]&}|                    t          | ||                     '|S r   )	r  r@   r   r   r  r   r   r   r  )rn   r   r  r  r   s        rT   unbindr  W  sx    
3
"
"CW33AJJLL4EFFFF6]] ) )fQQ''((((MrV   c                    |                                  }t          |          }t          ||          |dk    rt          t	          | d          |          S |         }t
          j        j        }|                    ||           |	                    d           t          ||z
            dz   }|                    |          dk    r9|                     |                    t          ||z  |                               g |d          ||dz   d          |}	fd}
t          t          j                            | |	|
                    S )Nr   )r  r#   c                 d    | d         |          z  z   }g | d          || dz   d         R S )Nr  r#   rj   )r  dim_idxr   r  s     rT   r  zunfold.<locals>.reindexeru  sF    b'CHtO+8TcT8G8c#'B,&7888rV   )r   r   r   r  r  r@   r   r   	guard_leqguard_ltr   r  r  r   r3   r&   r  r   )rn   	dimensionr  r  r  ndimdim_sizer   new_dim_sizeout_sizer  r   s      `       @rT   unfoldr  a  s^   JJLLEu::D
4
+
+Cqyyi1oo40000SzHwHtX&&&aHtOT22Q6L(##a''	X''t0CX(N(NOOPPPDttDlDU3799-=DtDH9 9 9 9 9 9 R^**1h	BBCCCrV   c                     t          | |d          }t          |                                           }|                    |t	          j        d                     t          | |          S r  )r  rs   r   insertr   r   rv  )rn   r   r  s      rT   r  r  |  sX    
3
"
"CQZZ\\""IS%-**+++9rV   c                     t          | |          }t          | t                    sJ t          |t                    sJ |j        | _        | S r]   )r  rW   r3   r<  r  s      rT   
unsqueeze_r    sK    
As

Ca#####c9%%%%%XAFHrV   c                    t           j        j        j                            t          j        |                    }t          |                                           }|dk     r|||z   z  }d|cxk    r	||z   k     sn J |S r   )	r@   r   r   	shape_envr  r   sympifyr   r   )rn   r   r  r  s       rT   r  r    s    
'

$
2
25=3E3E
F
FCqzz||D
Qwwtf}####dVm######JrV   r  c                 *   t          | |d          }t          j        j                            |                                 |                   dz  }t          | |d|          }t          | |||dz            }t          |t          |                    S )Nr   r!   )	r  r@   r   r   r  r   r  r*  sigmoid)rn   r   new_lenr   r   s        rT   glur    s}    
3
"
"Cg44QZZ\\#5FGG1LGq#q'""Aq#w!,,Aq'!**rV   c                 J     |rt                                            fd}|S )Nc                  d    d }t          j        |t          j        j        g| R i |          S )Nc                 b    t          | t          j                  rt          j        |           n| S r]   )rW   r&   r  r3   r   r   s    rT   wrap_tensorsz7fallback_handler.<locals>.handler.<locals>.wrap_tensors  s)    *4Q	*B*BI9#A&&&IrV   )pytreetree_mapr&   FallbackKernelr   )r   r   r  kernels      rT   handlerz!fallback_handler.<locals>.handler  sN    	J 	J 	J "+26KDKKKFKK
 
 	
rV   )rC   rv   )r  rc  r  s   `  rT   rh  rh    s=     f
 
 
 
 
 NrV   c                  .    t          j        d           d S )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnrj   rV   rT   _warn_complex_not_supportedr    s#    Mt    rV   r?  c                     |                                  rQ|r?|j        t          j        j        j        j        t          j        j        j        j	        fv rdS t                       dS dS )z0Do not support reading or writing to this tensorFT)re  rD  rL   r?   ru  rv  r   ri  rj  rk  r  r?  parents     rT   unsupported_input_tensorr    sd    ||~~  	fmIN%IO08(
 
 
 5#%%%t5rV   c                 L    t          | |          rdS | j        ot          j        S )z2Do not support writing tensor but can read from itT)r  is_cpur$   disable_cpp_codegenr  s     rT   unsupported_output_tensorr    s)    6** t8222rV   rM  c                     | j         t          j        j        u rdS | j         t          j        j        u rdS d }t          j        | j        i | j        D ]} ||| d          r dS  || | d          S )NFc                 4   t          | t          j        j                  sdS d| j        vrdS t          j        | j        d                   D ]K}t          |t          j        j                  s"|rt          ||          r dS 8t          ||          r dS LdS )NFr  T)rW   rL   fxNoder-  r  tree_leaves_subclasses
FakeTensorr  r  )rM  r  	is_outputr-  s       rT   check_skip_conditionzCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition  s    $.. 	5	!!5&ty'788 		  		 DdE$5$@AA   ,T6::  44  ,D&99  44  urV   )r  T)	rD  ru  view_as_complexrk  lift_fresh_copyr  arg_tree_leavesr   r   )rM  allow_cpu_inputsr  r   s       rT   %fallback_node_due_to_unsupported_typer    s    {d*222u {d*222u  * %ty@DK@@  TU;;; 	44	  dd;;;;rV   c                    | t           vsJ d|              |rt          t          j        d                    rt	          | g          r}t
          j        r| t          j        j	        j
        v sYt          j        j        j        r0dt          j        j        _        t                              d           t          d|  d          fd}t!          | t          j        j                  r4|                                 D ]}t)          | |          } ||           d S t!          | t          j        j        t          j        j        f          r ||            d S t/          d|  d	t1          |                      )
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                     t          |            t          |             t          | d           t          |                     S Nr`  )rk   r|   r   rh  )op_overloadlayout_constraints    rT   register_fallbackz(make_fallback.<locals>.register_fallback  sU    !+...(!+/@AAAG $GGG[))
 
 	
rV   zUnsupported fallback z with type )r(   r   osgetenvr)   r$   fallback_randomrL   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningrR   rW   rX   rw   ry   r_   rY   HigherOrderOperatorRuntimeErrorr+  )r   r  r  r  olr  s    `    rT   make_fallbackr    s   ^###%VRT%V%V###
4!!
 t$$
 "
 em:OOO =/ 	38EM 0KKH   fR f f f
 
 	

 
 
 
 
 "ej122 N,,.. 	+ 	+B!"b//Kk****	+ 	+ 
B.
0NO	P	P N"L2LL$r((LLMMMrV   c                 P    d}| D ]}||z  }t          |t          j                  S )z
    TorchInductor offset calculation differs from PyTorch eager offset
    calculation for random ops (tl.rand vs torch.rand). In future, we should
    strive for same impl for tl.rand and torch.rand.
    r#   r   tensorrL   r  )r  numelr  s      rT   philox_rand_offsetr  !  s8     E  	%u{++++rV   c                 f  	
 t          j        || t           j                            |                                                     
|                                |                                		
fd}t          j        ||t          |                     }t          |           }||fS )Nc                 n   t          j         g           t          j                  }t          j         g           t          j                  }t          j        t          j         |           t          j                  |          }t          j        ||          }t          j        |          S r]   )r?   r   rL   r  rv   r  rand)	r  seed_index_exproffset_index_exprrand_index_exprr  r   offset_loader
random_posseed_loaders	        rT   r&  zphilox_rand.<locals>.inner_fn:  s     ,{{2DDLr):):EKHH'N::e,,ek::<M
 
 
 
 |FE***rV   r'  )
r&   r  FlexibleLayoutcontiguous_stridesmake_indexerr  r0   r   rs   r  )r  seedr  r  r(  r   r&  random_values_nodeoffset_noder  r  r  s        `   @@@rT   philox_randr&  -  s     
,,T22	 
 lnn  ""$$K&&((M+ + + + + + + + #)Dzz	   %T**K{**rV   c           	          t           j        rNt          j        t          j        t          j                            t          j	        j
        | ||                    S t          d          )Nz&should be handled in replace_random.py)r$   r  r  r  r3   r   r&   r  ru  native_dropoutrk  rR   )rn   ptrains      rT   r(  r(  T  sY     G$$T%8%@!QNN
 
 	

 EFFFrV   c                 t   t           j        s4|                                 t          j        d          k    s
J d            |                                  t          |          dk    st          |d         t                    rt          j
        j        nt          j
        j        }t          j        || g|R   | S )Nrh  Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r$   r  r   rL   r(  rI  r   rW   r   ru  
bernoulli_Tensorr&   InplaceBernoulliFallback)rn   r   r  s      rT   r-  r-  _  s    ! ^Q\\^^u|8 8 & & &]& & & IIKKK t99>>ZQ77> 	_# 
 Q66666HrV   c                     t           j        s4|                                 t          j        d          k    s
J d            t          t          |           g|R  S )Nrh  r,  )r$   r  r   rL   r(  r-  r^  )rn   r   s     rT   bernoulli_pr1  n  sh    ! ^Q\\^^u|8 8 & & &]& & & eAhh&&&&&rV   c                     t           r]   rR   r   s    rT   _foobarr5  w  s    
rV   c                 :    t                               d           d S )Nz1using triton random, expect difference from eager)r  info)salts    rT   _warn_triton_randomr9  |  s    HH@AAAAArV   c                  B    t          t          j        j                   d S r]   )r9  r@   r   creation_timerj   rV   rT   warn_triton_randomr<    s    -.....rV   c                      |                     dd           t          | i |S t          j        r"|                    dd            t          | i |S t          d          N	generatorz-should have been handled in replace_random.py)r.  fallback_rand_generatorr$   r  popfallback_rand_defaultrR   r   r   s     rT   r  r    sj    zz+t$$0&7777		 6

;%%%$d5f555
H
I
IIrV   c                      |                     dd           t          | i |S t          j        r"|                    dd            t          | i |S t          d          r>  )r.  fallback_randn_generatorr$   r  rA  fallback_randn_defaultrR   rC  s     rT   randnrG    sj    zz+t$$0'8888		 7

;%%%%t6v666
H
I
IIrV   c                 j    t          j        |          }t           j                            | |          S r]   )r&   get_stride_orderExternKernelrequire_stride_order)input_tensorr  stride_orders      rT   inductor_force_stride_orderrN    s*    &v..L?//lKKKrV   c                      t          d          )Nz.should be handled in fuse_seed_creation_pass()r3  )r(  s    rT   inductor_seedrP    s    
I
J
JJrV   c                     t                       t          j        t          j        | t          |                              S r]   )r<  r3   r   r&   RandomSeedsr7   )r  r(  s     rT   inductor_seedsrS    s3    BN5-2G2GHHIIIrV   c                       fd}t          j                                                                          |g           S )Nc                 R    t          j                                                  S r]   )r?   	load_seedget_name)r   r  seedss    rT   r&  z&inductor_lookup_seed.<locals>.inner_fn  s    }U^^--u555rV   r'  )r0   r   r   r   )rX  r  r&  s   `` rT   inductor_lookup_seedrY    s`    6 6 6 6 6 6 !!oo	   rV   r  r  r#  modec                  	 t           j        rJ dv sJ g | } t          j        }|                                }t          j        ||| t
          j                            |           |          	                                |
                                		fd}t          j        |||g |           }|                                 |S )N)r  rG  rZ  c                      t          t                     g           t          j         |           t          j                            S r]   )r_   r?   r  rL   r  )r  r[  r  r  s    rT   r&  z!inductor_random.<locals>.inner_fn  sG    !wsD!!KOON::e,,ek::
 
 	
rV   r'  )r$   r  rL   r9  r   r&   r  r   r!  r"  r  r0   r   rI  )
r  r#  r[  r  r   r(  r&  r  r  r  s
     `     @@rT   inductor_randomr^    s    %%%%$$$$$T7DME__FtR.AA$GGPV  lnn  ""$$K
 
 
 
 
 
 
 w	  F NNMrV   lowhighc                j   	 t           j        rJ g |}t          j        }|                                }t          j        |||t
          j                            |          |          	                                |
                                	 	fd}t          j        |||g |          S )NrZ  c           	         t          j         g           t          j         |           t          j                  t          j        t          j                  t          j        t          j                            S r]   )r?   	randint64r  rL   r  r  )r  r`  r_  r  r  s    rT   r&  z"inductor_randint.<locals>.inner_fn  s_    }KOON::e,,ek::N3,,N4--	
 
 	
rV   r'  )r$   r  rL   r  r   r&   r  r   r!  r"  r  r0   r   )
r_  r`  r  r#  r  r   r(  r&  r  r  s
   ``      @@rT   inductor_randintrd    s     %%%%T7DKE__FtR.AA$GGPV  lnn  ""$$K
 
 
 
 
 
 
 
 w	   rV   	out_int32right
boundariesrf  rg  c                   t                                                    dk    sJ t          j                            | t
          j                  r*t          j                            t
          j                  s- t          t          j	        j
        d          | |          S                                                                  d         |                                 }|                                 |rt          j        nt          j        fd}t#          j        |||                                           S )Nr#   Frb  re  r   c                 r     |           }t          j        |                                          }|S r]   )r?   	bucketizerW  )r  r  r   rh  boundaries_sizeindex_dtyperD  rg  s      rT   r&  zbucketize.<locals>.inner_fn  sE    l5!!-!!
 
 rV   r'  )r   r   r@   r   rF  rG  	BUCKETIZErh  ru  rk  r.  rI  r   r  rL   r  r  r0   r   )	rP  rh  rf  rg  r(  r&  rl  rm  rD  s	    ` `  @@@rT   rk  rk    s_    z""$$%%**** 	
E>#;<<
G
N,DEE
 R 55QQQ:%
 
 
 	
  ))++A.OF$$&&L!*;%++K
 
 
 
 
 
 
 
 
 ~~	   rV   c                 r    t          j        t          j        t          j        j        ||f          \  }}||fS r]   )r  tree_map_onlyr&   r  rJ  require_stride1r   r   r   s      rT   require_densers  )  s6    '
	2?2T6N LD& <rV   c                 r    t          j        t          j        t          j        j        ||f          \  }}||fS r]   )r  rp  r&   r  rJ  require_contiguousrr  s      rT   ru  ru  0  s6    '
	2?5f~ LD& <rV   c                 r    t          j        t          j        t          j        j        ||f          \  }}||fS r]   )r  rp  r&   r  rJ  r  rr  s      rT   r  r  7  s6    '
	2?84. LD& <rV   c                      d t          fdt          | j                  D                       } fd|                                D             }||fS )Nc                     t          | t          j                  rQt          j        |j        d                                                   }t          j                            | |          S | S Nr  )rW   r&   r  rI  r-  r  rJ  rK  )r   fx_argrM  s      rT   apply_constraintz1constrain_to_fx_strides.<locals>.apply_constraint?  sW    c29%% 	K.v{5/A/H/H/J/JKKL?77\JJJ
rV   c              3   6   K   | ]\  }} ||          V  d S r]   rj   rm   r   rz  r{  s      rT   rr   z*constrain_to_fx_strides.<locals>.<genexpr>E  F        *5#vf%%     rV   c                 F    i | ]\  }}| |j         |                   S rj   r   rm   kr  r{  fx_nodes      rT   
<dictcomp>z+constrain_to_fx_strides.<locals>.<dictcomp>H  4    SSSDAqa!!!W^A%677SSSrV   rt   r   r   rE  r  r   r   r{  s   `  @rT   rP   rP   >  s           9<T7<9P9P    D TSSSSFLLNNSSSF<rV   ztorchvision::roi_alignc                      d t          fdt          | j                  D                       } fd|                                D             }||fS )Nc           
      T   t          | t          j                  s| S |j        d         }|                                }t          j        |          }|rT|d         dk    rHt          t          t          t          | 
                                                                        }|j        s t          j                            | |          S dt          | t                    sJ t          | 
                                          dvr| S fd}	 |                                   ||           r=t           j                            t          j                            |           |          S n# t(          $ r Y nw xY wfd}t          | j        t          j                  re ||           sZ ||                                           r=t           j                            t          j                            |           |          S t          j                            | |          S )Nr  r  r   r   r}   r~   c           	           t           fdt          t                                                     dz
            D                       }t          j        j                                                             d                   dk    o|S )Nc              3      K   | ]G}t           j        j                                                            |                   z  d k    V  HdS r   N)r@   r   r   r  
get_stride)rm   r   	ALIGNMENTrn   s     rT   rr   z`sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_realized_tensor.<locals>.<genexpr>n  sa       " " !++ALLNN1,=>>JqP" " " " " "rV   r#   r  )r   r   r   r  r@   r   r   r  )rn   aligned_stridesr  s   ` rT   is_aligned_realized_tensorzMsdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_realized_tensorm  s    ! " " " " "s1<<>>22Q677" " "  O
  **1<<>>"+=>> '&'rV   c                     t           j        j                            |                                 d                   z  dk    S )Nr  r   )r@   r   r   r  r   )rn   r  s    rT   
is_alignedz=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned  s4    G$..qzz||B/?@@9LQRRRrV   )rW   r&   r  r-  r  rI  rs   r   r   r   r   is_cudarJ  rK  r3   r  r@   r   try_match_insignificant_stridesrealize_inputAttributeErrorr<  r   r  )r   rz  meta_valmeta_striderM  r  r  r  s          @rT   r{  z)sdpa_constraint.<locals>.apply_constraintV  s/   #ry)) 	J;u%oo''*;77 	FL,11s3<<>>/B/B)C)C D DEEL 	K?77\JJJ 	#y)))))s||~~f,,J	' 	' 	' 	' 	'	NN))#.. w>>O11#66    	 	 	D		S 	S 	S 	S 	S ch,, 	:c?? :coo//00 7BB55c::K   33CFFFs   AE1 1
E>=E>c              3   6   K   | ]\  }} ||          V  d S r]   rj   r}  s      rT   rr   z"sdpa_constraint.<locals>.<genexpr>  r~  rV   c                 F    i | ]\  }}| |j         |                   S rj   r  r  s      rT   r  z#sdpa_constraint.<locals>.<dictcomp>  r  rV   r  r  s   `  @rT   sdpa_constraintr  S  s    3G 3G 3Gj     9<T7<9P9P    D TSSSSFLLNNSSSF<rV   )r  c                 
   |}|                                  |                                 k    r"t          ||                                            }|                                 |                                k    r"t          ||                                           }|                                 |                                k    r1t          ||                                           }t          |          S t          |          S r]   )r   r  r   r   r   r   r^  )selfr  non_blockingrn   r   s        rT   r_  r_  W	  s    ACNN,,,,a**++~~3==??**Q(())}}#,,..((Q((Szz88OrV   )memory_formatc          	          t          j        |                                 |                                 |                                 t          |                                                     S Nr'  )r0   r   r   r   r  rs   r   )rn   r  s     rT   r^  r^  e	  sR     ||~~kkmmAJJLL!!	   rV   c                    g }t          | t                    rt          | j        t          j                  rx| j        } t          | t          j                  rH|                    |                                            | j        } t          | t          j                  Ht          |           } t          |           } |r9| j        } |d d d         D ]}t          j        | |          } t          |           } | S r  )rW   r3   r<  r&   r  r   
get_layoutr^  )rn   reinterpret_view_layoutslayouts      rT   clone_preserve_reinterpret_viewr  p	  s    !!Y Jqvr7I$J$J FB.// 	$++ALLNN;;;A B.// 	 aLLaA F.ttt4 	. 	.F"1f--AAaLLHrV   r  c                b    fd}t          j        t          |          || g          S )Nc                 H    t          j        | d         z  z             S )Nr   r   r?   r  )r  r   r  r  s    rT   rH   ziota.<locals>.fn	  s%    ~dU1Xo5UCCCCrV   r'  )r0   r   r7   )lengthr  r  r   r(  requires_gradrH   s    ```   rT   iotar  	  sa    D D D D D D D V$$x	   rV   r   r  c           	      f   |                                  |                                 k    sJ |                                 t          | d          t          j        j                            t          j        d                    r| 	                                         z   t          j        j        
                    d           t          j        j                            | 	                                                    t          t          |          | 	                                          }|                                fd}t          j        |                                 |                                  |t#          | 	                                                    S )Nr   c           	          t          j        t          j        t          j        |          t          j                  t          j        t          j                             |            |                     S r]   )r?   r  eqr  rL   r  )r  r   r  
src_loaderr  s    rT   r&  z select_scatter.<locals>.inner_fn	  sd    yFs3x55uek22  JsOOHSMM
 
 	
rV   r'  )r   r  r  r@   r   r   r  r   r  r   r  r  r   r  r0   r   r   rs   )rn   r  r   r  r&  r  r  s     `` @@rT   select_scatterr  	  sf   ;;==CMMOO++++}}H
3
"
"Cw%%ehua&8&899 *

S))Gq%(((GeQZZ\\#%6777
3$$ajjll
3
3C""J
 
 
 
 
 
 
 
 ||~~kkmmAJJLL!!	   rV   c           	         	
                                   |                                 k    sJ                                  
t           d                                                    t          j                                       \  t                                                     }t          z
  dz
  z             |<   t          ||          }|                                		 
fd}t          j                                                                           |t                                                               S )Nr   r#   c                 2   	dk    rk    r
dk    r |           S t          j        |          t          j                  }t	          |           t          |          	z
  
          <   g }	dk    rW|                    t          j        |t          j        t          j	        	          t          j                                       k    rW|                    t          j
        |t          j        t          j	                  t          j                                       
dk    rz|                    t          j        t          j        t          |          	z
  d
          t          j                  t          j        dt          j                                       |sJ t          j        t           j        |          }t          j        |fdt%                    rdnd          }t          j        || |                     S )Nr   r#   c                                  S r]   rj   )src_idxr  s   rT   r  z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>	  s    JJw'' rV   r  )r?   r  rL   r  rs   r   r   r  r   r   r  r  r    r  r   r  r   r!  r   r  )r  r"  r'  src_valr  r   r  r  r  r  r  rn   r  s       @rT   r&  zslice_scatter.<locals>.inner_fn	  s   A::#//daii:c??".S5;77s))C5 0$77A::KKN5<#6#6DD    (??KKN5<#4#4ekBB    199KKN'C5(8!TBBEK  LEK00	    $//*''''' ##,AA
 

 yHSMM
 
 	
rV   r'  )r   r  r  r   r&   r  normalize_start_endrs   r   r   r0   r   r   )rn   r  r   r  r  r  src_sizer&  r  r  r  s   ` ````  @@@rT   slice_scatterr  	  sR   ;;==CMMOO++++}}H
3
"
"Czz||C H11!S%EEJE3AJJLL!!HS5[D1H5t<<HSM
h

C""J,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
\ ||~~kkmmAJJLL!!	   rV   c                     t          | t          t          f          r(t          |           dk    rt	          | d                   S | S r   )rW   rs   rt   r   _unwrapr   s    rT   r  r  	  s;    !dE]## A

qt}}HrV   r   r(  r  
pin_memoryc                :    t          |d t          j        fv d|            t          | d           t          t	                     t
                    rpt          j        npt          j                    g }t           t          j	                  r fd}nt           t          t
          f          r fd}nt                     dk    s5t           d         t          t
          f          rNt                     dk    r;|                    t          j        t                                           fd}n4t          j                            t          j         |                    S t%          j        t)          |          ||	          S )
Nlayout=r  c                 .    t          j                  S r]   r  r  r<  r   s    rT   r&  ztensor.<locals>.inner_fn
  s    >$...rV   c                 .    t          j                  S r]   r?   r  r  s    rT   r&  ztensor.<locals>.inner_fn
  s    <e,,,rV   r   r   c                       fdt                    dk    rt          j        d          S  dt                              S )Nc           	      l   | |k     sJ || z
  dk    rt          j        |                    S || z
  dz  | z   }t          j        t          j        t          j        d         t
          j                  t          j        |t
          j                             | |           ||                    S )Nr#   r!   r   )r?   r  r  r  r  rL   r  )r  r  midbinary_searchr<  r   r  s      rT   r  z/tensor.<locals>.inner_fn.<locals>.binary_search
  s    s{{{{;!##<UU;;;U{q(50yFuQx==S%+66  "M%--!M#s++  rV   r   )r   r?   r  )r  r  r<  r   s   `@rT   r&  ztensor.<locals>.inner_fn
  sh            4yyA~~|Au--- =CII...rV   r  r'  )rg   rL   stridedrW   r  r   r  get_default_dtyper   r   r   r   r   r   r@   r   add_tensor_constantr  r0   r   r7   )r<  r   r(  r  r  r%  r&  s   ``     rT   r  r  
  s   v$..0B&0B0BCCC:~|,,,'$--%% 3$2022!F$$$ $
	/ 	/ 	/ 	/ 	/ 	/ 	/ 
D5#,	'	' 
	- 	- 	- 	- 	- 	- 	- 
Ta:d1gs|<<TaemCII..///	/ 	/ 	/ 	/ 	/ 	/ 	/( w**LU6:::
 
 	
 V$$	   rV   c                     t          | t                    r&|t          | |          } |t          | |          } | S t	          | ||          S )Nr  )rW   r3   r   r  r  )r<  r   r(  s      rT   	as_tensorr  >
  sX    $	"" D%((DT6**D$eF3333rV   c                 8    t          | t          j                  S r7  r  )r<  s    rT   long_tensorr  I
  s    $ek****rV   c                    ddl m}  |t          j        j        j        t          j        j        j        d                   }t          |          dk    s
J |            t          t          |                                                    \  }}t          j        |||           }t          j                            |          |_        t          j                            |           t          j        j        j        d         }t#          |t$          j        t$          j        t$          j        f          r|j        j        S t1          j        |          S )Nr   )resolve_unbacked_bindingsunbacked_bindingsr#   r  )r  r  r@   r   r   r  r  r-  r   r  iterrE  r&   DynamicScalarregister_bufferr   register_operationrW   rL   SymIntSymFloatSymBoolrM  exprr   r  )r<  r  r  binding_symkeypathbufferr  s          rT   _local_scalar_denser  N
  s    OOOOOO 21	"AG$8$=>Q$R   !!Q&&&(9&&&   %6%<%<%>%> ? ?@@Kk7D99F'))&11FKGv&&& '

#E
*C#enemDEE "x}}S!!!rV   c                     d S r]   rj   )r<  rf   s     rT   _assert_scalarr  |
  s	     4rV   c                    | t          | t          t          f          st          d          rj        t          t          t          f          rfd}nat          t
          j                  rfd}n@t                                                    dk    sJ 	                                fd}t          j        ||t          |                    S )Nr   c                 .    t          j                  S r]   r  r  r   r   s    rT   r&  z_full.<locals>.inner_fn
  s    <u---rV   c                 .    t          j                  S r]   r  r  s    rT   r&  z_full.<locals>.inner_fn
  s    >%///rV   r   c                      g           S r]   rj   )r  value_loaders    rT   r&  z_full.<locals>.inner_fn
  s    <###rV   r'  )rW   r   r   r   r   r   r   r   r   r  r0   r   rs   )
fill_valuer(  r   r  r&  r   r  s     `  @@rT   _fullr  
  s'   Ej3,// GE74K4K %#u&& $	. 	. 	. 	. 	. 	. 	. 
E5;	'	' 
$	0 	0 	0 	0 	0 	0 	0 5>>##$$))))((**	$ 	$ 	$ 	$ 	$ Dzz	   rV   c                 H     t          t          |                    | fi |S r]   create_tensor_liketensor_constructor)rn   r  r   s      rT   r  r  
  s,    =0<<==aJJ6JJJrV   c                 "     d d d d dd d fd
}|S )NF)namesr   r(  r  r  r  c                    t          | d u d           t          |d t          j        fv d|            t          | d           t          |          }|pt          j                    }t          |          dk    rBt          |d         t          t          t          j	        f          rt          |d                   }|D ]}t          |t          j
                  rJ d |D             }t          |||          S )Nnamed tensorsr  r  r#   r   c                 6    g | ]}t          j        |          S rj   r  r  s     rT   ro   z5tensor_constructor.<locals>.inner.<locals>.<listcomp>
  s     ...AQ...rV   )rg   rL   r  r7   r  r   rW   rs   rt   Sizer  r  )	r  r   r(  r  r  r  r  r  r  s	           rT   r5  z!tensor_constructor.<locals>.inner
  s     	5D=/2226dEM224Ff4F4FGGGz><000v&&2022t99>>ja4
2KLL>a>>D  	3 	3A!!U\222222.....Z555rV   rj   )r  r5  s   ` rT   r  r  
  sA     6 6 6 6 6 6 6. LrV   )r  r   r  r(  r  r  c                    t          | d u d           t          |          }t          |          dk    rBt          |d         t          t
          t          j        f          rt          |d                   }t          |d ||||          S )Nr  r#   r   r   r  r(  r  )	rg   r7   r   rW   rs   rt   rL   r  empty_strided)r  r   r  r(  r  r  r  s          rT   r  r  
  s     u}o...6""F
4yyA~~*T!WtUEJ.GHH~T!W~~d%v*   rV   c                       dddddd fd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    NF)r   r(  r  r  r  c                B   t          | d           t          |d t          j        fv d|            ||                                 }nt	          |          }|p|                                 }t          |                                           } |||||          S )Nr  r  r  )rg   rL   r  r   r   r   rs   r   )rn   r   r(  r  r  r  r  creation_fns          rT   _constant_likez*create_tensor_like.<locals>._constant_like
  s     	z><0006dEM224Ff4F4FGGG=KKMMEE ''E)1<<>>AJJLL!!{fV

 
 
 	
rV   rj   )r  r  s   ` rT   r  r  
  s<     duTX
 
 
 
 
 
 
 rV   c                 :    t          t          |                     S r]   r  )r  s    rT   r  r  
  s    0<<===rV   c                      d d d d d fd
}|S )Nr  c                R   t          |t          t          f          sJ t          | d           t          |d t          j        fv d|            t          |          p|                                 }|p|                                 }d |D             }t          |||          S )Nr  r  c                 6    g | ]}t          j        |          S rj   )r   r   r  s     rT   ro   z7new_constant.<locals>._new_constant.<locals>.<listcomp>   s"    ///Qa  ///rV   )
rW   rs   rt   rg   rL   r  r   r   r   r  )rn   r  r   r  r(  r  r  s         rT   _new_constantz#new_constant.<locals>._new_constant
  s     $u.....z><0006dEM224Ff4F4FGGGU##4q{{}})1<<>>//$///Z555rV   rj   )r  r  s   ` rT   new_constantr  
  s7    tDT	6 	6 	6 	6 	6 	6 	6 rV   r  c                    ||                                  }||                                 }t          |d ||||          S Nr  r   r   r  )rn   r  r   r  r(  r  s         rT   	new_emptyr    sJ    }~d%v*   rV   c                   t          | t          t          f          sJ t          |t          t          t          d           f          sJ t	          | d           t	          |d t
          j        fv d|            t          |          pt          j                    }|pt          j	        d          j
        }t          d|||           }|                                 |j        j        }dgt          |           z  |j        _        t          |t           j                  sJ d | D             } |rd |D             nt           j                            |           }t!          j        ||| |          |_        |S )	Nr  r  r  r   )r  r(  r   r  c                 6    g | ]}t          j        |          S rj   r  r  s     rT   ro   z!empty_strided.<locals>.<listcomp>!  s     ***ELOO***rV   c                 6    g | ]}t          j        |          S rj   r  r  s     rT   ro   z!empty_strided.<locals>.<listcomp>#  r  rV   )r(  r   r  r  )rW   rs   rt   r+  rg   rL   r  r   r  r  r(  r  rI  r<  r   r%  r&   rt  r   r!  r  r  )r  r  r   r  r(  r  	pointwiser  s           rT   r  r    s    dT5M*****ftUDJJ788888:~|,,,v$..0B&0B0BCCC<5#:#<#<E/u|C((/F6TJJJI^ Fs4yyFKfb/00000**T***D 	8))&))))11$77 
 N	  FM rV   c                    ||                                  }||                                 }t          ||||||          S r  r  )rn   r  r  r   r  r(  r  s          rT   new_empty_stridedr	  0  sL     }~fE&J   rV   c                     d |D             }t          t          t          |                    |j                  }t          j                            | |          S )Nc                 V    g | ]&}t           j        j                            |          'S rj   )r@   r   r   r  r  s     rT   ro   z copy_strided.<locals>.<listcomp>?  s+    <<<ag((++<<<rV   )key)sortedr   r   __getitem__r&   rJ  rK  )rn   r  rM  s      rT   copy_stridedr  =  sP    <<V<<<F%F,,&2DEEEL?//<@@@rV   c                 l    |                     d          
J d             t          |          | fi |S )Nr   z(dtype should be handled by decomposition)r.  r  )r  r  r   s      rT   fullr  D  sD    ::g**,V***)j))$99&999rV   c                    t          | t                    sJ |                                dk    r"t          | |                                          S |                                t          j        k    sJ |                                 t                    dk    }t          | |          |rt          | dg          } dg|                                 |                                fd}t          j        |                                 |                                 ||                                          S )Nr   r#   c                     t          |           } t          j         |                              }t          |           dk    r|g} n|| <    |           S r   )rs   r?   indirect_indexingr   )r  
gather_idxr   index_loaderr  r  s     rT   rH   zgather.<locals>.fn_  s]    3ii*<<+<+<d3iHH
s88q==,CC!CHx}}rV   r'  )rW   r3   	get_numelr  r   r   rL   r  r   r  r   r  r0   r   r   )	rn   r   r  sparse_gradr  rH   r  r  r  s	    `    @@@rT   gatherr  J  s=    a#####AENN,,---??++++::<<DYY!^F
3
'
'C 1qcNNs}}H$$&&L        ||~~kkmm~~	   rV   c                 >  	
 |rJ t          | t                    sJ t          |t                    sJ dt          |                                          v sJ |                                 	|                                t          |                                          |                                 
g |                                
dd          	
fd}t          j        | 	                                |                                 |          S )Nr   r#   c                     t          |           t                    k    sJ |  d              | d                    }t          j        |d                   gg | d          z   } |          S )Nz != r   )r   r?   r  )r  	var_index
weight_idxindices_loaderindices_ndimr  weight_loaderweight_sizes      rT   rH   zembedding.<locals>.fn}  s    3xx3x==(((S*@*@h*@*@((("N3}}#566	+I{1~FFG K
K
 

 }Z(((rV   r'  )
rW   r3   strr   r  r   r   r0   r   r   )weightr   padding_idxscale_grad_by_freqsparserH   r  r  r  r   r!  s         @@@@@rT   	embeddingr'  p  s>   fi(((((gy)))))C))++,,,,,,&&((M((**Nw''))**L//##K6!!##6k!""o6H) ) ) ) ) ) ) ) )   ""  	   rV   c                     t          d  D                       sJ dd  D                          t          d  D                       rt          d          d t                     D             }t	          |          dk    s
J d            d gt	                     z  }t          |t           fd	|D                        D ]1\  }}|                                |k    rt          d
          |||<   2||fS )Nc              3      K   | ]H}||                                 t          j        t          j        t          j        t          j        fv V  Id S r]   )r   rL   r  r  r   rk  rm   r   s     rT   rr   z.check_and_broadcast_indices.<locals>.<genexpr>  sN        = 	
%+u{EJLL=== rV   z)indices must be int64, byte or bool. Got c                 :    g | ]}||                                 S r]   r   r*  s     rT   ro   z/check_and_broadcast_indices.<locals>.<listcomp>  s&    3d3d3daVWVcAKKMMVcVcVcrV   c              3   r   K   | ]2}||                                 t          j        t          j        fv V  3d S r]   )r   rL   r   rk  r*  s     rT   rr   z.check_and_broadcast_indices.<locals>.<genexpr>  sA        78q}%*ek22}}}} rV   zFallback for bool indicesc                 B    g | ]\  }}t          |t                    |S rj   r   r   s      rT   ro   z/check_and_broadcast_indices.<locals>.<listcomp>  s+    OOO1jI6N6NO!OOOrV   r   z"requires at least 1 non-None indexc                      g | ]
}|         S rj   rj   )rm   r   r   s     rT   ro   z/check_and_broadcast_indices.<locals>.<listcomp>  s    3S3S3S1GAJ3S3S3SrV   z.Fallback when indices is on a different device)r   r   rd   r   r   r   r   r   )r   r(  
valid_idxsnew_indicesr   rn   s   `     rT   check_and_broadcast_indicesr2    sg          g g g3d3d73d3d3dff	g g g
   <C     ? ""=>>>OO	' 2 2OOOJz??Q D&3w<<'KJ 13S3S3S3S
3S3S3S TUU  1 <<>>V##%&VWWWA
""rV   c           	      j    dt          dd                    D ]\  }}	|	|z
  dk    rd fdt                    D             g  t                    t                    z   d          d         }
rz   nd |
         z   |
d          z   f	d}|fS )NFr#   Tc                 *    g | ]\  }}||         S r]   rj   )rm   r   r  r  s      rT   ro   z2index_output_size_and_inner_fn.<locals>.<listcomp>  s"    MMMC6!9rV   r   c                 t  	 t          |           t                    k    sJ t                    t          
          k    sJ t                    }g }d         }rdn|}d}t          d         dz             D ]}||k    r||z  }|         6|t          |           k     sJ |                    | |                    |dz  }K|         }|J 
|         }|                    t          j         || |||z                      |	                     g || |d          }|n
 |          S )Nr   r  r#   r   )r   r   r   r?   r  )r  r  	new_indexfirst_tensor_indexstart_offsetnext_idxr   loaderr  r   indexed_sizer   indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizer  s            rT   rH   z*index_output_size_and_inner_fn.<locals>.fn  s   3xx3{++++++?##s<'8'88888;	+A.3Kqq9K~b)A-.. 	 	AL  D qz!#c((****  X///A(+)))#A  )s<,2E#EFGG#     

^
	 %,yy((92E2EErV   )r   r   r   )r  r   r@  rA  r=  r<  r  r   previouscurrentr8  rH   r>  r?  s   ````````    @@rT   index_output_size_and_inner_fnrD    sN   " $ 1CDD + +'X""&*#MMMM9W+=+=MMMKSKS&[)9)9C<O<O)O)Q)Q"RSK'* 
!K/ +++,,--./ 	F F F F F F F F F F F F FB ?rV   c                     t          | ||          \  }}}t          j        |                                 |                                 ||          S r  )index_impl_helperr0   r   r   r   )rn   r   r   r?  r&  r   s         rT   
index_implrG    sP    0GUCCK1||~~kkmm	   rV   c           
        	
 t          t          t          f          sJ |                                 
t	          |                                           \  }t          |          dk    s
J d            d D             }t          |d                                                            }|                                 fdt          t                              D             }|rd|v rd|vrt          d          fdt          t                              D             }t          ||||d |          \  }		
fd}||	fS )	Nr   z Must have at least one valid idxc                 >    g | ]}||                                 nd S r]   r  r*  s     rT   ro   z%index_impl_helper.<locals>.<listcomp>  (    SSSa!-q}}TSSSrV   c                 0    g | ]}|         
|         S r]   rj   )rm   r   r   r  s     rT   ro   z%index_impl_helper.<locals>.<listcomp>  s&    UUU!gaj>TF1I>T>T>TrV   z0index is out of bounds for dimension with size 0c                      g | ]
}|         S rj   rj   rm   r   r  s     rT   ro   z%index_impl_helper.<locals>.<listcomp>	      ;;;!F1I;;;rV   r6  c                 ,      |                     S r]   rj   )r  index_inner_fnr  s    rT   r&  z#index_impl_helper.<locals>.inner_fn  s    xs++,,,rV   )rW   rs   rt   r  r2  r   r   r   r   
IndexErrorrD  )rn   r   r   r@  r=  rA  r<  r?  r&  rP  r  r  s    `       @@@rT   rF  rF    s   ge}-----}}H9'1<<>>RRG^~"""$F"""SS7SSSO w~a01::<<==KZZ\\FUUUUUuS\\':':UUUL Ml""q';';KLLL;;;;uS\\':':;;;L"@	# 	# 	#K- - - - - - .00rV   c                     	 t          | |d          S # t          $ rA |                                   t          t          j        j        d          | |          cY S w xY w)NTr6  Frb  )rG  rd   rI  rh  ru  r  r.  rn   r   s     rT   r  r    sz    
!WD1111 
 
 
			M
 1uMMMw
 
 	
 	
 	

s    AAAc                 &    t          | |d          S NFr6  )rG  rS  s     rT   _unsafe_indexrV  '  s    a....rV   c                 @    t          t          |           |||          S r]   )
index_put_r^  rn   r   r   
accumulates       rT   	index_putr[  4  s    eAhh<<<rV   c                 D    t          t          |           |||d          S rU  )index_put_impl_r^  rY  s       rT   _unsafe_index_putr^  9  s!    588WfjNNNNrV   c                    |                                 |                                  k    r"t          ||                                            }|rt          | |          }t          | t	          |d         ||                     S r   )r   r  rv   r  r  )r  r   r   rZ  s       rT   index_put_as_masked_fillr`  >  ss    T__....%!2!233 !D%  T5UD99:::rV   c                 >   t          j                    }t          |          rN|s|rJ|sdnd}t          j        j        j                            dd           x}r| d| }|t          j        _        t          j
        t          j        j        j        | |||           | S )Nzindex put with accumulate.zdeterministic index put.stack_trace Found from : 
 )rL   $are_deterministic_algorithms_enabledr-   r@   r   r  r-  r.  disable_cudagraphs_reasonr&   IndexPutFallbackrD  )r  r   r   rZ  deterministicrf   rb  s          rT   index_put_fallbackrh  F  s    >@@M 0j 0M 0 !,((+ 	
 '.377tLLL; 	988;88C,/),3T7FJWWWKrV   c                 *    t          | |||d          S )NTr6  r]  r  r   r   rZ  s       rT   rX  rX  V  s    4&*DIIIIrV   c                 *    t          | |||d          S rU  rj  rk  s       rT   _unsafe_index_put_rm  [  s    4&*EJJJJrV   c           
         |                                 dk    rt          |          dk    r|d                                         t          j        t          j        hv r||d         }t          t          |                                          t          |                                                     D ]}t          |d          }t          | |g||          S t          j
                    rt          | |||          S |D ]D}|@|                                t          j        t          j        hv rt          | |||          c S E|                                 t                    }|rbt          |                                           rA|dk    rt          | dg          } t          | |||          } |dk    rt          | g           } | S t          ||                                           }	 t          ||                                           \  }}	n"# t"          $ r t          | |||          cY S w xY wd |D             }
t%          | t&                    sJ |                                  |dk    rt          | dg          } t+          ||	d                                                            }fdt          t          |                    D             }t-          ||	||
|d |          \  }}t/          ||          }t1          j        |                                 |                                 |                                |||rdnd           }t1          j        d t1          j        |           |          }t:          j                            |          |_         t:          j        !                    |           |dk    rt          | g           } | S )	Nr#   r   r  c                 >    g | ]}||                                 nd S r]   r  r*  s     rT   ro   z#index_put_impl_.<locals>.<listcomp>  rJ  rV   c                      g | ]
}|         S rj   rj   rM  s     rT   ro   z#index_put_impl_.<locals>.<listcomp>  rN  rV   r6  
atomic_addr(  r   r&  r%  output_indexerscatter_mode)"r  r   r   rL   r   rk  r   r   r  r`  rd  rh  r;   rv  r   r2  r   rd   rW   r3   rI  rs   rD  r   r&   Scatterr  rt  MutationLayoutSHOULDREMOVEr@   r   r  r   r  )r  r   r   rZ  r   r'  r   r  x_ndimr@  r=  rA  r<  expected_vals_sizer&  scatterr  r  s                    @rT   r]  r]  `  s    	aLLAAJ  ""uz5;&???qzs4==??++S-A-ABB 	' 	'AT2&&DD'tffjIII 133 E!$DDD  I I!2!2uz5;6O!O!O%dGVZHHHHH]]__F[[F B4>>CSCSTT Q;;qc??D!$DDQ;;b>>Dfdnn..//FE"=T__&&#
 #
  E E E!$DDDDDE TS7SSSOdI&&&&&LLNNN {{D1# w~a01::<<==K;;;;uS\\':':;;;L#A	$ 	$ 	$  F.//F j  nn##%%!%/9\\T  G 
%d++ F
 '))&11FKGv&&&{{D"~~Ks   1%H H65H6rb  c                   	 t          | |d          \  }}                                |                                 		fd}t          j        |                                 |                                 ||          S )NFr6  c                      j         t          j        k    r)t          j                    t          j                  }n            }t          j        | fd          S )Nc                  ,                           S r]   rj   )_unsafe_index_fnr  self_loaders   rT   r  z8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    KK8H8H8M8M,N,N rV   )r   rL   r   r?   r   r!  )r  mask_valr}  fillr'  mask_loaderr~  s   ` rT   r&  z&_unsafe_masked_index.<locals>.inner_fn  sf    :##|KK$4$4ejAAHH"{3''Hz($N$N$N$N$N$NPTUUUrV   r'  )rF  r  r0   r   r   r   )
r  r'  r   r  r%  r   r&  r}  r  r~  s
    ` `   @@@rT   _unsafe_masked_indexr    s    "3D'"O"O"OFA""$$K""$$KV V V V V V V V V   nn	   rV   c                     t          ||d          }|                                 fdt          t                              D             }t	          | ||d          S )Nr   c                 r    g | ]3}|         r't          |         |          |         d z
            nd4S r  r  )rm   r   r   r  s     rT   ro   z7_unsafe_masked_index_put_accumulate.<locals>.<listcomp>  sU        7>ajJgaj58)U1X\222d  rV   T)rZ  )r  r   r   r   r^  )rn   r'  r   r   masked_valueclamped_indicesr  s     `   @rT   #_unsafe_masked_index_put_accumulater    sv    vq))LJJLLE    s7||$$  O Q$OOOOrV   c                 R    t          j        |t          j        ||                     S r]   )r?   r;  r<  r   minmaxs      rT   r  r    s     ;sCKQ//000rV   c                 h    t          |           }t          ||||          }t          ||           |S r]   )r^  r  copy_)r  r  r  r  r  r   output_views          rT   as_strided_scatterr    s5    4[[FVT6>BBK	+sMrV   c                 <    t          t          |           |||fi |S r]   )scatter_r^  )rn   r   r  r  r   s        rT   ry  ry    s$    E!HHc5#88888rV   r  include_selfr  r  r  c          	      2   t          |t                    }t          | ||                                |r|                                nt	          |          |r|                                j        nd|          rt          j        | ||||||           |S d S )Nznot implr  )rW   r3   r>   r   r+  r   r&   ScatterFallback)r  r  r   r  r  r  r  src_is_tensors           rT   scatter_fallbackr    s     sI..M(7d3ii!.>J   	%	
 	
 	
 	
 4rV   r  c                    |dv sJ |Lt          t          j        t          j        j        j        j                  }t          || ||||          }||S |dk    rd}n|dk    rd}t          | ||||          S )N>   Nrv   multiplyr  rv   sumr  prod)
r_   ru  r  r@   r   r  rD  _overloadnamer  scatter_reduce_)r  r   r  r  r  r  fallback_results          rT   r  r    s    .....~dmQW-A-H-VWW*sE3v
 
 
 &""	:		4eS&999rV   c                 @    t          t          |           |||          S r]   )scatter_add_r^  rn   r   r  r  s       rT   scatter_addr  +  s    a#uc222rV   c                 (    t          | |||d          S )Nr  )r  r  s       rT   r  r  0  s    1c5#u555rV   c                 >    t          t          |           ||||fi |S r]   )r  r^  )rn   r   r  r  reduction_typer   s         rT   scatter_reducer  5  s&    588S%nOOOOOrV   )r  c          
          |dv sJ t          t          j                                                  dk    r dt          j                                        v s
J d            t	          t
                    rt                     t          t          j        j         |||          }|r|S t	           t                    sJ dt          |                                          v sJ t                                                     }|dk    rt           dg           t	          t                    r6t                                                    dk    rt          dg          t	          |t                    r6t          |                                          dk    rt          |dg          }|                                dk    r S t                                                       |                                t	          t                    r                                nd  fd} fd	}	d
 }
|st%          j                                                                           fd|                                |d           }t%          j        d t%          j                   |          }t.          j                            |          |_        t.          j                            |           t%          j                                                                          |	|                                | |
|                    }t%          j        d t%          j                   |          }t.          j                            |          |_        t.          j                            |           |dk    rt           g             S )N>   Nr  amaxaminmeanr  r#   twozKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r  r   r   c                                                      }t          |          }t          |           }t          j         |           |dk    rdn|         d          |<   |S )Nr   r#   F)wrap_neg)r   r   rs   r?   r  )r  r  r  indirect_idxr   r  r  s       rT   rs  z'scatter_reduce_.<locals>.output_indexerh  sk    5zzCyy1LDAIIqq5:
 
 
S rV   c                 l    r |           S t          j                                                  S r]   r?   r  r   )r  r  r  r  s    rT   rH   zscatter_reduce_.<locals>.fnr  s7     	7:c??" <T^^%5%5666rV   c                     | dk    rdS | J d S )Nr  rq  rj   r  s    rT   backend_reduce_strz+scatter_reduce_.<locals>.backend_reduce_stry  s     U??< >>>4rV   c                 R    t          j        d                                          S r   r  )r  r  s    rT   r  z!scatter_reduce_.<locals>.<lambda>  s    3<4>>3C3C#D#D rV   rr  )r   ru  r  ry   rW   r   r  r  r  r3   r"  r   r   rv  r  r  rI  r  r&   ru  r   rt  rv  r@   r   r  r   r  )r  r   r  r  r  r  r  r  rs  rH   r  zero_outr  ry  r  r  s   `` `          @@rT   r  r  :  s   BBBBBD **,,--22T)3355555T 655 #v #c""& !  O  dI&&&&&C))******t}}DqyyD1##y!! c#,,..&9&9Q&>&>3nn%## !ENN,<,<(=(=(B(BUQC  A
c
"
"CLLNNN$$&&L&0i&@&@J"""dJ      7 7 7 7 7 7 7    +:??$$..""DDDD>>##)
 
 
 ")$//
 

 g--f55	""6***
 j  nn~~%''//  G 
%d++ F
 '))&11FKGv&&&qyyD"~~KrV   scales_xnexactc                 &  
 |                                   |                                 |                                  d          
|                                 d           }d 
D             
t          |          k    sJ |}d t	          
|          D             t          |          D ]\  }}|d|z  |<   fd
fd}	t          j        |                                 | 	                                |	g ||          S )Nc                 V    g | ]&}t           j        j                            |          'S rj   r  r*  s     rT   ro   z&upsample_nearestnd.<locals>.<listcomp>  s+    JJJQqw55a88JJJrV   c                     g | ]
\  }}||z  S rj   rj   )rm   r   os      rT   ro   z&upsample_nearestnd.<locals>.<listcomp>  s     :::DAq!a%:::rV   rR  c                 z   t          j        | t          j                  } r2t          j        | t          j        dt          j                            } t          j        | t          j        |t          j                            } t          j        | t          j                  } t          j	        | |d          S )N      ?Fr6  )
r?   r  rL   r9  rv   r  r*  r   r  r  )rn   r>  r  r  s      rT   scale_fnz$upsample_nearestnd.<locals>.scale_fn  s     N1em,, 	=3<U];;<<AGAs|E5=99::LEK(($QE::::rV   c           	          |  d          }| d           } g |fdt          |          D                       S )Nc                 2    g | ]\  }}} |||          S rj   rj   )rm   r   r  r  r  s       rT   ro   z2upsample_nearestnd.<locals>.fn.<locals>.<listcomp>  s-    VVVJAq$88Aq$''VVVrV   )r   )r  rn   r   i_sizes
inv_scalesr  r  r  s      rT   rH   zupsample_nearestnd.<locals>.fn  sc    H!HxWaWVVVV#aW:U:UVVVW
 
 	
rV   r'  )
realize_hintr  r   r   r   r   r0   r   r   r   )rn   r?  r  r  r  batcho_sizesr   r>  rH   r  r  r  r  s      ``     @@@@rT   upsample_nearestndr    sZ    NN}}HjjllA233GJJLL1"EJJ'JJJGx==AG::C$9$9:::Jh'' ( (5%KJqM	; 	; 	; 	; 	;
 
 
 
 
 
 
 
 
 ||~~kkmm!!!	   rV   c                 *    t          | ||fd          S )Nr#   r  r  rn   r?  r,  s      rT   upsample_nearest1dr    s    avi1====rV   c                 ,    t          | ||fdd          S )Nr#   Tr  r  r  r  s      rT   _upsample_nearest_exact1dr    s    avi1DIIIIrV   scales_hscales_wc                 ,    t          | |||fd          S )Nr!   r  r  rn   r?  r  r  s       rT   upsample_nearest2dr    s      ax.BaHHHHrV   c                 .    t          | |||fdd          S )Nr!   Tr  r  r  s       rT   _upsample_nearest_exact2dr    s"     ax.BatTTTTrV   scales_dc                 .    t          | ||||fd          S )Nr}   r  r  rn   r?  r  r  r  s        rT   upsample_nearest3dr    s#     ax8.LPQRRRRrV   c                 0    t          | ||||fdd          S )Nr}   Tr  r  r  s        rT   _upsample_nearest_exact3dr    s-     	;8X6!4   rV   c                 :     t           fd|D                       S )Nc              3   B   K   | ]}t          j        |          V  d S r]   r  )rm   r   r   s     rT   rr   z$_create_constants.<locals>.<genexpr>  s/      66Aa''666666rV   )rt   )r   r   s   ` rT   r8  r8  
  s&    6666666666rV   c                     |                                  |                                 fd}t          j        |                                 |                                 |          S )Nc                     t          |           } t          |           t                    k    sJ D ]}|         dz
  | |         z
  | |<    |           S r  )rs   r   )r  r   r  r  r  s     rT   r;  zrev.<locals>.loader  sd    3ii3xx3u::%%%% 	3 	3Cc
Q#c(2CHHx}}rV   r'  )r  r   r0   r   r   r   )rn   r  r;  r  r  s    ` @@rT   revr    s|     }}HJJLLE       ||~~kkmm	   rV   c                 ^   t          |          dz  dk    sJ t          d |D                       rt          |           S |                                 }t	          t          t	          t          |d d d         |dd d                                                 t          |          t                    z
  g D ]>\  }}                    t          j	        j
                            |          |f           ?t	          |d                    }g t          |d                    D ]J\  \  }}}	                    |	           |                    t          j        |	|z   |z                        Kt          |          t          |          k    sJ  t          |                                                     fdfd}
|                                 t#          j        |                                 |                                 |
|          S )Nr!   r   c              3   "   K   | ]
}|d k    V  dS r  rj   rm   r)  s     rT   rr   z"constant_pad_nd.<locals>.<genexpr>'  s&      
#
#a16
#
#
#
#
#
#rV   r#   c                 f    g }t           	d                    D ][\  }\  }}}|dk    r#|                    t          |d                     |dk    r#|                    t          ||                     \t	          j        t          j        |          }t          j        | 
fd          S )Nr   c                                  S r]   rj   )r  r  s   rT   r  z/constant_pad_nd.<locals>.mask.<locals>.<lambda>D  s     rV   )	r   r   range_mask_lowrange_mask_highr   r  r?   r   r!  )r  r'  r  r_  r`  r  boundsr  
mask_sizesr  r  s   `     rT   r'  zconstant_pad_nd.<locals>.mask<  s    (+E!""Ivz(J(J 	: 	:$C#tfaxxN322333qyyOC88999$//z$ 7 7 7 7 7DDDrV   c                     t          | d                    }t          | d                    D ] \  }\  }}|                    ||z
             !t          |          t          |           k    sJ  |          S r]   )rs   r   r   r   )r  r7  r  r_  r`  bounds_precompr'  r  s        rT   	offset_fnz"constant_pad_nd.<locals>.offset_fnF  s    rrOO	 #E!""I~ > > 	( 	(C#tS3Y''''9~~U++++tIrV   r'  )r   r   r^  r   rs   r   r   r   r@   r   r   lookup_precomputed_sizer   r   r   r   r  r0   r   r   )rn   paddingr  r  lhr?  r_  r`  r  r  r  r  r'  r  r  r  s     `        @@@@@@rT   r  r  $  sT   LL1""""

#
#7
#
#
### QxxJJLLE(4GCCaCL'!$Q$- @ @AABBCCFE

S[[ A 68N P P1qw/GGJJANOOOOuRaRy//KJ qrr33 < <dT$5<s
T(9::;;;;{s5zz))))-q{{}}--j99JE E E E E E E E E       }}H||~~kkmm	   rV   r   c                     t          j        t          j        | t          j                  t          j        t          j        |          t          j                            S r]   )r?   r  r  rL   r  r   r   )r   r_  s     rT   r  r  V  sB    6q%+&&u}S))5;77  rV   c                     t          j        t          j        | t          j                  t          j        |t          j                            S r]   )r?   r  r  rL   r  )r   r`  s     rT   r  r  ]  s8    6q%+&&tU[))  rV   c                 d    t          j        t          | |          t          | |                    S r]   )r?   r   r  r  )r   r`  r_  s      rT   
range_maskr  d  s0    8q#4    rV   rR  c                                                         d                                           pdgz   fd}|S )Nr   c                    | d           |  d          t          j        t          j        	fdt	                    D                       }rt          j        |
fd          nt          j        |fd          S )Nc                 j    g | ]/}t          |         |         |         z   |                    0S rj   )r  )rm   r   r  ih	padding_hs     rT   ro   z=constant_boundary_condition.<locals>.load.<locals>.<listcomp>x  s<    WWWqZ1qtil2Yq\MBBWWWrV   c                  B     t                     g           S )Nr   )constant_boundary_condition)r   r  pad_fill_valueprefixrn   s   rT   r  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>}  s/    O3A~3OOO"fNrN  rV   c                  "     g            S r]   rj   )r  r  r  s   rT   r  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s    ((>V>b>*B*B rV   )r   r  r?   r   r   r!  )r  r'  r  r  r   r  r  r  r  r  rn   r  s     @@rT   r  z)constant_boundary_condition.<locals>.loadr  s    uuC455\HWWWWWWERUJJWWW
 
 PCJ           D"B"B"B"B"B"BJOO
	
rV   r   r  )	rn   r  r  r  r   r  r  r  r  s	   ````` @@@rT   r  r  k  s     	


cTUUA}}H$A39I
 
 
 
 
 
 
 
 
 
 
 
( KrV   c                    t          | d||         z  z   ||         dz
  z
  ||         dz
  z   ||                   }|rt          | d||         z  z   ||         dz
  z
  d||         dz
  z  z   ||                   }t          j        j                            |dz
  ||         z  | z
  ||         z
            dk    r?|dz  }t          j        j                            d|||         z  | z
  ||         z
             t          j        j                            ||z
            dk    r(t          j        j                            ||           d}n|}||fS )Nr!   r#   r   F)r   r@   r   r   r  r  r   )rn   r   kernel_sizer  r  	ceil_modex_outx_alts           rT   pooling_sizer    s]   	A
Nk!nq01VAY]CVAY E  GAJ+a.1"45VAY]8KKVTUY
 
 7%%uqyF1I&=&AGAJ&NOOSTTTQJEG&&q%&)*;a*?'!**LMMM7%%eem4499G))%777IIE)rV   c                     t          | d          } | d         | d         z  }|dk    pt          d |D                       S )Nr!   r   r#      c              3   "   K   | ]
}|d k    V  dS r  rj   r  s     rT   rr   z:should_fallback_max_pool2d_with_indices.<locals>.<genexpr>  s&      $=$=qQU$=$=$=$=$=$=rV   )r<   r   )r  dilationwindow_sizes      rT   'should_fallback_max_pool2d_with_indicesr    sJ    {A..Ka.;q>1K"=$=$=H$=$=$=!=!==rV   assert_fallbackc                   |dk    rddg}|dk    rddg}|s|}t          |d          }t          |d          }t          |d          }t          |d          }t          | t                    sJ t          |          dk    sJ t          |          dk    sJ t          |          dk    sJ t          |          dk    sJ t          |                                           dv sJ t          ||          }|||k    sJ |||||fS )Nr   r#   r!   r  )r<   rW   r3   r   r   r  )rn   r  r  r  r  r  use_fallbacks          rT   max_pool2d_checksr    s=    !||a&1}}q6 {A..K&!$$F7A&&GHa((Ha#####{q    v;;!w<<1x==Aqzz||&&&&:;QQL"....<??rV   c                 f   t          | |d          \  }}|                                  |                                 ^ }}}	t          |d|          \  }
}t          |	d|          \  }}| j        }|t
          j        u rdn.|j        rt          d          nt          j	        |          j
        }t          |          |
|gz   }d         sd         s|s|rt          | |d          n|                                 fd}t          j        |                                 |                                 t%          j        |d	          |
          }t          j        |                                 t
          j        t%          j        |d	          |
          }||fS )NFr  r   r#   -infr!   r  c                    | ^ }}}d }d }t          j        t          d                   t          d                             D ]\  }}|d         z  |z   d         z
  }	|d         z  |z   d         z
  }
 g ||	|
          }|rYt          j        |d         z  |z   t
          j                  }||}n)t          j        t          j        ||          ||          }||}t          j	        ||          }|r|S |S r  )
r   productr   r?   r  rL   rj  r  gtr;  )r  return_indexr  bhbwmaxvalmaxindexh_incw_incr  iwr  r  r  r  r  r  s                rT   rH   z/_low_memory_max_pool2d_with_offsets.<locals>.fn  s3   R%-+a.!!5Q#8#8
 
 	2 	2LE5 fQi%''!*4BfQi%''!*4B(,V,R,,--C Ou{1~'='EuzRR#$HH"yV)<)<eXNNH~S&11 	OMrV   r  r'  T)r  r  r   r  r   rL   r   rp  r   rs  r  rs   r  r  r0   r   r   r   r   r   rj  )rn   r  r  r  r  r  r   r  r  wh_out
ceil_mode1w_out
ceil_mode2r   	min_valuer  rH   r   offsetsr  s    ```                @rT   #_low_memory_max_pool2d_with_offsetsr(    s    1B	;51 1 1-K(A NN::<<LUAq$Q;SSE:$Q;SSE:GE EJ 	$6ReFmmmEK<N<N<R  E{{eU^+Hqz #WQZ #: # #.q)CCC==??       2 
||~~kkmm"2E:::	  C ||~~j"2D999	  G <rV   c                     |                                  fdfd}t          j        |                                 t          j        ||                                           }|S )Nc                 *   t          j        	t          j                  }t          j        |d         z  
d         z
  t          j                  }t          j        |d         z  
d         z
  t          j                  }|| z   }||z   }||z  |z   S r  )r?   r  rL   r  )r  r  r  r  w_inhbasewbaser  r  input_widthr  r  s            rT   increments_to_indexzF_low_memory_max_pool2d_offsets_to_indices.<locals>.increments_to_index  s~    ~k5;77rF1I~
:EKHHrF1I~
:EKHHU]U]Dy2~rV   c                     | ^ }}} 
g |||          }t          j        	t          j                  }||z  }|||z  z
  } ||||          S r]   r?   r  rL   r  )r  r  r  r  r  kw_constr  r  r/  kernel_widthoffsets_loaders           rT   offsets_to_indiceszE_low_memory_max_pool2d_offsets_to_indices.<locals>.offsets_to_indices  sp    R 1& 1" 1b 122<ek::("%(*+""5%R888rV   r'  )r  r0   r   r   rL   r  r   )	r'  r3  r.  r  r  r5  r   r/  r4  s	    ````  @@rT   )_low_memory_max_pool2d_offsets_to_indicesr6    s     ((**N      9 9 9 9 9 9 9 !!##k#!!	  G NrV   c                 d   dk    rddg|dk    rddg}st          |t                    sJ t                    dk    sJ t                    dk    sJ t                    dk    sJ t          |          dk    sJ t          |                                          dv sJ |                                  	 |                                 }n# t          $ r d }Y nw xY wt          |t                    rt          |j        j        t                    r|j        j        }	t          j
        d t          j        |	                                |	                                |	                                          |	          }
|
                                 |
                                }n(	 |                                }n# t          $ r d }Y nw xY w|d ur|d         dk    p|d uo|d         dk    }t          d |D                       rt!          | ||||          S |                                ^ }}|                                 ^ }|                                |                                 t%          |                                          }t'          fdt)          d         dz            D                       t'          fd	t)          d         dz            D                       z  }|d
k    rt!          | ||||          S |                                fd}t          j        |                                 |                                 ||          }|rt          j                            |          S |S )Nr   r#   r!   r  )r(  r   r  )r   r  r<  c              3   "   K   | ]
}|d k    V  dS r  rj   r  s     rT   rr   z3max_pool2d_with_indices_backward.<locals>.<genexpr>h  s&      
$
$a16
$
$
$
$
$
$rV   c           
   3      K   | ]@}t          |d          z  t          d |d          z
  d          z            z
  d          V  AdS r   r#   Nr  rm   r  r  r  s     rT   rr   z3max_pool2d_with_indices_backward.<locals>.<genexpr>u  j         	ANSQQ%7F1I$EFFFJJ     rV   c           
   3      K   | ]@}t          |d          z  t          d|d          z
  d          z            z
  d           V  AdS r#   r   Nr;  rm   r!  r  r  s     rT   rr   z3max_pool2d_with_indices_backward.<locals>.<genexpr>y  r=  rV   r	  c                 d   | ^ }}}t          j        |z  |z   t          j                  }|d         z   }|d         z   }t          j        t	          |d         z
  d         z   d                   t          j                  }t          j        t	          |d         z
  d         z   d                   t          j                  }t          j        t	          |d                   dz   t          j                  }t          j        t	          |d                   dz   t          j                  }t          j        |t          j        dt          j                            }t          j        |t          j        dt          j                            }t          j        |t          j        t          j                            }t          j        |t          j        t          j                            }d }	t                    D ]}
t                    D ]}t          j	        |t          j        |
t          j                            }t          j	        |t          j        |t          j                            }g |t          j
        t          j        |t          j        |t          j        dt          j                                      d         d          t          j
        t          j        |t          j        |t          j        dt          j                                      d         d          } |          } |          }t          j        ||          }|	5t          j        ||t          j        dt          j                            }	t          j        t          j        t          j        ||          t          j        ||                    |          }t          j        |t          j	        |	|          |	          }	|	J |	S )Nr   r#   Fr6  r  r  )r?   r  rL   r  r   r;  r  r<  r   rv   r  rM  r  r  r9  r   r  )r  r  r  r!  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   r'  grad_loaderh_window_sizer  indices_sizer  r  pooled_heightpooled_widthr  w_window_sizewidths                      rT   rH   z,max_pool2d_with_indices_backward.<locals>.fn  s   A^AIM5;??

N
N.QQ'&)3VAY??
 
 .QQ'&)3VAY??
 
 x6!95595;GGx6!95595;GG+gs|Au{'C'CDD+gs|Au{'C'CDDE3>-#M#MNNE3>,#L#LMM'' #	W #	WC]++ "W "WWWcl3&D&DEEWWcl3&D&DEE)Bs|Au{7S7S(T(TUU$R(#   )Bs|Au{7S7S(T(TUU$R(#  
  .~j99'K
33	|Z88#"yy#,sEM*J*J   HH 8F2u--F2u--   D  #yswx/K/KXVVHHE"WF ###rV   r'  )rW   r3   r   r   r  r  r  r<  r0   r&   rt  r   r   r   decide_layoutr   )fallback_max_pool2d_with_indices_backwardr  rs   r  r   r   rJ  r  )grad_outputrn   r  r  r  r  r  r   	gO_strider<  x_bufferx_strideis_channels_lastr  heightr   r  r  rH   r   rP  rQ  r  rR  rS  rT  rU  rV  s     ```               @@@@@@@@rT    max_pool2d_with_indices_backwardr_  8  s    !||a&1}}q6 a#####{q    v;;!w<<1x==Aqzz||&&&& **,,		    			 !Y Jqv{I$F$F v{$$((nn&&]]__  
 
 
 
 	   &&((	||~~HH 	 	 	HHH	 !,A!1A 3)A,!"3  
$
$8
$
$
$$$ 
8K(Iw
 
 	
 JJLLUFE&1&:&:&<&<#Q|((**N))++KAJJLL!!H     {1~)**    M      {1~)**    M
  -/KR8K(Iw
 
 	
 ##%%L9 9 9 9 9 9 9 9 9 9 9 9 9 9 9v 
%%''##%%	  C  44S999
s$   	C C-,C-<G G G r  c                 n    |                                  ^ }}}|                                 fd}|S )Nc           
         	
 |\  	|\  
|\  }}t          j        t          j        t          j        z   t          j                  t          j        |t          j                            t          j        t          j        
	z   t          j                  t          j        |t          j                                      }t          j        |	 
fd          S )Nc                  2     g  z   z             S r]   rj   )h_start_indexr  r  r  w_start_indexr  s   rT   r  z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s+    HHNvN}r'9N=2;MNOO rV   )r?   r   r  r  rL   r  r!  )r  
incrementsstart_indicesend_indicesh_end_indexw_end_indexr'  rc  r  r  rd  pad_valr  s   `      @@@@rT   r  z!pad_adaptive_loader.<locals>.load  s    B'4$}#. [xF}r15;??{EK88  F}r15;??{EK88 	
 	
 zOOOOOOOOO
 
 	
rV   r  )rn   rj  r   r  r!  r  r  s    `    @rT   pad_adaptive_loaderrk    sI    zz||HQ1}}H
 
 
 
 
 
, KrV   c                     t          j        | ||          }t          j        |||          }t          j        | ||          }t          j        |||          }	||||	fS )N)out_diminp_dim)r   r   )
start_index	end_indexh_inr+  r"  r$  rc  rh  rd  ri  s
             rT    compute_indices_adaptive_poolingrr    sm    %k5$OOOM#IudKKKK%k5$OOOM#IudKKKK+}kAArV   c                 l    |\  }}|\  }}	t          | |||||	          \  fd}
|
S )Nc                 <   | ^ }}} |          } |          } |          } |          }d }	t          j        t          d                   t          d                             D ]*\  }
} |||
|g||g||g          }|	|}	 ||	          }	+|	S r  )r   r  r   )r  r;  r  r  r  rc  rh  rd  ri  r  r  r  r  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                rT   rH   z _adaptive_pooling_fn.<locals>.fn  s    R((,,$nR((((,,$nR(('l1o(>(>lSTo@V@VWW 
	1 
	1FB&R.k*	 C ~#C00rV   rr  )ro  rp  rw  in_sizes	out_sizesrx  rq  r+  r"  r$  rH   ru  rv  ry  rz  s     `  `     @@@@rT   _adaptive_pooling_fnr~    s     JD$LE5 	)YdE5	 	
         . IrV   c                 p   
 |\  }|\  }}t          | ||||          \  

fd}	|	S )Nc                    | ^ }}} |          } |          } |          } |          }d }	d }
t          j        t          d                   t          d                             D ]\  }} ||||g||g||g          }t          j        ||z   z  |z   |z   t
          j                  }|
|}
n)t          j        t          j        ||	          ||
          }
|	|}	w ||	          }	|
S r  )	r   r  r   r?   r  rL   r  r  r  )r  r;  r  r  r  rc  rh  rd  ri  r  r  r  r  r  r  ru  rv  rw  rx  ry  r+  rz  s                  rT   rH   z)_adaptive_pooling_fn_with_idx.<locals>.fn.  s:   R((,,$nR((((,,$nR(('l1o(>(>lSTo@V@VWW 	1 	1FB&R.k*	 C N#t+m;b@%+ E  9SVC%8%8%JJ~#C00rV   r{  )ro  rp  rw  r|  r}  rx  rq  r"  r$  rH   ru  rv  ry  r+  rz  s     `  `    @@@@@rT   _adaptive_pooling_fn_with_idxr    s     JD$LE5 	)YdE5	 	
! ! ! ! ! ! ! ! ! ! !F IrV   c                 F    t           t                    sJ t          |          dk    sJ                                                                    ^ }}}t
          j        j                            |          }t
          j        j                            |          }|\  }}||k    r||k    rt                     S |dk    s|dk    r>g |||}t          |                                                                            S ||z  dk    r#||z  dk    r||z  ||z  g}t           |          S t          ||z   dz
  |          }	t          ||z   dz
  |          }
t          |          ||gz   }                                 }|	|
z  }|dk    rt!           |          S d }d }t#          |||	|
g||g||gt$          j                  t)          t+                                fd	}t-          j                                         |||
          }|S )Nr!   r   r  r#   r	  c                 (    t          | |z  |          S r]   r   r  rm  rn  s      rT   ro  z)_adaptive_avg_pool2d.<locals>.start_index|      7333rV   c                 :    t          | dz   |z  |z   dz
  |          S r  r  r  s      rT   rp  z'_adaptive_avg_pool2d.<locals>.end_index  %    g-7!;WEEErV   ro  rp  rw  r|  r}  rx  c                 p    t          j         | t                               |                     S r]   )r?   truedivrk  )r  fn_sumones_loaderrn   s    rT   rH   z _adaptive_avg_pool2d.<locals>.fn  s=    {F3+A..//[1I1I
 
 	
rV   r'  )rW   r3   r   r  r   r@   r   r   r  r^  r  r   r   
avg_pool2dr6   rs   fallback_adaptive_avg_pool2dr~  r?   rv   rk  	ones_liker0   r   )rn   r?  r  rq  r+  r"  r$  o_sizer  h_kernel_maxw_kernel_maxr  r   r  ro  rp  rH   rvr  r  s   `                 @@rT   _adaptive_avg_pool2dr  Y  sr   a#####{q    NNUD$711$77D711$77DLE5 u}}QxxzzUaZZ'5'%''V1;;==HHHHe|qTE\Q..u}dem4![)))D5L1,u55LD5L1,u55LE{{eU^+HKKMME-KR+A{;;;4 4 4F F F ""L1%.7  F &ill33K
 
 
 
 
 
 

 
	||~~	
 
 
B IrV   c                     t           t                    sJ t          |          dk    sJ                                                                    ^ }}}t
          j        j                            |          }t
          j        j                            |          }|\  }}|dk    s|dk    rlg |||}t          | 
                                                                           t          |t          j                                                   fS ||z  dk    rr||z  dk    ri||z  ||z  g}t          |ddg          rt           |          S t!           ||ddgddgd          \  }	}
t#          |
|d         ||ddg          }|	|fS t%          ||z   dz
  |          }t%          ||z   dz
  |          }t'          |          ||gz   } 
                                }||z  }|d	k    rt)           |          S d
 }d }t+          ||||g||g||gt,          j                  t1          ||||g||g||gt,          j                   fd} fd}t3          j                                         |||          }t3          j                                         t          j        ||          }||fS )Nr!   r   r  r#   )r  F)r  r  r  r  )r  r	  c                 (    t          | |z  |          S r]   r  r  s      rT   ro  z(adaptive_max_pool2d.<locals>.start_index  r  rV   c                 :    t          | dz   |z  |z   dz
  |          S r  r  r  s      rT   rp  z&adaptive_max_pool2d.<locals>.end_index  r  rV   r  c           	      R     | t          t          d                              S Nr  rk  r   )r  inner_func_max_valrn   s    rT   inner_fn_max_valz-adaptive_max_pool2d.<locals>.inner_fn_max_val  '    !!#':1eFmm'L'LMMMrV   c           	      R     | t          t          d                              S r  r  )r  inner_func_max_idxrn   s    rT   inner_fn_max_idxz-adaptive_max_pool2d.<locals>.inner_fn_max_idx  r  rV   r'  )rW   r3   r   r  r   r@   r   r   r  r  r   r   rL   r  r  max_pool2d_with_indicesr(  r6  r6   rs   fallback_adaptive_max_pool2dr~  r?   r;  r  r0   r   )rn   r?  r  rq  r+  r"  r$  r  r  r  r'  r   r  r  r  r   r  ro  rp  r  r  r  rir  r  s   `                      @@rT   adaptive_max_pool2dr    s   a#####{q    NNUD$711$77D711$77DLE5zzUaZZ'5'%''V1;;==HHH%%+allnnK
 K
 K
 
 	
 e|qTE\Q..u}dem42;!QPPP 	*1k:::<"AQ  JAw @Q{QF  G g:D5L1,u55LD5L1,u55LE{{eU^+HKKMME-KR+A{;;;4 4 4F F F ."L1%.;   7"L1%.;  N N N N N NN N N N N N 
	||~~!	
 
 
B 
	||~~k!	
 
 
B r6MrV   c                                                 t          z
  dz
                                              fd}|S )Nr#   c                 6    g |           }t          j        |                                          }t          j                                                  }t          j        ||z   |z            t          j        ||z            z
  }t          j        |t
          j                  }t          j        |t          j        dz
  t
          j                            }t          j        ||t          j        	
z
  t
          j                            S r  )	r?   r  r   r  r   rL   r  r  r  )r  r   samplei_expr
alpha_exprseq_ir'  r  r   in_sz	kernel_szout_szsamplessamples_loaders          rT   r  z)_fractional_pooling_offsets.<locals>.load	  s    &#//7#4#4#6#677^E7+<+<+>+>??
	6F?j899CIZ=
 =
 
 UEK00vN6A:u{33
 
 yucnUY5F&T&TUUUrV   )r   r  )r  r  r  r  r   r  r  r  s   ````` @@rT   _fractional_pooling_offsetsr    s    C[F#JE#Iuy(&1*55E((**NV V V V V V V V V V V KrV   c                    |                                   |                                 ^ }\  }}|\  }}||z  dk    rt          | ||          S t          j        t
          |g|          }	 |	d           |	d          |                                 fd}
t          |          ||gz   }t          j	        | 
                                |                                 t          j        |
d          |	          }t          j	        | 
                                t          j        t          j        |
d
          |	          }||fS )Nr	  )r  r  r  r  r   r  r#   c           	         | ^ }}}t          j         ||                    }t          j         ||                    }d }d }t          j        t	          d                   t	          d                             D ]\  }	}
 g |||	z   ||
z             }|r~t          j        ||	z   z  |z   |
z   t          j                  }||}nNt          j        t          j	        t          j
        ||          t          j        |                    ||          }||}t          j        ||          }|r|S |S r  )r?   r  r   r  r   r  rL   r  r  or_r  r  r;  )r  r  r  r  r  rc  rd  r  r  r  r  r  r  
h_index_fninp_hinp_wr  
w_index_fnr  s                rT   rH   z!fractional_max_pool2d.<locals>.fn3  sd   R-jj.D.DeLL-jj.D.DeLL'k!n(=(=u[QR^?T?TUU 	2 	2FB(LVL]R%7L9KLMMC 	"R'50=@2Eu{  #$HH"ysF 3 3SYs^^DDeX   H ~S&11 	OMrV   Fr   r'  T)r  r   fallback_fractional_max_pool2dr   r   r  r  rs   r0   r   r   r   rL   r  )rn   r  r?  random_samplesr  kernel_hkernel_wr"  r$  gen_offsets_for_dimrH   r  r  r  r  r  r  r  r  s    `            @@@@@rT   fractional_max_pool2dr    s   NN::<<UE5$HhLE5(b  -{K
 
 	
 $+#en   %$+++J$$+++J}}H         : E{{eU^+H		||~~kkmm"2E:::	
 
 
B 
	||~~k"2D999	
 
 
B r6MrV   c                                                                                         ^ }}}t          j        j                            |          }t          j        j                            |          }|^ }}}	||z  dk    r#||	z  dk    rt           ||z  ||	z  gd          S t          ||          }
t          ||	          }d fd}t          ||
|g||g||	gt          j
                   fd}t          j                                                                          |t          |                    }|S )	Nr   r#   )divisor_overridec                 L    t          | |z  t          j        |                    S r]   )r   r   r  r  s      rT   ro  z0upsample_nearest2d_backward.<locals>.start_indexs  s     uwg(>(>???rV   c                 $     | dz   ||          S r  rj   )r  rm  rn  ro  s      rT   rp  z.upsample_nearest2d_backward.<locals>.end_indexv  s    {EAI999rV   r  c                 6     | t                              S r]   )rk  )r  r  rn   s    rT   rH   z'upsample_nearest2d_backward.<locals>.fn  s    vc.q11222rV   r'  )r  r   r@   r   r   r  r  r6   r~  r?   rv   r0   r   r   r   rs   )rn   r?  
input_sizer  r  r  r  r  out_hout_wr  r  rp  rH   r  r  ro  s   `              @@rT   upsample_nearest2d_backwardr  a  s    NN::<<UE5G22599EG22599E%UE5u}eemq00!euneun=PQRRRR5%((L5%((L@ @ @: : : : : ""L1%.7  F3 3 3 3 3 3 
	||~~kkmmJ	
 
 
B IrV   rj   c           
      0    t          | ||||||d          S )Nr!   r  _avg_poolndrn   r  r  r  r  count_include_padr  s          rT   r  r    3     		 	 	 	rV   c           
      0    t          | ||||||d          S )Nr}   r  r  r  s          rT   
avg_pool3dr    r  rV   c           	         ssdgz  t                    t                    t                    t          | t                    sJ t                    k    sJ t                    k    sJ t                    k    sJ t          |                                           dz   dz   fv sJ |                                  |                                 d           }|                                  d          t          fdt                    D              \  }	}
t                    st          |
          rt          | d          d}n| 
                                d}t          |          t          |	          z   }|                                 t          j        t          j                  }|d	k    r?dk    rt"          }n d
k    rt$          }nt'          d            || |          S fd|r|r|rd|z  nd|z  fd}nfd}t)          j        |                                 ||          }|S )Nr   r#   r!   c           
      D    g | ]}t          |         |          S rj   )r  )rm   r   r  r  r  r  r  s     rT   ro   z_avg_poolnd.<locals>.<listcomp>  s?     

 

 

 1q+vw	JJ

 

 

rV   r  r  TFr	  r}   zUnknown dim: c                    | d           }|  d          d }t          j        	fdt                    D              D ]I
fdt                    D             } |g ||          }||}4t          j        ||          }J|S )Nc                 :    g | ]}t          |                   S rj   )r   )rm   r   r  s     rT   ro   z/_avg_poolnd.<locals>.fn_sum.<locals>.<listcomp>  s%    %P%P%PeKN&;&;%P%P%PrV   c                 V    g | ]%}|         |         z  |         z   |         z
  &S rj   rj   )rm   r   r   r  r  r  s     rT   ro   z/_avg_poolnd.<locals>.fn_sum.<locals>.<listcomp>	  s8    MMMQ1Q4&)#be+gaj8MMMrV   )r   r  r   r?   rv   )r  r;  r  totalr   r  r   r  r   r  r  r  s         @@rT   r  z_avg_poolnd.<locals>.fn_sum  s    UsdUJ#%P%P%P%PU3ZZ%P%P%PQ 	, 	,BMMMMMMM%**MMMC&6C))C}U++rV   rR  c                 h    t          j         |           t          j                            S r]   )r?   r*  r  )r  r   r  r>  r  s    rT   rH   z_avg_poolnd.<locals>.fn  s-    766#x00#,ue2L2LMMMrV   c                 4   | d 
          }| 
 d          }g }t          
          D ]}||         |         z  |         z
  }t          j        ||         z   |         |         z             }	s0t          j        |d          }t          j        ||                   }t	          j        ||z
  t          j                  }|                    |           t          j
        t          j        |          }t	          j         |           |          S r   )r   r   MinMaxr?   r  rL   r  r   r   r  r*  r  )r  r  r  divide_factorsr   hstarthendfactordivide_factorr  r   r  r  r  r  r  r  s            rT   rH   z_avg_poolnd.<locals>.fn  s   3$ZFcTUUBN3ZZ . .A*WQZ7y+a.!8!A$:KLL( 1"Yvq11F 9T1Q400Dvu{CC%%f----%,SWnEEM;vvc844mDDDrV   r'  )r<   rW   r3   r   r   r  r   r   r   r  r  rs   r   r   r  operatorr*  fallback_avg_pool2dfallback_avg_pool3d
ValueErrorr0   r   r   )rn   r  r  r  r  r  r  r   r  r"  
ceil_modeshad_paddingr  r  fallbackrH   r  r   r  r  r>  r  s    ````` `         @@@@@rT   r  r    so      #){C00K&#&&F7C((Ga#####{s""""v;;#w<<3qzz||q#' 22222NNJJLL3$E	

cTUUA

 

 

 

 

 

 

 

3ZZ

 

 

E: 7|| s: .q#3???==??E{{T%[[(HKKMME"8<==KR!88*HHAXX*HH2S22333x
 
 	
         E* E 	&((EE+%E	N 	N 	N 	N 	N 	N 	N 	N 	N
	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E  
	||~~	
 
 
B IrV   c                    dk    s
J d            ssddgt          | t                    sJ t          |t                    sJ t                    dk    sJ t                    dk    sJ t                    dk    sJ t          |                                          dv sJ |                                  |                                ^ }t          d|          \  }	}
t          d|          \  }}|                                 d         pd         p|
p||                                 ^ }t          |                                          }|                                }t          fdt          d         dz            D                       t          fdt          d         dz            D                       z  }|dk    rt          | ||          S fd	fd
}t          j        |                                 |||          }|S )Nr   divisor must be not zeror!   r  r#   c           
   3      K   | ]@}t          |d          z  t          d |d          z
  d          z            z
  d          V  AdS r:  r;  r<  s     rT   rr   z&avg_pool2d_backward.<locals>.<genexpr>b  r=  rV   c           
   3      K   | ]@}t          |d          z  t          d|d          z
  d          z            z
  d           V  AdS r?  r;  r@  s     rT   rr   z&avg_pool2d_backward.<locals>.<genexpr>f  r=  rV   r	  c           	         t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        t          j        | |          |          }t          j        t          j        ||          |          }	t          j        t          j        ||          t          j        t          j        t          j                  |                    }
t          j        t          j        |	|          t          j        t          j        t          j                  |                    }t          j	        |t          j        dt          j                            }t          j	        |	t          j        dt          j                            }	t          j        |
t          j        t          j                            }
t          j        |t          j        t          j                            }t          j        t          j        |
|          t          j        ||	                    }|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r#   )
r?   r  rL   r  rM  r*  r<  rv   r  r;  )rK  rL  stride_hstride_wpad_hpad_wr  r  r  wstartr  wendr  r^  r  r  r  rV  s                rT   !compute_pool_size_without_paddingz>avg_pool2d_backward.<locals>.compute_pool_size_without_paddingy  s   
 <q	5;77<q	5;77WQZ55WQZ55<A<<<A<<X..66X..66{GFH%%GCN65;77??
 
 {GFH%%GCN5%+66>>
 
 VS\!U[%A%ABBVS\!U[%A%ABB{4!D!DEE{4u{!C!CDDf 5 5swtV7L7LMMrV   c                    | ^ }}}|d         z   }|d         z   }t          j        t          |d         z
  d         z   d                   t          j                  }t          j        t          |d         z
  d         z   d                   t          j                  }t          j        t          |d                   dz   t          j                  }t          j        t          |d                   dz   t          j                  }t          j        |t          j        dt          j                            }t          j        |t          j        dt          j                            }t          j        |t          j        t          j                            }t          j        |t          j        t          j                            }d }t                    D ]}	t                    D ]}
t          j	        |t          j        |	t          j                            }t          j	        |t          j        |
t          j                            }}n"ssd         d         z  }n ||          }t          j
         g |t          j        t          j        |t          j        |t          j        dt          j                                      d          t          j        t          j        |t          j        |t          j        dt          j                                      d                    |          }t          j        t          j        ||          t          j        ||                    }|5t          j        ||t          j        dt          j                            }t          j        |t          j	        ||          |          }|J |S )Nr   r#   Fr6  r  )r?   r  r   rL   r  r;  r  r<  r   rv   r  r  rM  r   r  r  r9  )r  r  r  r!  rD  rE  rF  rG  rH  rI  rJ  rK  rL  r>  partr'  r  r  r  rP  rQ  r  r  r  rS  rT  r  rU  s                   rT   rH   zavg_pool2d_backward.<locals>.fn  s   A
N
N.QQ'&)3VAY??
 
 .QQ'&)3VAY??
 
 x6!95595;GGx6!95595;GG+gs|Au{'C'CDD+gs|Au{'C'CDDE3>-#M#MNNE3>,#L#LMM'' *	R *	RC]++ )R )RWWcl3&D&DEEWWcl3&D&DEE#/,EE& Fk F'N[^;EE==b"EEE{K#1 #$&s|Au{7S7S(T(T!" !" !.&+    1 #$&s|Au{7S7S(T(T!" !" !-&+   & ) . xF2u%%F2u%%  #"ytS\#u}5U5UVVHH"yswx/F/FQQHHS)RT ###rV   r'  )rW   r3   r   r   r  r  r  rs   r   r  r   fallback_avg_pool2d_backwardr0   r   r   )rY  rn   r  r  r  r  r  r  r  r"  r#  r$  r%  r   r  r   r  rH   r  r  rP  rQ  r  r^  rS  rT  rU  rV  s     ``` ``           @@@@@@@@@rT   avg_pool2d_backwardr  ;  s8    #'71'<'<'<>X'<'<'<  a&k9-----a#####{q    v;;!w<<1qzz||&&&&JJLLUFE$VQVWiXXE:$UA{FGYWWE:))++K!*F
FjFJK&1&:&:&<&<#Q|AJJLL!!HKKMME     {1~)**    M      {1~)**    M
  -/KR+	
 	
 		
        8? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?B 
	%%''	
 
 
B IrV   c                 j    dk    s
J d            ssg dt          | t                    sJ t          |t                    sJ t                    dk    sJ t                    dk    sJ t                    dk    sJ t          |                                          dv sJ |                                  |                                ^ } t          d|          \  }	}
t          d|          \  }}t           d|          \  }}|                                 t                    p|
p|p||                                 ^ }t          |                                          }|	                                }fdt          d          D             \  z  z  }|d	k    rt          | ||          S  fd
fd}t          j        |                                 |||          }|S )Nr   r  )r   r   r   r}   )r~   r   r#   r!   c              3      K   | ]7t          fd t                   dz            D                       V  8dS )c           
   3      K   | ]@}t          |         z  t          d |         z
           z            z
  d          V  AdS r:  r;  )rm   r  r   r  r  s     rT   rr   z0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>  sj       
 
 VAYQ[^);q	(I!J!JJANN
 
 
 
 
 
rV   r!   N)r  r   )rm   r   r  r  s    @rT   rr   z&avg_pool3d_backward.<locals>.<genexpr>  s       3 3
 	 	 
 
 
 
 
 
;q>A-..
 
 
 	
 	
3 3 3 3 3 3rV   }   c           	      @   d D             \  }}}d D             \  }}}d D             \  }	}
}d t          | ||g|||g|||g          D             \  }}}d t          |||g|	|
|gg|||g          D             \  }}}d |||fD             \  }}}d t          |||gg          D             \  }}}t          j        t          j        t          j        ||          t          j        ||                    t          j        ||                    }|S )Nc              3   T   K   | ]#}t          j        |t          j                  V  $d S r]   r1  r  s     rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>$  s0      'U'UQ(D(D'U'U'U'U'U'UrV   c              3   T   K   | ]#}t          j        |t          j                  V  $d S r]   r1  r  s     rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>%  s0      MMs|Au{;;MMMMMMrV   c              3   T   K   | ]#}t          j        |t          j                  V  $d S r]   r1  )rm   r  s     rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>&  sA       (
 (
-.CLEK(((
 (
 (
 (
 (
 (
rV   c              3   n   K   | ]0\  }}}t          j        t          j        ||          |          V  1d S r]   )r?   rM  r*  )rm   r)  r  pads       rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>*  sR       "
 "
1c GCGAqMM3''"
 "
 "
 "
 "
 "
rV   c           
   3      K   | ]a\  }}}}t          j        t          j        ||          t          j        t          j        |t          j                  |                    V  bd S r]   )r?   r<  rv   r  rL   r  )rm   r  r  r   r  s        rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>0  sw       

 

 #q#s Kq!!373>#u{+K+KS#Q#Q 

 

 

 

 

 

rV   c              3   z   K   | ]6}t          j        |t          j        d t          j                            V  7dS r  r?   r;  r  rL   r  )rm   r  s     rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>;  sP       "
 "
 Ks|Au{;;<<"
 "
 "
 "
 "
 "
rV   c              3      K   | ]9\  }}t          j        |t          j        |t          j                            V  :d S r]   r?   r<  r  rL   r  )rm   r  r   s      rT   rr   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>?  sT       
 
S KS^C==>>
 
 
 
 
 
rV   )r   r?   r*  rM  )pdrK  rL  stride_dr  r  pad_dr  r  kernel_dr  r  dstartr  r  dendr  r  r  depthr^  r  r  r  rV  s                      rT   r  z>avg_pool3d_backward.<locals>.compute_pool_size_without_padding#  s   'U'Uf'U'U'U$(HMMWMMMue(
 (
2=(
 (
 (
$(H"
 "
 Rx8<ueU>S "
 "
 "


 

 '*(8X.&u%	' '	

 

 

dD"
 "
 &&1"
 "
 "

 
tT 2UFE4JKK
 
 
dD GCGD&))374+@+@AA374QWCXCX
 
 rV   c                 X   | ^ }}}}d t          |||g          D             \  }}}d t          |||g!          D             \  }}}d t          |||g!          D             \  }}	}
d |||fD             \  }}}d t          ||	|
g g          D             \  }}	}
d }t                    D ]o}t                    D ][}t          "          D ]G}d t          |||g|||g          D             \  }}}}n,ssd         d         z  d	         z  }n |||          }t          j         g |t          j        t          j        |t          j        |t          j        dt          j	                                      d
          t          j        t          j        |t          j        |	t          j        dt          j	                                      d
          t          j        t          j        |t          j        |
t          j        dt          j	                                       d
                    |          }t          j
        t          j
        t          j        ||          t          j        ||	                    t          j        ||
                    }|5t          j        ||t          j        dt          j                            }t          j        |t          j        ||          |          }I]q|J |S )Nc              3   &   K   | ]\  }}||z   V  d S r]   rj   )rm   r  r  s      rT   rr   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>J  s*      AAvq#1s7AAAAAArV   c              3      K   | ];\  }}}t          j        t          ||z
  |z   |          t          j                  V  <d S r]   r?   r  r   rL   r  )rm   r  r  r  s       rT   rr   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>L  s\       %
 %
1a N8AEAIq115;??%
 %
 %
 %
 %
 %
rV   c              3   |   K   | ]7\  }}t          j        t          ||          d z   t          j                  V  8dS r  r  )rm   r  r  s      rT   rr   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>Q  sU       
 
1 N8Aq>>A-u{;;
 
 
 
 
 
rV   c              3   z   K   | ]6}t          j        |t          j        d t          j                            V  7dS r  r  )rm   pstarts     rT   rr   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>V  sP       %
 %
 KQ < <==%
 %
 %
 %
 %
 %
rV   c              3      K   | ]9\  }}t          j        |t          j        |t          j                            V  :d S r]   r  )rm   pend
pooled_dims      rT   rr   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>Z  sT       
 
 j KcnZEEFF
 
 
 
 
 
rV   c              3      K   | ]9\  }}t          j        |t          j        |t          j                            V  :d S r]   )r?   rv   r  rL   r  )rm   r  p_s      rT   rr   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>f  sT       " "&FB R(E(EFF" " " " " "rV   r   r#   r!   Fr6  r  )r   r   r?   r  r  r<  rM  r  rL   r  r   r  r  r9  rv   )#r  r  r  r  r!  pdstartrD  rE  pdendrF  rG  rH  pd_rI  rJ  r  rK  rL  r>  r  r'  r  r  d_window_sizer  rP  rQ  r  r  r  pooled_depthrS  rT  r  rU  s#                        rT   rH   zavg_pool3d_backward.<locals>.fnH  s   AqAAaAY)@)@AAA1a%
 %
1ay+v>>%
 %
 %
!'

 
Q1Iv..
 
 
ue
%
 %
"GW5%
 %
 %
!'
 
$'u%m\'R% %
 
 
ue '' 8	V 8	VC]++ 7V 7V // 6V 6VC" "*-$gw7#sC+ +" " "JBB (3 0* N+ N +AQ ?+a. P A A"b" M M;#!' # 5$'K(*CGE3<5;;W;W,X,X%& %& %1*/!" !" !" !$ 5$'K(*CGE3<5;;W;W,X,X%& %& %2*/!" !" !"  !$ 5$'K(*CGE3<5;;W;W,X,X%& %& %1*/!" !" !"! 4 7 D< 8E!2!2CF2u4E4EFFr5)) D  '#&9 $S%-(H(H$ $ $'9T378T3J3JH#U#Um6V7Vp ###rV   r'  )rW   r3   r   r   r  r  r  r   rs   r   r   fallback_avg_pool3d_backwardr0   r   r   )!rY  rn   r  r  r  r  r  r  r  d_outceil_mode_dr"  ceil_mode_hr$  ceil_mode_wr   r  r   r  rH   r  r  r  r  rP  rQ  r  r^  r  rS  rT  rU  rV  s!     ``` ``             @@@@@@@@@@@@rT   avg_pool3d_backwardr"    s+    #'71'<'<'<>X'<'<'<  ))k9-----a#####{q    v;;!w<<1qzz||&&&&#$::<< UE65%eQVWiXXE;%; E; &eQVWiXXE;))++Kg,,K+KKK4?4H4H4J4J1Qm\AJJLL!!HKKMME3 3 3 3 3
 q3 3 3/M=-  -/-?KS+	
 	
 		
# # # # # # # # # #JU U U U U U U U U U U U U U U U U Un 
	%%''	
 
 
B IrV   c                    |                                  }t          |t                    r|g}n|st          t	          |                    }t	          |          dk    r t          |          dv sJ d|             g S t          |          }t          t	          |                    D ]}||         dk     r.||xx         t	          |          rt	          |          ndz  cc<   d||         cxk    rt	          |          k     s#n t	          |          dk    r||         dk    sJ t	          t          |                    t	          |          k    s
J d            |S )Nr   )rj   r]  r  zinvalid axis: r#   zreduction axis not unique)r   rW   r   r   r   rt   rs   ru   )rn   r.  r  r   s       rT   _validate_reduction_axisr%    s]   ::<<D$  v  SYY
4yyA~~T{{////1H$1H1H///	::D3t99 M M7Q;;GGGCII4s4yyy14GGGDG''''c$ii''''CIINNtAw!||||s4yy>>SYY&&&(C&&&KrV   c                  
 |t          | |          } |                                 t          t          | |                    }g }g g }g t	          t                              D ]g}||v r1                    |           |                    |                    7                    |           |                    |                    h
fd}r,t                    }	D ]}t          j	        d          |	|<   n|}	| 
                                
t          |                                 |p|                                 |                                 ||	|          S )Nc                     t          |          t                    k    sJ r0t                     t          	          k    sJ  fdD              t                     t                    k    sJ d gt                     t          |          z   z  }t          j        t                     t          |                    D ]
\  }}|||<    |          S )Nc                      g | ]
}|         S rj   rj   )rm   r   r  s     rT   ro   z9_make_reduction_inner.<locals>.loader.<locals>.<listcomp>  s    000!U1X000rV   )r   r   chainr   )
r  reduction_indexr7  r  varinner_loaderkeepdimskept_idxreduced_idxr  s
   `    rT   r;  z%_make_reduction_inner.<locals>.loader  s    ?##s;'7'77777 	1u::T****0000x000E5zzS]]****Fc%jj3+?+??@	!%  #k?"C"C
 
 	! 	!HC !IcNN|I&&&rV   r#   )r(  	dst_dtyper[  r&  r%  reduction_ranges)r   r   ru   r%  r   r   r   rs   r   r   r  r   r   r   )rn   r.  r-  r   r  
kept_sizesreduced_sizesr   r;  r  r,  r.  r/  r  s     `       @@@@rT   _make_reduction_innerr4    s   Q::<<D'40011DJHMK3t99 ' '99q!!!  a))))OOAd1g&&&&' ' ' ' ' ' ' ' '  :: 	+ 	+A-**HQKK	+ ==??L||~~'81;;==++--&   rV   r  c                      dd d fd}|S )NFr   c                    t          | |||          }t          j        d| d|}t          |j        j        t                    r|                                 |S )Nr.  r-  r   r  )r  
input_noderj   )r4  r1   r   rW   r<  rI  )rn   r.  r-  r   r   r  r  r  s         rT   r5  zmake_reduction.<locals>.inner  sy    &"7
 
 
 !XAXXQWXXKi
 
 	 NNrV   r   rj   )r  r  r5  s   `` rT   make_reductionr9    s:    T         LrV   c                *   |t          | |          } |                                 }t          | |          }t          |                                 |                                 f|                                 f|                                 |          S )N)r(  dtypes	inner_fnsr  r.  )r   r   r  r   r   r   r  )rn   r.  r   r  s       rT   _make_scan_innerr=    s    Q::<<DD!!D||~~==??$ZZ\\   rV   r   c                p   |t          | |          } |                                 t          | |          }|                                 }|t          j        t          j        fv rt          | t          j                  } t          | ||          }t          fd|D                       }t          j        ||                                 |                                           }t          j        |t          |                                                    }t          t!          ||          |          S )Nc              3   (   K   | ]}|         V  d S r]   rj   rm   r   r  s     rT   rr   zmean.<locals>.<genexpr>  '      00a$q'000000rV   )r   r   r%  r   rL   r0  r/  r   sum_r=   r&   r,   r   r+   r   rs   div)rn   r.  keepdimr   output_dtype
sum_resultdenomr  s          @rT   r  r    s    Q::<<D#At,,D;;==Lu~666Q$$aw''J0000400000Eq{{}}allnnEEEeT**=*=*?*?%@%@AAEC
E**L999rV   c                   
 |d}|                                  
t          | |          }t          | |d          }|r|                                 t	          t          | |                    }t          |||          }t          
fd|D                       }|rt          j	        ||z
  d          }t          j        ||                                 |                                           }t          j        |t!          |                                                     }t#          ||          }	|s|	fS |r|nt%          ||          }|	|fS )Nr#   T)rD  c              3   (   K   | ]}|         V  d S r]   rj   r@  s     rT   rr   z var_mean_sum_.<locals>.<genexpr>+  rA  rV   r   )r   r%  r  rI  squarerM  rB  r=   r   r  r&   r,   r   r   r+   r   rs   rC  r  )rn   r.  
correctionrD  return_meanx_meandiffsrF  rG  x_varr  s             @rT   var_mean_sum_rP    sF   
::<<D#At,,D!T4(((F 3q&>>""EeT7++J0000400000E 1	%*,a00q{{}}allnnEEEeT**=*=*?*?%@%@AAE
E""E x9VVGFD$9$9F&=rV   c                    t          | |          }t          | ||d d           }|d         }t          |d                   }t          |t          j                  o/t          |          t          j        k     ot          |          dk    S )Nr7  r%  r1  r#   )	r%  r4  r=   rW   r   r   r   r$   unroll_reductions_threshold)rn   r.  rD  r   r%  reduction_numels         rT   use_two_step_variancerT  8  s    #At,,D"	wd$  F HF#F+=$>??O?EM22 	'  6#EE	'&!!Q&rV   c                j   dt          | ||d d           }|                    d          }|                    d           |                    d           t          j        j        d|fd|                                 d|\  }}}	|                                 |                                 |                                 t          | |          }t          fd|D                       d	 fd
}
 t          |
          |          }|r|                                 ||fS |fS )Nr#   r7  r&  r0  r[  welford_reduce)r<  r  r   c              3   (   K   | ]}|         V  d S r]   rj   r@  s     rT   rr   z$var_mean_welford_.<locals>.<genexpr>^  s'      11q47111111rV   c                     t          | t          j                  r9| j        s2t	          j        t	          j        | t          j                  |          S t	          j	        | |          S r]   )
rW   r   r   	is_numberr?   r   r  rL   r  r  r  s     rT   get_constant_or_index_exprz5var_mean_welford_.<locals>.get_constant_or_index_expr`  sS    a$$ 	GQ[ 	G<q%+ > >FFF|Au%%%rV   c                                }           }t          j        d          }| t          j        |||z
            z  S r   )r?   r  r;  )r<  cNzerorK  r   rZ  rnumels       rT   r  z#var_mean_welford_.<locals>.scale_fne  sS    &&z599&&vu55|Au%%ck$A....rV   rj   )r4  rA  r&   WelfordReductionr   r   rI  r   r%  r=   r7  )rn   r.  rK  rD  rL  r   r;  r  m2r   r  r+  r   rZ  r_  r  s     `         @@@@rT   var_mean_welford_rb  H  sz   
"	wd$  F ZZ
##F
JJ{
JJ{%, )'kkmm  	 KD"a JJLLLKKMME::<<D#At,,D1111D11111F& & &
/ / / / / / / / #.
"
"2
&
&C Dy6MrV   c                8   |                                  t                    }t          | |d          } t          | ||||          }t	          | ||          rt          di |nt          di |}t          fd|D                       }|s|d         n|S )NFrd  )rn   r.  rK  rD  rL  )r.  rD  c              3   <   K   | ]}t          |d           V  dS )Frd  Nro  )rm   rn   	out_dtypes     rT   rr   z#var_mean_helper_.<locals>.<genexpr>  s2      FF!8Ayu555FFFFFFrV   r   rj   )r   r   r   r   rT  rP  rb  rt   )	rn   r.  rK  rD  rL  compute_dtyper   r   re  s	           @rT   var_mean_helper_rg  s  s    I))44MM...A
  F !w???	)(((( 
 FFFFvFFFFFF'36!99V3rV   )rK  rD  c                *    t          | |||d          S )NFr.  rK  rD  rL  rg  rn   r.  rK  rD  s       rT   var_rl    s$    	W%   rV   c                *    t          | |||d          S )NTri  rj  rk  s       rT   var_meanrn    s$    	W$   rV   c                 .   |dk     r$t          t          j        |           | |          S |dk    rt          j        d|          S |dk    r| S t          | |dz  |          }t          j        ||          }|dz  dk    rt          j        ||           }|S )Nr   r#   r!   )pow_recursiver?   r:  r  r*  )rn   r   r   r  s       rT   rp  rp    s    1uuS^A..E:::Avv|Au%%%Avv1a1fe,,FWVV$$F	A!||##MrV   c                 ,    t          j        | |          S r]   )r?   powr   r   s     rT   
pow_nativert    s    71a==rV   )r   c                 X    t          t                    r0t                    k    rt           t                              S t          t                    rdk    rt	                     S t          t                    rdk    rt                     S t          d  fD                       }t          |          }t          t                    odcxk     odk     nc p|odk    }|ri                                  fd}t          j
                                                                          |                                           S t           t                    rL dk    rt          d          S  d	k    r0t!                                                    rt#                    S |rZt           t                    rt%                     S t          t                    rt'                     S t)                     S t+                     S )
Nr  r#   c              3   r   K   | ]2}t          |t          j                  |                                V  3d S r]   )rW   r&   r3   r   rl   s     rT   rr   zpow.<locals>.<genexpr>  s;      NN1*Q2M2MNNNNNNNrV   i    r   c                 \    t           |                                                     S r]   )rp  r   )r  r   r   r;  s    rT   rH   zpow.<locals>.fn  s%     a???rV   r'  r!   )rW   r   r   rr  sqrtr^  r  r   r  r0   r   r   r   r   r   r  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorrt  )r   r   r   is_integer_powembed_exponentrH   r;  s   ``    @rT   rr  rr    s0   !U SVV1c!ff~~	Au		 !s((Aww	As		 QQxx NNANNNNNE%e,,N  3'' a"32AF   
	@ 	@ 	@ 	@ 	@ 	@ 	@ <<>>++--::<<	
 
 
 	
 !V 66Q??"66nQ[[]]33677N 4a   	4&q!,,,6"" 	4-a333-a333arV   c                    t          | t                    r| j        }n| }t          |t                    r|j        }t          |t          j                  st          j        |                                 |                                 |	                                | 
                                          j        }t          |t          j                  sJ t          |t          j                  ri|                                sU|                                sAt          |j        t          j                  s"|                                 |j        |_        | S t          j                            |||           | S )Nr'  unsafe_alias)rW   r3   r<  r&   rq  r0   r   r   r   r  r   is_input_bufferis_module_buffer	NopKernelrI  rv  realize_into)changedr  r  changed_datas       rT   r  r    se   '9%% |#y!! hc2=)) .%%''##%%__&&##%%	
 
 

  	 #r}-----,.. 	$$&&	 ((**	 l'66		 	H!..\ /    NrV   c                 >    t          | t          | |                    S r]   )r  r  )rn   r  s     rT   fill_r    s    Q	!Z00111rV   c                     | |u r| S t          ||                                           }t          ||                                           }t	          ||                                           }t          | |          S r]   )r  r   r   r   r   r   r  )rl  r  r  s      rT   r  r  
  sg    
czz

C))
*
*C
3
(
(C
cllnn
%
%CS#rV   c                 ,    t          j        | |          S r]   )r?   floordivrs  s     rT   r  r        <1rV   c                 ,    t          j        | |          S r]   )r?   truncdivrs  s     rT   r  r    r  rV   c                    t          |           ot          |          }t          |           ot          |          }|dk    r;|r
J d            |rt          | |          nt          t	          | |                    S |dk    r;|r
J d            |rt          | |          nt          t	          | |                    S t	          | |          S )Nr  z5floordiv operands can not be boolean at the same timer  z5truncdiv operands can not be boolean at the same time)r   r   r  r  rC  r  r  )r   r   rounding_modeboth_integerboth_booleans        rT   div_moder    s    "1%%</!*<*<L"1%%</!*<*<L XX!XXXX!-Cx1~~~5Q3C3CCXX!XXXX!-Cx1~~~5Q3C3CCq!99rV   c                     t          |           ot          |          }|rt          | |          S t          t          j        j                  } t          |          | |          S r]   )r   logical_andr.   ru  r*  r  r7  )r   r   	both_boolrH   s       rT   r*  r*  /  s`    ""9q'9'9I (1a   *++!~b!!!Q'''rV   c                 L   t          | t          j                  rt          | j                  S t          | t          j                  r!t          |                                           S t          | t          j                  r| S t          | t          j                  sdS t          j
        j                            |                                           }t          j        |          5  t!          j        t          j        dd          5   | j        |                                  }ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   t          |t          j
        j        j                  sJ t          |j        t          j                  r|j        S dS )z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)rW   r&   
MutableBoxget_constant_valuer<  r   r  r   LoopsrL   	_inductorops_handlerExtractConstantsHandlerr   r@   set_ops_handlerr   objectr   r&  inner_fn_argsvirtualizedOpsValuer   )rn   r  r   s      rT   r  r  9  s   
 !R]## *!!&)))!R[!! 3!!--//222!R[!!  a"" to)AA!,,..QQG	
	7	#	# - -U\
+T& & - - aj!//++,- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 c5?6?@@@@@#)R[)) y4s6   ,!ED6*E6D:	:E=D:	>EEEc                 P   t          d | |fD                       }|rt          | |          S t          |          x}M|j        dk    r(t	          j        t          d          |j                  }n
d|j        z  }t          | |          S d } t          |          | |          S )Nc              3   R   K   | ]"}t          |          pt          |          V  #d S r]   )r   r   rl   s     rT   rr   zdiv_prim.<locals>.<genexpr>Z  s7      OO1oa((>OA,>,>OOOOOOrV   r   infrR  c                      t          j        |  S r]   )r?   r  r  s    rT   rH   zdiv_prim.<locals>.fng  s    {D!!rV   )	r   r  r  r   mathcopysignr   r*  r7  )r   r   is_integraldivisorr:  rH   s         rT   div_primr  X  s    OOAOOOOOK 1~~%a(((5=AuU||W]CCJJw},J1j!!!" " " >"a###rV   c                 b    t          | |ft          j                  \  } }t          | |          S r   )r
  r   INT_TO_FLOATr  rs  s     rT   rC  rC  m  s9     	
A$C$P  DAq Aq>>rV   c                     t          |           pt          |           }|rd }nd } t          |          | |          S )Nc                 ,    t          j        | |          S r]   )r?   modrs  s     rT   rH   zfmod.<locals>.fn  s    71a== rV   c                 ,    t          j        | |          S r]   )r?   fmodrs  s     rT   rH   zfmod.<locals>.fn  s    8Aq>>!rV   )r   r   r7  )r   r   r  rH   s       rT   r  r  y  sc    !!$$:(:(:K "	! 	! 	! 	!
	" 	" 	" >"a###rV   c                     |                                  }t          |          st          |          r!t          | t	          j                              } d } t          |          |           S )Nc                 *    t          j        |           S r]   )r?   rsqrtr   s    rT   _rsqrtzrsqrt.<locals>._rsqrt  s    y||rV   )r   r   r   r   rL   r  r7  )rn   r   r  s      rT   r  r    sp    KKMME 3"25"9"9 3Q/1122   ">&!!!$$$rV   c                    t          |                                           s!t          |                                           r|t          j        }t          d|          } || |||          S )Nr  r]  r   r   r   r   rL   r  r9  rn   r.  r-  r   rH   s        rT   rB  rB    sk     	''+;AKKMM+J+J
-	U	;	;	;B2axu----rV   c                    t          |                                           s!t          |                                           r|t          j        }t          |                                           dk    r.|dv sJ |p|                                 }t          | |d          S d }t          | ||          }t          j
        j        d	i |d|i\  }|t          | ||          S |S )
Nr   r   r  Trd  c                 >    | \  }|\  }t          j        ||          fS r]   )r?   rv   a_tupleb_tupler   r   s       rT   
combine_fnzcumsum.<locals>.combine_fn  #    1rV   r.  r   r  r   r   rj   )r   r   r   rL   r  r   r   r   r=  r&   Scanr   fallback_cumsumrn   r.  r   r  r   r  s         rT   cumsumr    s     	''+;AKKMM+J+J
-
1::<<Aw&5t,,,,     
 ad%888F????J???IV~qd%8888MrV   c                    t          |                                           s!t          |                                           r|t          j        }t          |                                           dk    r.|dv sJ |p|                                 }t          | |d          S d }t          | ||          }t          j
        j        d	i |d|i\  }|t          | ||          S |S )
Nr   r  Trd  c                 >    | \  }|\  }t          j        ||          fS r]   )r?   r*  r  s       rT   r  zcumprod.<locals>.combine_fn  r  rV   r  r  r  rj   )r   r   r   rL   r  r   r   r   r=  r&   r  r   fallback_cumprodr  s         rT   cumprodr    s     	''+;AKKMM+J+J
-
1::<<Aw&5t,,,,     
 ad%888F????J???IV~t59999MrV   c                 *   d }|                                  }t          |                                           dk    r|dv sJ t          |           S t	          | ||          }t          j        j        di |d|i\  }|t          | |          S |S )Nc           	          | \  }|\  }t          j        ||          }t          j        ||          }||k    t          j        |           z  }t          j        |t          j        t          j        ||z
                      |z   |          fS r]   )r?   r<  r;  r  r  log1pexp)r  r  r   r   min_vmax_vr'  s          rT   log_add_exp_helperz(logcumsumexp.<locals>.log_add_exp_helper  s    Aq!!Aq!!CIe$4$4#45	$	#'%%-*@*@ A AE I1MMOOrV   r   r  r  r  r  rj   )	r   r   r   r^  r=  r&   r  r   fallback_logcumsumexp)rn   r   r  r   r   r  s         rT   logcumsumexpr    s    P P P KKMME
1::<<Ag~~~~Qxxac777FGGGG4FGGGIV~$QC0000MrV   c                 v   t          |                                           dk    r0|dv sJ t          |           t          | t          j                  fS |                                 }t          j        d|d          }|t          j	        u rdn8|j
        rt	          j        |          j        nt	          j        |          j        }t          | ||          }|t          j        f|d<   |                                 d	 f|d
<   t          j        j        di |d|i\  }}|t%          | |          S ||fS )Nr   r  r   argmaxFr   arg_break_ties_leftr  r;  c                     dS Nrindexrj   r4  s    rT   r  zcummax.<locals>.<lambda>      h rV   r<  r  r  rj   )r   r   r^  rf  rL   r  r   r&   get_reduction_combine_fnr   rp  rq  r  rs  r=  r  r  r   fallback_cummax)rn   r.  r   r  r&  r   r   r   s           rT   cummaxr    sI   
1::<<AwQxxAU[99999KKMME,5  J EJ 	 &(EK""U##'  ad%888Fu{+F8==??,>,>?F;gnEEvEE*EEEOFG~qd++++7?rV   c                 v   t          |                                           dk    r0|dv sJ t          |           t          | t          j                  fS |                                 }t          j        d|d          }|t          j	        u rdn8|j
        rt	          j        |          j        nt	          j        |          j        }t          | ||          }|t          j        f|d	<   |                                 d
 f|d<   t          j        j        di |d|i\  }}|t%          | |          S ||fS )Nr   r  r   argminFr  Tr  r;  c                     dS r  rj   r4  s    rT   r  zcummin.<locals>.<lambda>#  r  rV   r<  r  r  rj   )r   r   r^  rf  rL   r  r   r&   r  r   rp  rq  r  rs  r=  r  r  r   fallback_cummin)rn   r.  r   r  	max_valuer   r   r   s           rT   cumminr    sI   
1::<<AwQxxAU[99999KKMME,5  J EJ 	 &(EK""U##'  ad%888Fu{+F8==??,>,>?F;gnEEvEE*EEEOFG~qd++++7?rV   c                    t          |                                           s!t          |                                           r|t          j        }t          d|          } || |||          S )Nr  r]  r   r  r  s        rT   r  r  *  sk     	''+;AKKMM+J+J
-	e	<	<	<B2axu----rV   c                 l    t          | t          j                  }  t          d          | ||          S )Nr   r.  r-  )r   rL   r   r9  rn   r   rD  s      rT   
reduce_anyr  5  s2    EJA >%  w????rV   c                 r    |$t          | ||          t          | ||          fS t          | d |          S Nr  )reduce_amaxreduce_argmaxr  s      rT   
reduce_maxr  ;  L    
g666!#888
 	

 qtg6666rV   c                 r    |$t          | ||          t          | ||          fS t          | d |          S r  )reduce_aminreduce_argminr  s      rT   
reduce_minr  F  r  rV   xor_sumr  r  r  r]  r  
logical_or)r2  r$  stabler   
descendingc          	         |d}|                                  }|                                 }t          t          |          |          }t          |          dk    r+t	          |           t          d|t          j        |          fS t          |          r||         nd}t          j	        j
                            |t          j        t          j                  j                  st          | |||          S t!          |ddt          j        |d          }dgt          |          z  }t          |          r|||<   t#          ||          }t%          ||          }t&          j                            || j        |j        f|                                 |                                f||||          \  }	}|	t          | |||          S |J |	t1          |t          j                  fS )NFr   r#   r  )r  r  r   r(  r  )r(  r;  r<  r  r.  r  r  )r   r   r   r   r^  r  rL   r  r@   r   r   statically_known_ltrs  int16r  sort_fallbackr  rv  r   r&   Sortr   r   r  r   )
rn   r  r   r  r  r(  r  r   
view_shaper   s
             rT   sort_stabler  b  s   ~JJLLE\\^^F
3u::s
+
+C
5zzQQxxq&%+u==== ZZ.uSzzQH7//%+ek:R:R:VWW OQv3:NNNNVSX  G s5zz!J
5zz #"
37J''GWe$$Ggnn'==??G$7$7$9$9: %  OFG ~Qv3:NNNN8GU[1111rV   c                 (    t          | d||          S )NFr  )r  )rn   r   r  s      rT   sortr    s    qCJGGGGrV   c                 <    t          | |t          j        |          S )Nr   r   r4  r  r   r  )r   r   r4  s      rT   register_pointwise_numericr	    s(    
;H'	   rV   c                 :    t          | t          j        d          S )NT)r   r  r  r  s    rT    register_pointwise_numeric_ldf64r    s%    
;H"   rV   r  logical_not)r$  )r   r   r  identity)rG  pointwise_overrides_datac              #   H  K   t           |         t          | j        d           }|d S fd}t          |t          j        j                  r>|                                D ]'}t          ||          }|j         ||          fV  (d S |j         ||          fV  d S )Nc                 4    j         t          |           S d S r]   )tritonrh  )r   r<  s    rT   make_triton_fallbackz6_get_pointwise_overrides.<locals>.make_triton_fallback  s!    ;#B''' rV   )	r  r_   r   rW   rL   rX   rw   ry   r   )nsr   r   r  olnamer  r<  s         @rT   _get_pointwise_overridesr    s      #D)D	TY	%	%B	z( ( ( ( ( "ej122 Ellnn 	I 	IFV$$Bd.0D0DR0H0HHHHHH	I 	I $*,@,@,D,DDDDDDDrV   r  c                 z    | t           |<   t                              |            fd}t          | |           d S )Nc                       | i |}g }t          | d         |          D ]*\  }}|                    t          ||d                     +|S )Nr   Tr  )r   r   r  )r   r   resultsmut_resultsr   r  outplace_ops         rT   rH   z$register_foreach_inplace.<locals>.fnL  sk    +t.v..tAw00 	J 	JKCyf4HHHIIIIrV   )rG   rF   rv   r   )aten_opoutplace_aten_opr  rH   s     ` rT   register_foreach_inplacer  H  sT    07,-G$$$     w+++++rV   c                 @    t          | d           fd            }|S )Nr`  c                       | i |}t          || d                                                   }t          | d         |          S r   )r   r   r  )r   r   r  r  s      rT   rH   zregister_inplace.<locals>.fnl  sJ    d-f--&$q'"3"3"5"566a&)))rV   )r   )r  r  rH   s    ` rT   register_inplacer   k  s;    wD999* * * * :9*
 IrV   c                     d S r]   rj   r  s      rT   sym_constrain_ranger"        4rV   c                     t           j        j        j        d         }t	          |t
          j                  sJ |j        j        S ry  	r@   r   r  r-  rW   rL   r  rM  r  r   r   r  s      rT   sym_sizer'    s8    
'

#E
*C c5<(((((8=rV   c                     t           j        j        j        d         }t	          |t
          j                  sJ |j        j        S ry  r%  r&  s      rT   
sym_strider)    s6    
'

#E
*Cc5<(((((8=rV   c                 *    |                                  S r]   )r  )r   s    rT   	sym_numelr+    s    ;;==rV   c                      t          d          )NzHelpful for debuggingrc   )r  r   r   s      rT   foobarr-    s    
5
6
66rV   c                 H    |                                   t          |           S r]   )rI  r^  r   s    rT   _realizer/    s    IIKKK88OrV   c                 X    |                                   t          j        | |           | S r]   )rI  r&   ResizeStorageBytes)variabler  s     rT   resize_storage_bytes_r3    s,    (H---OrV   c                     |                                   |                                  t          j        t          j        | |                    S r]   )rI  r3   r   r&   SetSourceTensorKernelr  source_tensors     rT   set__source_tensorr8    s?    LLNNNB4T=IIJJJrV   set_c                     |                                   |                                  t          j        | |           d S r]   )rI  r&   r5  r6  s     rT   	fsdp_set_r;    s:    
 }55555rV   c                  	
 t          | t                    sJ t          |t          t          f          sJ |t          j        }|t          j        k    rt          d|           |t          j        k    rt          |          dk    sJ |t          j
        k    rt          |          dk    sJ |                                 
|                                 }|                                 }t          | j        t          j                  r| j                                        | _        t	          j                    rat          j        j        j        rKt-          |          rt/          d          n.t1          |          rt	          j        |          j        ndndt6          j        j                            
d          rt?          |||          S tA          | 
gd	g          }|!                                	t          j"        #                    ||          }t          j$        ||||          %                                	
fd
}tM          j'        |||t          |                    }|S )Nzunsupported memory format: r~   r   nanTr  r   r  r#   c                      |           t          j        t          j                  }t          j        t          j                  }t          j        ||          }t          j        |fd          S )Nc                        g          S r]   rj   )
flat_indexflat_loaders   rT   r  z*resize.<locals>.inner_fn.<locals>.<lambda>  s    ZL(A(A rV   )r?   r  rL   r  r  r!  )	r  flat_index_exprlimitr'  r@  rA  	old_numelout_indexeruninitalized_vals	       @rT   r&  zresize.<locals>.inner_fn  sl     [%%
.U[AAy%+66vou--z$ A A A A ACSTTTrV   r'  )(rW   r3   rs   rt   rL   contiguous_formatpreserve_formatr  channels_lastr   channels_last_3dr  r   r   r<  r&   r   r  rd  utilsrg  fill_uninitialized_memoryr   r   r   rs  r  r@   r   r   statically_known_equalsr  r  r  r    stride_ordered_for_memory_formatr  r"  r0   r   )rn   r  r  r   r(  x_flat
out_strider&  r   rA  rD  rE  rF  s            @@@@rT   resizerQ    s}   a#####dT5M*****/---HHHIII+++4yyA~~~~...4yyA~~~~IKKMME\\^^F!&"+&& &##%% 	244K%? %   	$$U||e$$ 	$${5115# w//	1== HD*%GGGG		
 	
 F $$&&K"CCD-XXJ.jAANNPPKU U U U U U U U 
UXd4jj  C JrV   )auto_functionalizedc                     ddl m} |                    |          }t          j        | |i ||           d |                                D             S )Nr   )kernel_side_table)
kernel_idxgridkernel_argsc                 D    i | ]\  }}t          |t                    ||S rj   r   )rm   r  r  s      rT   r  z'triton_kernel_wrap_.<locals>.<dictcomp>/  s-    RRRcz#y7Q7QRCRRRrV   )*torch._higher_order_ops.triton_kernel_wraprT  get_constant_argsr&   UserDefinedTritonKernelrE  )rU  constant_args_idxrV  r   rT  constant_argss         rT   triton_kernel_wrap_r^  %  st    LLLLLL%778IJJM/v//   
 SRV\\^^RRRRrV   c                    t          |           s"t          t          t           |                    rFd}t          j        j        j                            dd           x}r| d| }|t          j        _        t          j
                            | |||          }t          t          t          j        |                    S )Nz"control flow operator: torch.cond.rb  rc  )r-   r   r)  r@   r   r  r-  r.  re  r&   Conditionalr   rs   r3   )predtrue_fnfalse_fnoperandsrf   rb  r  s          rT   re   re   2  s     0#c)X6677 02'.377tLLL; 	988;88C,/)^""4(HEEFI$f--...rV   c                 j   t          t          t          ||z                       rFd}t          j        j        j                            dd           x}r| d| }|t          j        _        t          j
                            | |||          }t          t          t          j        |                    S )Nz(control flow operator: torch.while_loop.rb  rc  )r   r)  r-   r@   r   r  r-  r.  re  r&   	WhileLoopr   rs   r3   )cond_fnbody_fncarried_inputsadditional_inputsrf   rb  r  s          rT   
while_looprk  >  s    
3y.+<<==>> 08'.377tLLL; 	988;88C,/)\  '>CTUUFI$f--...rV   r  c                 ~  	 ddl mm} fdt          j        ||          D             } || |          		fd}t          |d         |d           }t          d |D                       |d<   t          d	 |D                       |d
<   t          j        j	        d|dd|}|d         t          d          |S )Nr#   )InputDescriptorlower_pointwise_subgraphc                 r    g | ]3} |                                 |                                           4S )r  )r   r   )rm   rn   rm  s     rT   ro   z$associative_scan.<locals>.<listcomp>N  sH        	akkmmALLNNCCC  rV   c                 `     g t          j        |           t          j        |          R  S r]   )r  r  )lhsrhslowered_combine_fns     rT   wrapped_combine_fnz,associative_scan.<locals>.wrapped_combine_fnT  sC    !! 
$$
$$
 
 
 	
rV   r   r  c              3   >   K   | ]}|                                 V  d S r]   r,  rl   s     rT   rr   z#associative_scan.<locals>.<genexpr>[  s*      ::qQ[[]]::::::rV   r;  c              3   >   K   | ]}|                                 V  d S r]   r  rl   s     rT   rr   z#associative_scan.<locals>.<genexpr>\  s*      ??A??????rV   r<  F)r  can_fallback_to_atenz/Unable to generate code for associative_scan oprj   )subgraph_loweringrm  rn  r   r)  r=  rt   r&   r  r   r  )
r  rP  r   rn  subgraph_inputsrt  r   r  rm  rs  s
           @@rT   associative_scanrz  J  s'   LLLLLLLL   ..  O 21*oNN
 
 
 
 
 eAhS===F::E:::::F8???????F;W^ %"   F
 ayLMMMMrV   c                     d S r]   rj   )tokenss    rT   _sink_tokensr}  g  r#  rV   c                 <   t          j        j        |g|R i |}ddlm}  ||||          }|J t
          j        j        |         }||fS t          j	        t           j
        t          j        |          }t          |t          t          f          s||fS |g|R S )Nr   )get_effect_key)r&   EffectfulKernelr   torch._higher_order_ops.effectsr  r@   r   effectful_opsr  rp  MultiOutputr3   rW   rs   rt   )tokenr   r   r   r  r  effect_typeeffectful_kernels           rT   with_effectsr  l  s    &r;D;;;F;;F>>>>>> .T622K"""w,[9~ ""!".)2BFKKFftUm,, + &)) *6***rV   c                 2   t          |           } t          j        rJ|                                  t          j        j                            |                                            t          j
                            t          j        j        | ||           | S r]   )r^  r$    reorder_for_compute_comm_overlaprI  r@   r   no_fuse_buffer_namesrv   rW  r&   _CollectiveKernelcreate_inplacer   all_reduce_rk  r   	reduce_op
group_names      rT   _all_reducer    sy    Cjj2 	= KKMMMG(,,S\\^^<<<
++(0#y*	
 	
 	
 
rV   c                 h    t           j                            t          j        j        | ||           | S r]   )r&   r  r  r   r  rk  r  s      rT   _all_reduce_r    s3    
++(0#y*	
 	
 	
 
rV   c                     d | D             } t           j                            t          j        j        | ||           | S )Nc                 ,    g | ]}t          |          S rj   )r^  r  s     rT   ro   z)_all_reduce_coalesced.<locals>.<listcomp>  s    ///%**///rV   r&   r  r  r   all_reduce_coalesced_rk  r  r  r  s      rT   _all_reduce_coalescedr    sI    /////
++2:		
 	
 	
 rV   c                 h    t           j                            t          j        j        | ||           | S r]   r  r  s      rT   _all_reduce_coalesced_r    s6    
++2:		
 	
 	
 rV   c                     t           j                            t           j                            t
          j        j        | ||                    S r]   )r&   r3   r   r  create_out_of_placer   all_gather_into_tensorrk  )r   
group_sizer  s      rT   _all_gather_into_tensorr    sD    |"" 44 7?	 
 
 	
rV   c           	          t          j        t          j        j        t          j                            t          j        j	        | ||                    S r]   )
r  r  r&   r3   r   r  r  r    all_gather_into_tensor_coalescedrk  )r  r  r  s      rT   !_all_gather_into_tensor_coalescedr    sF    L 44 AI	 
 
 	
rV   c                l    t           j                            t          j        j        | |||           |S )N)r   )r&   r  r  r   all_gather_into_tensor_outrk  )r   r  r  r   s       rT   _all_gather_into_tensor_outr    s>    
++7? 	, 	
 	
 	
 
rV   c           	          t           j                            t           j                            t
          j        j        | |||                    S r]   )r&   r3   r   r  r  r   reduce_scatter_tensorrk  )r   r  r  r  s       rT   _reduce_scatter_tensorr    sG    |"" 44 6> 
 
 	
rV   c           
          t          j        t          j        j        t          j                            t          j        j	        | |||                    S r]   )
r  r  r&   r3   r   r  r  r   reduce_scatter_tensor_coalescedrk  )r  r  r  r  s       rT    _reduce_scatter_tensor_coalescedr    sI    L 44 @H 	
 	
 		
rV   c           	          t           j                            t           j                            t
          j        j        | |||                    S r]   )r&   r3   r   r  r  r   all_to_all_singlerk  )r   output_split_sizesinput_split_sizesr  s       rT   _all_to_all_singler    sG    |"" 44 2:"! 
 
 	
rV   c                     t          |           } t          j                            t          j        j        | ||           | S r]   )r^  r&   r  r  r   
broadcast_rk  r   r  r  s      rT   
_broadcastr    s>    Cjj
++'/c:	
 	
 	
 
rV   c                 h    t           j                            t          j        j        | ||           | S r]   )r&   r  r  r   r  rk  r  s      rT   _broadcast_r     s3    
++'/c:	
 	
 	
 
rV   c                 d    t           j                            t          j        j        |            | S r]   )r&   _WaitKernelcreate_waitr   wait_tensorrk  )r   s    rT   _wait_tensorr    s&    
""#3#?#GMMM
rV   c           	          t           j                            t           j                            t
          j        j        j        j	        | |||                    S r]   )
r&   r3   r   r  r  rL   r?   _dtensorshard_dim_alltoallrk  )r   
gather_dim	shard_dimr  s       rT   _shard_dim_alltoallr    sL    |"" 44	"5= 
 
 	
rV   zRInductor support for distributed collectives depends on building torch.distributed)r  )quantized_lowerings)mkldnn_lowerings)jagged_lowerings)NN)NNNNFN)Fr]   )r   r   r  r#   Tr]  )r   r   r#   )r   Tr$  )Tr   )r   NNr#   )r  FF)r!   F)NNN)NrR  N)r  )NNNN)rj   r   FTNr   )r  F(  r   r   loggingr  r  r  r  collectionsr   typingr   r   r   r   r   r	   r
   r   unittest.mockr   r   rL   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreerK  _pytreer  (torch._higher_order_ops.associative_scanr   rY  r   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r   torch.utils._sympy.functionsr   r   r   r   r    _dynamo.utilsr"    r$   r%   r&   r'   decompositionr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r  r?   r@   	getLoggerr  r  rA   rX   rY   __annotations__rB   ru   rC   ru  tr_c10dri  rD   rE   rF   rG   quantized_decomposedr[   rZ   rg   rk   r|   r  r  r  r  bmmconvolutionconvolution_backwardr  r_  mmr  r  _int_mmrk  rj  r  r  r  r0  r9  r  	complex32	complex64r   r/  r   r   r   r   r   r   r   r   r   r   r   r  r   r   r
  r7  rX  r   r   rj  rm  rz  rv  r}  r(  r  
device_putr  r  r  r  r  r   aliasdetachdetach_liftview_ofr  r   r  r  r  r  r  r  r  r  r  rk  r  r   r  r  r  _unsafe_viewreshaper  slicer  r  r+  quantize_per_channelrH  dequantize_per_channelrO  quantize_per_tensorr   rU  dequantize_per_tensorrX  r  rc  rf  r  r  r  r  r  r  r  r  r  r  r  r  r  rh  	lru_cacher  r.  r  r  r  r  r  r  r  rngprimsr&  r(  r-  	bernoullir)  r1  r5  r9  r<  r  rB  r?  r@  rG  rF  rE  randintforce_stride_orderrN  r#  rP  rX  rS  lookup_seedrY  randomr"  r^  rd  rk  rs  ru  r  rP   FALLBACK_ALLOW_LISTr  _adaptive_avg_pool3dadaptive_max_pool3dfractional_max_pool3dmax_pool3d_with_indicesuniformexponential_pdist_forwardsoft_margin_loss_backwardsearchsorted_cdist_forward_cdist_backwardmax_unpool2dmax_unpool3d
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation
_cudnn_rnn_cudnn_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backwardr  r  kthvaluetopkr[  median	nanmedianrandpermresize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_statemasked_scattermasked_scatter_backwardr  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   r  '_scaled_dot_product_efficient_attention0_scaled_dot_product_efficient_attention_backward#_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducer_  r^  r  r  r  r  r  r  scalar_tensorr  
LongTensorr  r  r  r  r  r  r  r  r  rf  r  
zeros_liker  r  r  r	  r  r  r  r'  r2  rD  rG  rF  r  rV  r[  r^  r`  rh  rX  rm  r]  r  fallback__unsafe_masked_indexr  ,fallback__unsafe_masked_index_put_accumulater  r  ry  r  r  r  r  r  r  r  r  r  r  r  r8  r  r  r   r  r  r  r  r  r  r  r(  r6  rX  rk  rr  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r  r%  r4  r9  r=  r  rP  rT  rb  rg  r+  rl  rn  rp  rt  rr  Tensor_Tensorr}  Scalarr{  Tensor_Scalarr|  r  r  r  r  r  rC  r  r*  r  r   r  r  true_divider  r  r  r  rB  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  rv   r   r  r	  r  r  rz  expm1relur  ry  rJ  rM  cossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfr  tantanhr  r  r  logical_xorr;  r<  	clamp_min	clamp_maxnegr:  	remaindersignsignbit	_neg_viewler  r  r  r  necoshsinhacosacoshasinasinhatan2atanatanhr  erfcerfinvhypotlog10log2	nextaftercodegen.commonrG  r  r  r   r   r   r4  _foreach_addforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyr  _foreach_add__foreach_mul__foreach_div_r   add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r"  r'  r)  r+  rE  methodfuncr-  _inductor_testrI  r/  inductorr3  r9  source_Tensorr8  fsdpr;  rQ  *torch._higher_order_ops.auto_functionalizerR  r^  higher_orderre   rk  Subgraphrz  r}  r  )torch.distributed._functional_collectivesr   
all_reducer  r  r  all_reduce_coalescedr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  ImportErrorr7  r  r  register_quantized_opsregister_woq_mm_opsr  register_onednn_fusion_opsr  register_jagged_opsrj   rV   rT   <module>r     sN               				  # # # # # # I I I I I I I I I I I I I I I I I I I I         + + + +  $ $ $ $ $ $ $ $ $ H H H H H H U U U U U U                          M L L L L L L L              - , , , , , 8 8 8 8 8 8 8 8 8 8 8 8 = = = = = = = =                           
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
          g!!=?	4
%xS'99: ? ? ?  4	J8HS#X$677    ),	3uz$% - - -y~
)
	47CEE s5:01 9 9 9*-#%%S&' / / /25#%% S./ 7 7 7NP ej3UZ5JJK P P Py5 -#s(!3 -#s(AS8T - - - -6M M MF F F

 
 
3 3 3   !$-&  * {z{{{}}}
 .    " " "# # #3R         " " "J  20 0 0j 7?	   $# # #0) ) ) )\  $!%W W W WtM M M M`E E	 E%+ E E E E 5-4HHH)Y )u{ ) ) ) IH) @E 5 5 5	 5%+ 5 5 5 5( 49?===&9 &U[ & & & >=& ;@ = = = =EL = = = = 5#>>>+9 +el + + + ?>+ 
7? $) ) ) )X  >         4:DIII  JI0 4)UPTUUU  VU" DJT\49emTUU  VU 74 ,&do&&s+++ 4<T:::B B B ;:B, 4$$???" " " @?" DM?##   $# 4:C C C 4:C C C 49! ! ! 4:! ! ! 4:%&&% % '&% 4:! ! ! 4;D999> > :9>2 5)tDDD
 
 ED
 4>t<<<# # =<# 4;. .  .b 4$$???49$7774<T:::1 1 ;: 87 @?1 4<T:::> > ;:> 4:4888V V V 98V 4?===> > > >=>$ 4#>>>   ?> 4'TBBB   CB
B B B BJ '<RVWWW/// / 	/
 / / ;/ / / / XW/d /T  ))) ) 	)
 ) ) ;) ) ) ) )X ,4$  !!! ! 	!
 ! ;! ! ! ! !H .6D    	
  ;    < ,3  *** * 	*
 * ;* * * * *Z .54  &&& & 	&
 & ;& & & & &R 48v: v: v: v:r 4=d;;;3E 3EC 3E3 3E# 3E 3E 3E <;3El 4%4@@@6 6 6 6s 6 6 6 A@6 4(dCCC   s    DC 4;D9996 6 :96
 4:4888   98$ 4(dCCC- - - DC- 4;D999   :9 4;D999D D :9D4 4>t<<<  =< 4?===  >=    48       T       3 3 3 3 3 3"< "< "< "< "< "<J,N ,N ,N ,N^	, 	, 	, 59%1tLLL#+ #+ ML#+L 4&DAAAG G BAG 4?===  >= 4>#>>>' ' ?>' 4<    !  QB B B/ / /
 )():;; **49+>?? ))$**<== ++DJ,@AA  dl    49J J J 4:J J J >4$OOOL L POL
 >&DAAAK%, K K K BAK >'TBBBJ J CBJ
 >-4HHH	 	 IH	 >(dCCCRS   $s) 9 C C    DC6 >)tDDDLM  	#Cy09FI   ED8 4>t<<<
 - - --- 	-
 - - - =<-`        "  
< < <@ d' ( ( ( d& ' ' ' d( ) ) ) d* + + + dl ' ' ' ' d&U 3 3 3 3 d! " " " d,5 9 9 9 9 d       d! " " " d" # # # d       d       do    d!) * * * d+3 4 4 4 dj    dn# $ $ $ d)1 2 2 2 d,4 5 5 5 dk    d$5 1 1 1 1 d')@ A A A do} - - - d&(: ; ; ; d!#5 6 6 6 d.0B C C C d* + + + d= > > > d= > > > d2 3 3 3 d= > > > d3 4 4 4 d0- @ @ @ d0 1 1 1 d/ 0 0 0 d/ 0 0 0 d1 2 2 2 d1 2 2 2 d- . . . d- . . . d- . . . d.0B C C C d0 1 1 1 d+] ; ; ; d" # # # di    di    dm    di    di    dk    dn    dm    dl    do    d    d- . . . d  ! ! ! d' ( ( ( d# $ $ $ dn    d& ' ' ' d" # # # d$ % % % dn    d" # # # d# $ $ $ d* + + + d    dn    dj    d' ( ( ( d/ 0 0 0 d       d# $ $ $ d% & & & d# $ $ $ d! " " " dj    dm    dl" # # # dhU + + + + d(- 8 8 8 el$; < < < el$7 8 8 8 d! " " " d* + + + d"$6 7 7 7 dj    d' ( ( ( d; < < < dn    do    dj    08	   
 9A	   
 ,4	   
 5=	   
 ,4	   
 5=	   
 4<	   
 =E	   
 d+3_ E E E d,4o F F F d/7 I I I d08/ J J J d      
 49$777
 
 
 87
 4:"       ( 74"## 3+d*++E222 5:  ( 4&DAAA C    BA: 4%4@@@@ @ @ A@@F   EL$"4566tDU 5 5 5 5 765p 5?##4 4 4 $#4 5#$$+ + %$+ 4+,,*" *" -,*"Z 4&''  ('  < 4>t<<<K K =<K  8 EK,-- 
    .-$  .> > > 0t//0B0B50I0IJJ
11!4455	 2 21 5 566
   4>"" $T$4     #" 4%&&Td    '&< 4)**"4	 	 	 	 +*	 5%-..A A /.A EJ	*++: : ,+:
 4;D999" " " :9"J 4>t<<<   =<8# # #0D D DN  !1 !1 !1H 4:4888
 
 98
 4%4@@@/ / A@/ 4>""= = = #"= 4)**O O O +*O; ; ;    4?===J J J >=J >4$OOOK K K POKX X Xv !1 0%5! ! !  0@/?,4%0 0 0 ,
 4,$GGG  HG( 4;QUVVV	P 	P WV	P 1 1 1 4*EEE   FE 4<T:::9C 9 9 9 ;:9 !  & 
 SM    B 4=d;;;DH : : : :HSM : : : <;:" 4#>>>3 3 3 3 ?>3 4$$???6 6 6 6 @?6 4&DAAAP3 P P P BAP 4'TBBBPT m m ms m m m m CBmh , , HUOS(), 	,
 , , , ,^ 4*233> >x > > > 43> 419::J Jhuo J J J ;:J 4*233RVI I&uoI@HI I I 43I 419::RVU U&uoU@HU U U ;:U 4*233 !% $ $S S uoS uo	S
 uoS S S 43S 419:: !% $ $	 	 uo	 uo		
 uo	 	 	 ;:	7 7 7 59$%%  &%* 4'TBBB. . . CB.bej uUZ_'=    uz     %* EJ UZ     :>   <  ,> > > CG@ @ @ @ @: 5<RVWWW E E E XWEP 	3    B d* + + + -=,<)1- - - ) 48dSSSS S TSSl   :B B B& & &R2 2 2j  0/%5      
 4,--? ? .-?D  0/$%      
 4+,,X X -,Xv "2!1&E" " " 
  2 4-..B B /.BJ 43;<<BF* * * =<*Z '&O    '&O   
 4?===    >=* 4?===    >=*p p pf  0/$%      
 4+FFF ` ` ` GF`F  0/$%      
 4+FFF @ @ @ GF@F  $. . .b 3    &   49:t : : : : :   4   ( ( (V4 4 4( DHei())T5     *) 4=!!u     "!      .-H    '&txERRR --H   
 48t,,,, , -,,^       F 4:2 2 2 4:4888   98       48t,,,   -, DH:...( ( /.(") (=    > EI;$///$ $ 0/$( 	tx'7D  
  
 DIuz*d;;;$ $ <;$  4:% % % DHei()). . . . . *). #"4;#677##DL$899 (():)BCC ""4;#677""4;#677 4;    . 4<     ! . 4$%%  &%* 4;D999   :9: 4;D999   :9: 49. . . . . . 48@ @ @ @
 486667 7 7 767 486667 7 7 767 !  %-    	!:!: ; ; ;*	**>>%+@+@AA*	**>>%+@+@AA.!!$+..N85;???  /!!$+..N85;???  H$L   ! !1uMMM 49#>>>!re $2 $2 $2 $2 ?>$2N 49$$???H H H @?H      '&tx00!!$),,""4:..$)$$
*
*4<
8
8''	22		DK	(	(t444     * * *     * * *""  !122''(?@@   -    00
(()ABB   !122  4; ' ' '  **  *I*V      4: & & &  48 $ $ $  49 % % %     * * *  *	   ! *	    O*	  
 ! *	   
T\
*
*

T\
*
* !  $. ! !' * * * !  $. ! !' * * *""""''88
  4> " " "$)LLL  49     4<uz B B B B !  $. ! !# & & &  47%* = = = =  47%* = = = =  47%* = = = =uzBBB  47%* = = = =  47%* = = = =  49 % % %  49 % % %  49 % % %  4: & & &  49 % % %  4: & & &  4: & & &  49 % % %  4: & & &  4= ) ) )  49 % % %  4; ' ' '  4: & & &  4: & & &  49 % % %  4> * * * D D D D D D D DE E E$ % 
 
D4L4Ld5 5 
 
0 	 3+		
 	
 	
 	
 	
 5M4Lt5 5 
 
0 	 3+		
 	
 	
 	
 	

 .-CT    0/ct     4,3Sd K K K K--d.?.DcJJ //0A0H#NN   4,13 7 7 7  4,3S 9 9 9  4,4c : : :  4,4c : : :  4,3S 9 9 9  4,<c B B B--d.?.DcJJ //0A0H#NN   4-t 4 4 4  405w ? ? ?  407 A A A  405w ? ? ?  407 A A A  427 A A A  4297 C C C  427 A A A  4297 C C C  43Z @ @ @  4-t 4 4 4  4-t 4 4 4, , ,  T.35E    t079K    T.35E    t079K    T.35E    t079K  
    C        "K 0 0 0  )+= > > >  "K 0 0 0  !: . . .  *,? @ @ @  "K 0 0 0  C        !3 ' ' '  & 1 1 1  "K 0 0 0  "K 0 0 0  !: . . .  "K 0 0 0  C        T " " "   ( ( (    $,   , , , "  $/ " "#5 6 6 6   $+  z * * * "  $/ " "#6 7 7 7   $,   , , ,   - - -  !4? 3 3 3  t{ + + +  !4? 3 3 3   - - - 4+,,   -, 4=$%%  &%  4?&''  (' 4>""  #" (M')) 8 8LFD1((0011$7777 4<  7 7 ! 7 59+344  54
 59%;<<  =< 59>&455K K 65K 759>6"" 6uy~*2336 6 436 59>())%) = = = = *)=@ K J J J J J ! " " " 122	S 	S 32	S 59)./// / 0// 59)455/ / 65/ &DAAA #    BA8 59?/788  98 59)677+ + 87+&[4444y1'233  43 '344  54 '<==  >= '=>>  ?> '>??
 
 @?
 'HII	
 	
 JI	
 'BCC  DC '=>>	
 	
 ?>	
 'GHH

 

 IH

 '9::	
 	
 ;:	
 '122  32 '233  43 '344  54 uy)<==	
 	
 >=	
 	
 	$   HH\                ! ! ! ! ! ! +  * , , , '  ' ) ) )       ,  + - - -       %  $ & & & & &s   CF3CI< I< CJJCJ