
    קg             -       :^   d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z	d dl
mZ d dl	mZmZmZ d dlmZmZmZmZ d dlmZ d dlmZmZ d dl
mZmZmZmZmZmZm Z m!Z! d d	l"m#Z#m$Z$m%Z%m&Z&m'Z' d d
l(m)Z)m*Z* d dl+m,Z- e	j.        j/        Z/e	j0        1                    ddd          Z2d Z3defdZ4d Z5d Z6 e3e/j7        e/j8        g           e'            ddde	j9        ddfd                        Z: e3e/j;        j<        e/j;        j=        g           e'            d                         Z> e3e/j?        j<        e/j?        j=        g           e'            ddd                        Z? e3e/j@                   e'            d                         Z@ e3e/jA        j<        e/jA        j=        e/jB        j<        e/jB        j=        g           e'dd          d                         ZC e3e/jD        j<        e/jD        j=        g           e'            d                         ZDd ZE e3e/jF        j<        e/jF        j=        g           e'            d                          ZG e3e/jH        j<        e/jH        j=        g           e'            d!                         ZI e3e/jJ        jK                  dd"d#            ZL e3e/jJ        j<                  e	jM        dddd$d%            ZN e3e/jO        j<        e/jO        j=        g           e'            e	jM        dddd$d&                        ZP e3e/jO        jQ        e/jO        jR        g           e'            e	jM        dddd$d'                        ZS e3e/jT        j<        e/jT        j=        g           e'            ddddd$d(                        ZU e3e/jV        j<        e/jV        j=        g           e'            d)                         ZW e3e/jX        j<                  dd*            ZYd+ ZZ e3e/j[        j<                  d,             Z\ e3e/j]                  	 	 	 dd-ed.ed/ed0ee         d1ee^         d2ee	j_                 fd3            Z` e3e/ja                  	 dd4ed5ed6ed2ee	j_                 fd7            Zb e3e/jc                  d8d8dd9d-ed4ed5ed6ed2ee	j_                 f
d:            Zd e3e/je                  	 	 	 	 dd;e	j        d<e	j        d0ee         d=ee         d2ee	j_                 d>effd?            Zg e3e/jh        j<                  d@dAdBedCeidDedEe	j        dFe^dGefdHefdI            Zj e3e/jk        j<                  d@dAdBedCeidDedEe	j        dFe^dGefdHefdJ            Zl e'             e3e/jm        j<                  dK                         Zn e3e/jo        j<                  dddd dddLdMedFe^dNee         dee         dOee         dPeidQefdHefdR            Zp e3e/jq        j<        e/jq        jr        g           e'            dS                         Zs e3e/jq        jt                  ddT            Zu e3e/jv        j<        e/jv        jr        g           e'            dU                         Zw e3e/jv        jt                  ddV            Zx e3e/jy        j<                  dW             Zz e3e/jy        j=                  dX             Z{ e3e/j|        j<                  dY             Z} e3e/j|        j~                  dZ             Z e3e/j        j<                  d[             Z e3e/j        j<                  dddddd\d]            Z e3e/j        j<                  dd^            Z e3e/j        j<                  dd_            Z e3e/j        j<                  dd`            Z e3e/j        j<                  da             Z e3e/j        j~                  db             ZdBedce^fddZdBedeedfe^fdgZ	 ddhedce^dieffdjZddeedce^dke^fdlZdeedmednefdce^fdoZ	 ddqe^dped-edre^fdsZdte^fduZ e3e/j        j<        e/j        j        g           e'dvdw          ddeedte^dyeffdz                        Z e3e/j        j<        e/j        j=        g           e'            d-edHefd{                        Z e3e/j        g           e'dvdw          d-efd|                        Zd}edHefd~Z e3e/j                   e'            dBedeedefdHefd                        Z e3e/j                   e'            ddBedeedefdHefd                        Z e3e/j                   e'            ddBedefdHefd                        Z e3e/j                   e'            ddBedefdHefd                        Z e3e/j        j<                  ddeedefdeffd            Z e3e/j        j<        e/j        j=        g           e'            d-ededHefd                        Z e3e/j        j<                  ddeedeffd            Z e3e/j        j<        e/j        j=        g           e'ddd          ddddBedefdefdHeeeef         fd                        Z e3e/j        j<        e/j        j=        g           e'            dddededmedefdHef
d                        Z e3e/j        j<        e/j        j=        g           e'ddxd          d@ddeedefdHeeeef         fd                        Z e3e/j        j<        e/j        j=        g           e'ddd          d@dddeedefdefdHeeeef         fd                        Z e3e/j        j<        e/j        j=        g           e'            d@dddededmednefdefdHefd                        Z e3e/j                   e'ddxd          	 	 ddededefdefdHeeeef         f
d                        Zde^dHeefeff         fdZ e3e/j        j<        e/j        j=        g           e'dd          ddeede^dHeeef         fd                        Z e3e/j        j<        e/j        j        g           e'dddd          deedHeeeeef         fd                        Z e3e/j        j<                  	 	 	 ddeedefdefdee^         fd            ZdededHeeei         eei         f         fdZdededfee^         dHeeef         fdZd-ededHeffdZ e3e/j                  d@dddddddeedmednefdefdpee         dee         dee         dee         dHeeeeef         fd            Z e3e/j        j<        e/j        j=        g          d@ddddeedmedefdnefdefdee         dHefd            Z e3e/j                   e'dd          	 	 	 ddBedeedefdefdefdHeeef         fd                        Z e3e/j        j<                  d             Z e3e/j                   e'            	 	 dd-ededednefdefdHefd                        Zd Zd Z e3e/j                   e'            d                         Z e3e/j                   e'            d                         Zd Z e3e/j                   e'dæ          dĄ                         Z e3e/j                   e'dæ          dń                         ZdƄ Z e3e/j                   e'            dǄ                         Z e3e/j                   e'            dȄ                         Z e3e/j        j<        e/j        j        e/j        j<        e/j        j        g           e'dæ          dɄ                         Zdʄ Z e3e/j                   e'            d˄                         Z e3e/j                   e'            d̄                         Z e3e/j        j<        e/j        j        e/j        j<        e/j        j        g           e'dæ          d̈́                         Z e3e/j                   e'            ddBededHefdЄ                        Z e3e/j                   e'            dedBedededHef
dӄ                        Z e3e/j        j<        e/j        j=        g           e'            d8d8dԜdՄ                        Z e3e/j        j<        e/j        j=        g           e'            dd"dք                        Z e3e/j        j                  dd؄            Z e3e/j        j                  ddل            Z e3e/j        j<        e/j        j=        g           e'            ddڄ                        Z e3e/j        j<                  	 	 ddۄ            Z e3e/j                   e'            d܄                         Zd݄ Zdd߄Z	 dde	j        d.e	j        deeei         eif         deeei         eif         deeei         eif         defdeideeeei         eif                  fdZd Z e3e/j        j<                  de	j        d.e	j        d0e	j        deei         deei         deei         defdeei         deifd            Ze	j        j        re	j0        1                    ddd          Z e3e	j.        j        j        j<                  d             Z e3e	j.        j        j        j<                  d             Ze	j        j        rCe	j0        1                    ddd          Z  e3e	j.        j        j                  d             Ze	j0        1                    ddd          Z e3e	j.        j        j        j<                  d             Z e3e	j.        j        j        j<                   e3e	j.        j        j        j	                  d                         Z
e	j0        1                    ddd          Z e3e	j.        j        j                  	 	 	 	 dd            Zd Z e3e/j        j<                  	 	 	 	 	 dd            Zd Z e3e/j        j<                  d             Z e3e/j                   e'            	 	 	 	 	 dd                        Z e3e/j                   e'dæ          d                         Z e3e/j        j<                  d             Z e3e/j        j<                  d             Z e3e/j        j<                  d             Z e3e/j                   e'dæ          d                          Z dedke^fdZ! e3e/j"                   e'dd          d                         Z# e3e/j$                   e'dæ          d                         Z% e3e/j&                   e'dd          d                         Z' e3e/j(                   e'dæ          d                         Z) e3e/j*        j                  dd            Z+ e3e/j,        j<        e/j,        j=        g           e'            d                         Z- e3e/j.        j<        e/j.        j=        g           e'            dd	d
eideifd                        Z. e3e/j/        j        e/j0        j        g          d             Z1 e3e/j2        j<        g          d             Z3 e3e/j4        j<        e/j4        j=        g           e'            d8d8dԜd                        Z5 e3e/j6        j<        e/j7        j<        g          dddd            Z8 e3e/j9        j<        g          dddd            Z: e3e/j;        g           e'            d                         Z< e3e/j=        g          d             Z> e3e/j?        g          d             Z@ e3e/jA        g          d             ZB e3e/jC        j<                  d             ZD e3e/jE                   e'            d                         ZF e3e/jG        j<                  	 	 	 	 	 	 dd            ZH e3e/jI        j<                  d             ZJddZK e3e/jL        j<        e/jL        j=        g           e'            dddd                        ZM e3e/jN        j<        e/jO        j<        g          d             ZP e3e/jN        jt        e/jN        jQ        e/jO        jt        e/jO        jQ        e/jR        j<        e/jR        jS        g           e'dd          dd                        ZT e3e/jU        j<                  d              ZV e3e/jW        j<                  d!             ZX e3e/jY        j<                  d"             ZZ e3e/j[        j\        e/j]        j\        e/j[        j        e/j]        j        e/j^        j<        e/j_        j<        e/j`        j<        g          d#             Za e3e/jb        j\        e/jc        j\        e/jb        j        e/jc        j        g          dd$            Zd e3e/je        j<        e/je        jf        g          d%             Zgd& Zh e3e/ji        j        e/ji        j\        g          d'             Zj e3e/jk        j        e/jk        j\        g          d(             Zl e3e/jm        j<                  d)             Zn e3e/jo        j        e/jo        j\        g          d*             Zp e3e/jq        j        e/jq        j\        g          d+             Zr e3e/js        j<                  d,             Zt e3e/ju        j<        e/jv        j<        g          dd-            Zw e3e/jx        j\                  d.             Zy e3e/jz        j<                  d/             Z{ e3e/j|                  d0             Z} e3e/j~                   e'            d1                         Z e3e/j                  d2             Z e3e/j        j<                  dd3            Z e3e/j        j<                  d4             Zdd5Z e3e/j        j<                  d6             Zd7 Zd8 Zd9 Zd: Z	 dd-ed;eid<eid=eid>eid?eid@eidAeidBeidCeidDeidEeidFeidGeidHeidIeidJeidKeidLeidMeidqe^dNeff,dOZdP Zd-eded;eid<eid=eid>eid?eid@eidAeidBeidCeidDeidHeidIeidJeidKeidLeidMeidqe^f&dQZdR Z e3e/j        j<                  dS             Z e3e/j        j<                  	 	 	 	 ddT            Z e3e/j        j<                  dU             Z e3e/j                   e'            dV                         ZdW Z e3e/j                   e'            dX                         Z e3e/j                   e'dd          	 	 	 	 ddY                        Z e3e/j                   e'dæ          dZ                         Zd-ed[efd\Z G d] d^e          Zd-ed[ed_eifd`Z e3e/j        j<                  da             Z e3e/j                   e'            db                         Z e3e/j                   e'dÐdc          dd                         Z e3e/j        j<        g          de             Z e3e/j        j<                  	 	 	 	 	 ddf            Z e3e/j        ji                  dg             Z e3e/j        j<                  dh             Z e3e/j        j<                  ddi            ZddCeidjeidkeffdlZdm Zdn Z e3e/j        j<                  ddo            ZddpZddqZdr ZddsZd dtZ e3e/j        j<                  du             Z e3e/j                  dv             Z e3e/j        j        e/j        j        e/j        j        e/j        j        g           e'            ddw                        Z e3e/j        j        e/j        j        e/j        j        e/j        j        g          ddx            Z e3e/j        g          	 	 	 	 ddzed{ed|ed}eߐd~efdefdee         fd            Z e3e/j        g          	 	 	 	 ddzed{ed|edee         defd}eߐd~efdefdee         fd            Z e3e/j        g          	 ddedzed{ed|edededededeideid}eߐd~efdededee         fd            Z e3e/j        g          	 	 	 	 ddzed{ed|ed}eߐd~efdee         dee         fd            Z e3e/j        g          	 	 ddedzed{ed|ededed}eߐd~efdee         dee         fd            Z e3e/j        g          	 	 	 ddzed{ed|edee         defd~efdee         fd            Z e3e/j        g          	 	 ddedzed{ed|edee         dedededed}eߐdeef         d~efdee         fd            Z e3e/j        g          	 ddedzed{ed|ededededededededeideid}eߐd~efdee         f d            Z e3e/j        g          	 	 	 	 	 ddzed{ed|edee         dee         deideid}eߐd~efdefdee         deei         deei         dee         dee         fd            Z e3e/j        g          	 	 	 ddedzed{ed|edededededeideid}eߐd~efdededee         deei         deei         f"d            Z e3e/j        g          	 	 	 	 	 ddzed{ed|ed0ee         dee         dee         deei         deei         d}eߐdeidefdee         dee         dee         deei         fd            Z e3e/j        g          	 	 	 d dedzed{ed|ed0ee         dee         dee         de	j        de	j        ded}eߐdededeidefdee         deei         deff$d            Z e3e/j        j<        g          	 	 	 	 ddBe	j        d6e	j        de	j        de	j        d0ee	j                 dee	j                 d2ee	j_                 deffd            Z e3e/j        j        e/j        j        g           e'            dd                        Z e3e/j        j                  dd            Z e3e/j        j<        e/j        j=        g           e'            ddd"d                        Zd Zd Z e3e/j        j<        e/j        j<        g          dd            Z e3e/j        j<        e/j        j<        g          dd            Z e3e/j        j<        e/j        j<        g          	 	 ddedeeeie	j        f                  deeeie	j        f                  dee         dee         f
d            Z e3e/j        j<        e/j        j<        g          dd            Z e3e/j        j<        e/j        j        e/j        jS        e/j        j        g          dd            Zd Z e3e/j        j<                  	 	 dd            Z e3e/j        j<                  d             Z e3e/j        j<                  d             Zd Zd Z e3e/j        j<        e/j        j<        g          dd            Z e3e/j         j<                  dd            Z  e3e/j        j<                  ddÄ            Z e3e/j        j<        e/j        jS        g           e'dd          ddĄ                        Ze	j        Zdń Z e3e/j        j<                  dƄ             Z e3e/j	        j<                  dǄ             Z	 e3e/j
        j<                  dȄ             Z e3e/j        j<                  dɄ             Z e3e/j        j        e/j        j        g           e'            dddʜd˄                        Z e3e/j        g           e'            d	d̈́                        Z e3e/j        j<        e/j        j<        g          	 	 dd΄            Z e3e/j        j<                  dτ             Z e3e/j        j<        e/j        j=        g           e'            ddЄ                        Z e3e	j.        j/        j                  dф             Z e3e	j.        j/        j                  d҄             Z e3e/j                   e'            dddddӜdԄ                        ZdՄ Z e3e/j                  	 d
dք            Z e3e/j                  	 d
dׄ            Z e3e/j                   	 d
d؄            Z! e3e/j"                   e'            dddٜdڄ                        Z# e3e/j$                   e'            deidBedHefd܄                        Z% e3e/j&                  dBefd݄            Z' e3e/j(        j<                  	 ddedOee         deei         defd            Z) e3e/j*        j<                  	 ddedOee         deei         fd            Z+d Z,d Z- e,e/j.                    e,e/j/                    e,e/j0                    e,e/j1                    e,e/j2                    e,e/j3                    e,e/j4                    e,e/j5                    e,e/j6                    e-e/j7                    e-e/j8                    e-e/j9                    e-e/j:                    e-e/j;                    e-e/j<                    e-e/j=                    e-e/j>                    e-e/j?                    e-e/j@                    e-e/jA                    e-e/jB                   d dl(Z	d dlCZ	d dlDZ	d ZE eE             dS (      N)Enum)ListOptionalSequenceTupleUnion)SymBoolSymFloatTensor)_add_op_to_registry_convert_out_paramsglobal_decomposition_table
meta_table)
OpOverload)_prim_elementwise_meta$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)corresponding_complex_dtypecorresponding_real_dtypeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDIntLikemake_contiguous_strides_forNumber
TensorLike)_maybe_convert_to_dtype_maybe_resize_out_resize_output_check_safe_copy_outout_wrapper)_broadcast_shapes_maybe_broadcast)_pytreeatenIMPLMetac                       fd}|S )Nc                 \     t                       fd}t          j        |            S )Nc                 4    t          t          |            d S N)r   r   )opfns    U/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/_meta_registrations.pyregisterz0register_meta.<locals>.wrapper.<locals>.register0   s    
B33333    )r   pytree	tree_map_)r+   r-   r*   s   ` r,   wrapperzregister_meta.<locals>.wrapper-   sD     $$	4 	4 	4 	4 	4 	2&&&	r.    )r*   r1   s   ` r,   register_metar3   ,   s#         Nr.   type_promotionc                     t          j        |d| i\  }fd|D             }t          | }t          |dt          j        iS )Ntype_promotion_kindc                 0    g | ]}t          |          S r2   )r   ).0xresult_dtypes     r,   
<listcomp>z$elementwise_meta.<locals>.<listcomp>B   s$    CCC#A|44CCCr.   r4   )utilsr   r!   r   r   DEFAULT)r4   args_r:   s      @r,   elementwise_metar@   9   sp    
 .	* OA| DCCCdCCCD T"D "	BJ  r.   c                     t           j        t           j        t           j        t           j        t           j        t           j        i}|                    | |           S r)   )torch	complex32halfcfloatfloatcdoubledoubleget)dtypefrom_complexs     r,   toRealValueTyperL   M   s<    eku|L
 E5)))r.   c                 x     t          t           g|R            t          j         k     fd           d S )Nc                      d d  S )Nzoutput with shape z# doesn't match the broadcast shape r2   )broadcasted_shape
self_shapes   r,   <lambda>z)check_inplace_broadcast.<locals>.<lambda>Z   s    gZggTegg r.   )tupler    rB   _check)rP   
args_shaperO   s   ` @r,   check_inplace_broadcastrU   V   sX    /
HZHHHII	LZ'ggggg    r.   Fc	                 \   	 t           t          j                  r,t          j                                         dk    d            t          t          j                  r,t          j                                        dk    d            t          d  fD                       rVt          j        t          j                              		nAt          j        t          j	                  	fd           npt          j                    t          t          j
                  sJ t          j        t          t                     fd           t          t                    sJ t          j        dk    d            t          j        f|d||	          S )
Nr   c                      dS Nz:linspace only supports 0-dimensional start and end tensorsr2   r2   r.   r,   rQ   z(meta_linspace_logspace.<locals>.<lambda>n       P r.   c                      dS rX   r2   r2   r.   r,   rQ   z(meta_linspace_logspace.<locals>.<lambda>s   rY   r.   c              3   @   K   | ]}t          |t                    V  d S r)   )
isinstancecomplex)r8   args     r,   	<genexpr>z)meta_linspace_logspace.<locals>.<genexpr>v   s,      
C
C:c7##
C
C
C
C
C
Cr.   c                      d  d S )Nzlinspace(): inferred dtype z& can't be safely cast to passed dtype r2   )default_complex_dtyperJ   s   r,   rQ   z(meta_linspace_logspace.<locals>.<lambda>   s    z6Kzzsxzz r.   c                      dt                    j         dt                     j         dt                    j         dS )Nz4received an invalid combination of arguments - got (, ))type__name__)endstartstepss   r,   rQ   z(meta_linspace_logspace.<locals>.<lambda>   sY     Hu++H Hs)),H H04U0DH H H r.   c                      dS )Nz$number of steps must be non-negativer2   r2   r.   r,   rQ   z(meta_linspace_logspace.<locals>.<lambda>   s    %K r.   meta)rJ   layoutdevice
pin_memoryrequires_grad)r\   rB   r   rS   dimanyr<   r   get_default_dtypeis_complex_dtyperJ   _check_typer   empty)
rh   rg   ri   baserJ   rm   rl   rn   ro   ra   s
   ``` `    @r,   meta_linspace_logspacerw   ^   s    %&& 
IIKK1PP	
 	
 	
 #u|$$ 
GGIINPP	
 	
 	

 
C
CsE/B
C
C
CCC 3 % A#%%!
 !
 =)EEL&u--zzzzz   
 2022eU[))))) 
5'""	H 	H 	H 	H 	H 	H  
 eW%%%%%	L!KKLLL;	#   r.   c                    t          j        j        t           j        k    fd           t          j        |                                 dk    o                                dk     d            |                     j                  S )Nc                      d j          S )Nz2take(): Expected a long tensor for index, but got rJ   indexs   r,   rQ   zmeta_take.<locals>.<lambda>   s    RU[RR r.   r   c                      dS )Nz*take(): tried to take from an empty tensorr2   r2   r.   r,   rQ   zmeta_take.<locals>.<lambda>   s    < r.   )rB   rS   rJ   long_check_indexnumel	new_emptyshape)selfr|   s    `r,   	meta_taker      s     
Luz!RRRR  
 
ZZ\\Q55;;==A#56<<   >>%+&&&r.   rp   c                J     j         }j         }t          j        ||k    d            t          j                                       dk    o                              dk     fd           t	           j        j                  }                     |          S )Nc                      dS )Nz=linalg.cross: inputs must have the same number of dimensions.r2   r2   r.   r,   rQ   zlinalg_cross.<locals>.<lambda>       O r.      c                  f    d  d                                 d                                 S )Nzlinalg.cross: inputs dimension z must have length 3. Got  and size)rp   otherr   s   r,   rQ   zlinalg_cross.<locals>.<lambda>   sG    :c : :99S>>: :(-

3: : r.   )ndimrB   rS   r   r    r   r   )r   r   rp   x_dy_d	out_shapes   ```   r,   linalg_crossr      s     )C
*C	Ls
OO   
L		#!4

31 4	
 	
 	
 	
 	
 	
   "$*ek::I>>)$$$r.   c                     t          | d           t          | d           t          j        | t          j                  S )Nzlinalg.matrix_expmemory_format)squareCheckInputscheckFloatingOrComplexrB   
empty_likecontiguous_formatr   s    r,   linalg_matrix_expr      s?     d/0004!4555D0GHHHHr.   valuesindicesc                    t          j        | j        | j        | j                  }t          j        | j        | j        t           j                  }|                                 dk    r | j        dk    rt          || j                   ||fS )Nrm   rJ   r   )	rB   ru   r   rm   rJ   int64r   r   maybe_wrap_dim)r   rp   r   r   s       r,   	cummaxminr      su    
 [DKtzJJJFk$*T[LLLGzz||qTY!^^sDI&&&7?r.   c                 x    t          || j                   t          j        |                                           S r)   )r   r   rB   r   
contiguous)r   rp   s     r,   logcumsumexpr      s3     3	"""D!!,,...r.   c                    |j         }t          |          }||z
  }t          t          |                    }d t          |          D             }	|D ]}
d|	|
<   g g }}|D ]5}
|	|
         s|                    |
            |                    |
           6||z   }t          |          }|                                |d |         }|                    fdd           |||d          z   }|                    |          }dgt          |j        |d                    z   }|	                    |          }|
                    d          }||d<   |}t          t          |                    D ]}|||                  ||dz   <   | 	                    |          } d t          |          D             }d}|dz
  }|dk    r=||                     d          z  |||         <   ||||                  z  }|dz  }|dk    =t          ||          D ]&}|                     d||z
  z             |||         <   '|                     |||                                           S )	Nc                     g | ]}d S Fr2   r8   r?   s     r,   r;   z_exec_fft.<locals>.<listcomp>   s    555A%555r.   Tc                     |          S r)   r2   r9   self_stridess    r,   rQ   z_exec_fft.<locals>.<lambda>   s    <? r.   keyreverser   r      c                     g | ]}d S r   r2   r   s     r,   r;   z_exec_fft.<locals>.<listcomp>  s    ***1***r.   )r   lenlistrangeappendstridesortpermuter   reshaper   
as_stridedstorage_offset)outr   	out_sizesrp   forwardr   signal_ndim
batch_dimsdim_permuteis_transformed_dimdleftright	batch_endtmpinputbatched_sizes
batch_sizebatched_out_sizesiout_stridesbatch_numelr   s                         @r,   	_exec_fftr      s   9Dc((K#J uT{{##K55t555 % % $1 b%D  !!$ 	KKNNNNLLOOOO,KD		I;;==L
jyj
!CHH****DH999IJJ//KLL%%E D4JKK 8999MMM-((EAJ!M!%3s88__ 5 5#,SV#4!a%  
++'
(
(C +*eDkk***KKQA
q&&&1CJJqMM&AKN#yQ00	Q q&& :t$$ G G&)jja*n1E&F&FKN##>>)[#2D2D2F2FGGGr.   c                     | j         j        sJ | j        }|                     |          }|s|S |d d          }|                                 |                    fdd           t          || |||          }|S )Nc                     |          S r)   r2   r   s    r,   rQ   zmeta_fft_c2c.<locals>.<lambda>  s    <? r.   Tr   )rJ   
is_complexr   r   r   r   r   )r   rp   normalizationr   r   outputsorted_dimsr   s          @r,   meta_fft_c2cr     s     :    
I^^I&&F aaa&K;;==L2222DAAAvtYWEEFMr.   c                     | j         j        sJ t          |                                           }|r|d         }||         dz  dz   }|||<   |                     |t          j        | j                             S )Nr      r   rz   )rJ   is_floating_pointr   r   r   r<   r   )r   rp   r   onesidedoutput_sizeslast_dimlast_dim_halfsizes          r,   meta_fft_r2cr   #  s     :''''		$$L 3r7)(3q8A=!2X>>E=djII    r.   )	generatorc                H    t          |t          j        | g                    S r)   )r   rB   Size)nr   r   s      r,   meta_randpermr   3  s    S%*aS//222r.   rJ   rl   rm   rn   c                4    t          j        | ||||          S Nr   rB   ru   )r   rJ   rl   rm   rn   s        r,   meta_randperm_defaultr   8  s(     ;	vf   r.   c                4    t          j        |||||          S r   r   )highr   rJ   rl   rm   rn   s         r,   meta_randintr   F  s(     ;E&J   r.   c                4    t          j        |||||          S r   r   )lowr   r   rJ   rl   rm   rn   s          r,   meta_randint_lowr   V  s(     ;E&J   r.   c                4    t          j        | ||||          S r   r   )r   rJ   rl   rm   rn   s        r,   meta_rand_defaultr   g  s(     ;E&J   r.   c                     | j         j        sJ t          |                                           }|||d         <   |                     |t          | j                             S )Nr   rz   )rJ   r   r   r   r   rL   )r   rp   r   lastdimr   s        r,   meta_fft_c2rr   o  sX     :    		$$L#LR>>,odj.I.I>JJJr.   c                    ddl m}  ||           s't          j        |           dk    rt	          d          t          |t                    rr|                    | |          }|                                 |                                k    r2t          j
                            ||                                            | S )Nr   )free_unbacked_symbolsr   zQmore than one element of the written-to tensor refers to a single memory location)%torch.fx.experimental.symbolic_shapesr   rB   _debug_has_internal_overlapRuntimeErrorr\   r   tor   r#   expand_copydefault)r   srcnon_blockingr   intermediates        r,   
meta_copy_r  x  s     LKKKKK "!$''
,1,Md,S,SWX,X,X_
 
 	
 #v @vvdL1199;;,++----$$\499;;???Kr.   c                 <   t          |                                           }t          |                                           }||                                 k    rdn||         ||         z  }|                    |d           |                    ||           ||fS Nr   )r   r   r   rp   insert)tensorrp   result_sizesresult_strides
new_strides        r,   inferUnsqueezeGeometryr
    s    &&L&--//**NVZZ\\))|C/@>RUCV/VJQ#z***''r.   c                     t          ||                                 dz             }t          | |          \  }}|                     ||           | S r  )r   rp   r
  as_strided_)r   rp   g_sizes	g_stridess       r,   meta_unsqueeze_r    sL    
dhhjj1n
-
-C/c::GYWi(((Kr.   r   weight_metabias_activation_opt	out_dtypec                 l   t          | j                  }|6|                    d          |                    d          k    s
J d            |                    d          |                     d          dz  k    sJ |                    d          |d<   t          | j                  dk    s
J d            d|                     d          f}|/| j        t
          j        k    r|t
          j        k    s
J d            |                     ||| j        n|          	                    ||          }|S )	Nr   zoutput size mismatchr   r   r   z*we can only handle the squashed input case9out_dtype is only supported for i8i8->i32 linear operatorrz   )
r   r   r   r   rJ   rB   int8int32r   r   )	r   r  r  r  r  r  r   transposed_stridesr   s	            r,   meta_sparse_structured_linearr    s8    $$L{{1~~1---/E---;;q>>UZZ^^a/////{{1~~L u{q   "N   UZZ]]+K5:%%)u{*B*B*BF +C*B*B__&.ekkI    j122 
 Mr.   mat1	mat1_metamat2c                    t          | j                  dk    sJ t          |j                  dk    sJ t          |j                  dk    sJ |                     d          |                    d          dz  k    sJ |                     d          |                    d          g}|/|j        t          j        k    r|t          j        k    s
J d            |                    |||j        n|          }|S )Nr   r   r   r  rz   r   r   r   rJ   rB   r  r  r   )r  r  r  r  r   r   s         r,   meta_sparse_structured_mmr     s     tz??ay1$$$$tz??a99Q<<499Q<<!+++++IIaLL$))A,,/LJ%*$$ek)A)A)AF *B)A)A^^%-djj9   F
 Mr.   r   )alphabetar  c                   t          | j                  dk    s
J d            t          |j                  dk    sJ t          |j                  dk    sJ t          |j                  dk    sJ |                     d          |                    d          k    s
J d            |                    d          |                    d          dz  k    sJ |                    d          |                    d          g}|/|j        t          j        k    r|t          j        k    s
J d            |                    |||j        n|          }|S )Nr   zEonly input broadcasted to columns of mat1 * mat2 product is supportedr   r   r  rz   r  )	r   r  r  r  r!  r"  r  r   r   s	            r,   meta_sparse_structured_addmmr$    se    	EKAN 	tz??ay1$$$$tz??a::a==DII	    N   99Q<<499Q<<!+++++IIaLL$))A,,/LJ%*$$ek)A)A)AF *B)A)A^^%-djj9   F
 Mr.   compressed_Adense_Br!  transpose_resultc                    |j         t          j        t          j        t          j        t          j        hv s
J d            | j         |j         k    s
J d            t          |j                  dk    s
J d            | j         t          j        k    }|rdnd}|                    d          }|                    d          }	| 	                                d	z  ||z  z  }
||
|                    d          k    sJ |1|r%|t          j        t          j        t          j
        hv s
J d
            |r|	|
fn|
|	f}|                    ||          }|S )Nz2_cslt_sparse_mm only supports fp16, bf16, and int8zinputs must have the same dtyper   z'_cslt_sparse_mm only supports 2d inputs
   	   r   r      z?out_dtype is only supported for i8i8->fp16, bf16, or i32 matmulrz   )rJ   rB   float32float16bfloat16r  r   r   r   r   r  r   )r%  r&  r  r!  r  r'  is_int8_input_typecompression_factorkr   moutput_shaperesults                r,   meta__cslt_sparse_mmr5     s    =
	   
 <   ...0Q...w}"""$M"""%+uz918qQAQA						"(:Q(>?ADIIaLL    ! 	MiMNK4
 '
 '
 '
 M	'
 '
 '

 .9Aq66Aq6L|9==FMr.   T)include_selfr   rp   r|   sourcereducer6  returnc                B    t          j        | t           j                  S Nr   )rB   r   r   r   rp   r|   r7  r8  r6  s         r,   meta_index_reducer=  %  s     D0GHHHHr.   c                    | S r)   r2   r<  s         r,   meta_index_reduce_r?  2  s	     Kr.   c                     t          |                                           }|                                 dk    r|                                ||<   |                     |          S )Nr   )r   r   rp   r   r   )r   rp   r|   result_sizes       r,   meta_index_selectrB  @  sN     tyy{{##KxxzzA~~ ;;==C>>+&&&r.   )lengthsr   offsetsaxisunsafeinitialdatarC  rD  rE  rF  c                     |t          d           fd}| ||j                  S |,|j        d d         |j        d         dz
  fz   }	 ||	          S t          d          )Nz?segment_reduce(): indices based reduction is not supported yet.c                 x    t          j        | j        dz   d          z   j        dt           j                  S )Nr   rk   rJ   rm   r   )rB   ru   r   rJ   r   )lengths_shaperE  rH  s    r,   segment_reduce_lengths_tensorz:meta_segment_reduce.<locals>.segment_reduce_lengths_tensorZ  sA    {DJtaxzz22*1	
 
 
 	
r.   r   r   z<segment_reduce(): Either lengths or offsets must be defined.)NotImplementedErrorr   r   )
rH  r8  rC  r   rD  rE  rF  rG  rM  rL  s
   `    `    r,   meta_segment_reducerO  I  s     !M
 
 	

 
 
 
 
 
 ,,W];;; crc*gmB.?!.C-EE,,];;;
U
V
VVr.   c                 ,    |                      d          S Nr2   r   r   s    r,   meta_maxrS  m       >>"r.   c                     t          j        | j        |f          }t          | ||          }|                     |          |                     |t
          j                  fS Nrz   r<   reduction_dimsr   _compute_reduction_shaper   rB   r~   r   rp   keepdimr3  s       r,   meta_max_dimr\  s  W    

tzC6
2
2C+D#w??L|$$|5:66 r.   c                 ,    |                      d          S rQ  rR  r   s    r,   meta_minr_  }  rT  r.   c                     t          j        | j        |f          }t          | ||          }|                     |          |                     |t
          j                  fS rV  rW  rZ  s       r,   meta_min_dimra    r]  r.   c                     |                                  rt          | j                  }nt          | t          j                  \  }}t          j        | |          S )Nr6   rz   )r   r   rJ   r   r   INT_TO_FLOATrB   r   )r   r:   r?   s      r,   
meta_anglere    s^     
/
;;, ? L
 
 
< D5555r.   c                     t          j        ||                                 | j                   |                    t          j        |                     S r)   )rB   _resize_output_r   rm   copy_angle)r   r   s     r,   meta_angle_outrj    s=    	#tyy{{DK88899U[&&'''r.   c                     d S r)   r2   )vals    r,   assert_asyncrm        
Fr.   c                     d S r)   r2   )rl  
assert_msgs     r,   assert_async_metarq    rn  r.   c                     d S r)   r2   )ss    r,   
print_metart    rn  r.   rJ   rl   rm   rn   r   c                 .    t          j        dd          S )Nr   rk   rm   r   ru  s        r,   make_dep_tokenrx    s     ;q((((r.   c                     ddl m} t          | t          t          f          rt          d           || ||           d S )Nr   )constrain_range'Constraining SymFloat or Symbool is nyiminmax)r   rz  r\   r
   r	   
ValueError)r   r}  r~  rz  s       r,   sym_constrain_ranger    s[     FEEEEE$7+,, DBCCCODcs++++++r.   c                 @    t                               | ||           |S Nr|  )r#   r  r   r}  r~  	dep_tokens       r,   functional_sym_constrain_ranger    s#    Ts444r.   c                     ddl m} t          | t          t          f          rt          d           || ||           d S )Nr   )_constrain_range_for_sizer{  r|  )r   r  r\   r
   r	   r  )r   r}  r~  r  s       r,   sym_constrain_range_for_sizer    s\     POOOOO$7+,, DBCCCd555555r.   c                 @    t                               | ||           |S r  )r#   r  r  s       r,   'functional_sym_constrain_range_for_sizer    s#    %%d%===r.   c                     |S r)   r2   )rl  rp  r  s      r,   functional_assert_async_metar    s    r.   f_namec                    |                                  dk    sJ | d            |                     d          |                     d          k    s9J | d|                     d           d|                     d           d            d S )Nr   z3: The input tensor must have at least 2 dimensions.r   z5: A must be batches of square matrices, but they are  by 	 matrices)rp   r   )r   r  s     r,   r   r     s    

a
EEE 	 			"2&&&
rrtyyQS}}rrZ^ZcZcdfZgZgrrr 	'&&&&r.   Anamec                     t          j         j        j        k     fd           t          j         j        j        k     fd           t          j                            d                              d          k    fd           t          j                            d                               d          k     fd           d S )Nc                  *    dj          d j          dS )Nz:Expected b and A to be on the same device, but found b on z
 and A on 	 instead.rw  r  r   s   r,   rQ   z(linearSolveCheckInputs.<locals>.<lambda>  s0    :{: :&'h: : : r.   c                  *    dj          d j          dS )Nz=Expected b and A to have the same dtype, but found b of type z and A of type r  rz   r  s   r,   rQ   z(linearSolveCheckInputs.<locals>.<lambda>  s0    =z= =*+'= = = r.   r   r  c                  b    d                      d           d                      d           dS )Nz3A must be batches of square matrices, but they are r  r  r   r  r   r  s   r,   rQ   z(linearSolveCheckInputs.<locals>.<lambda>  sF    BFF2JJB B,-FF2JJB B B r.   c                      d d                      d           d                      d           d                     d           d                     d           
S )NzIncompatible matrix sizes for z: each A matrix is r   r  z but each b matrix is r  r   )r  r  r   s   r,   rQ   z(linearSolveCheckInputs.<locals>.<lambda>  s    HT H HH H)*H H%)YYr]]H H8<		"H H r.   )rB   rS   rm   rJ   r   )r   r  r  s   ```r,   linearSolveCheckInputsr    s   	Lqx	
 	
 	
 	
 	
   
L
ag	
 	
 	
 	
 	
   
L	r

affRjj 	
 	
 	
 	
   
L	r

diimm#	
 	
 	
 	
 	
 	
    r.   tallow_low_precision_dtypesc                 ,   | j         t          j        |                                 p|                                 fd           |sIt          j        t          j        t          j        t          j        t          j        fv fd           d S d S )Nc                       d  S )Nz<: Expected a floating point or complex tensor as input. Got r2   rJ   r  s   r,   rQ   z(checkFloatingOrComplex.<locals>.<lambda>  s    6^^W\^^ r.   c                       d  S )Nz*: Low precision dtypes not supported. Got r2   r  s   r,   rQ   z(checkFloatingOrComplex.<locals>.<lambda>  s    vPPPP r.   )	rJ   rB   rS   r   r   rF   rH   rE   rG   )r  r  r  rJ   s    ` @r,   r   r     s    
 GE	L	/^^^^^   & 
ek5<u}MMPPPPP	
 	
 	
 	
 	

 
r.   arg_namec                 h    t          j        |                                 dk    fd           d S )Nr   c                       d  dS )Nz: The input tensor z! must have at least 2 dimensions.r2   )r  r  s   r,   rQ   zcheckIsMatrix.<locals>.<lambda>%  s    6YYhYYY r.   )rB   rS   rp   )r  r  r  s    ``r,   checkIsMatrixr  "  s@    	L	1YYYYY    r.   Br   c                 4    t                      t                     t          j        r,                     d                              d          k    n+                     d                              d          k     fd           d S )Nr  r   c                       drdnd d                      d           d                      d           d                     d           d                     d           d	S )
Nz2: Incompatible shapes of A and B for the equation zAX = BzXA = Bz (r  r9   r   r   rd   r   )r  r  r  r   s   r,   rQ   z#checkInputsSolver.<locals>.<lambda>.  s     J J-xxXJ JJ JffRjjJ J/0vvbzzJ J<=FF2JJJ J J r.   )r   r  rB   rS   r   )r  r  r   r  s   ````r,   checkInputsSolverr  )  s    a   !V	L$(Fr

affRjj  affRjjAFF2JJ.F	
 	
 	
 	
 	
 	
 	
    r.   r4  fn_nameresult_namec                 `     t          j        j        j        k     fd           d S )Nc            	      8      d d dj          dj          	S )Nz: Expected z5 and input tensors to be on the same device, but got z on z and input on rw  )r  r   r4  r  s   r,   rQ   z!checkSameDevice.<locals>.<lambda>>  sV     L L; L LL L &L L=B\L L r.   )rB   rS   rm   )r  r4  r   r  s   ````r,   checkSameDevicer  6  sU     
L%	
 	
 	
 	
 	
 	
 	
    r.   UPLOc                                                        }t          j        t                     dk    o|dk    p|dk     fd           d S )Nr   ULc                      d  S )Nz1Expected UPLO argument to be 'L' or 'U', but got r2   )r  s   r,   rQ   zcheckUplo.<locals>.<lambda>I  s    JDJJ r.   )upperrB   rS   r   )r  UPLO_uppercases   ` r,   	checkUplor  E  s[    ZZ\\N	LD		QKNc1J^s5JJJJJ    r.   eigenvalueseigenvectorsr  	compute_vc                    t          | d           t          |           t          | j                  }|r;|                     |          }|                    |t          |d                     n|                     dg          }|                                 |                     |t          | j	                            }||fS )Nzlinalg.eighF	row_majorr   rz   )
r   r  r   r   r   r  r   poprL   rJ   )r  r  r  r   vecsvalss         r,   meta__linalg_eighr  M  s     a'''dOOOMME  {{5!! ;EU S S STTTT{{A3	IIKKK;;uOAG$<$<;==D:r.   c                     t          | d           t          j        | j                  r| j        nt          j        | j                  }|                     | j        d d         |          S )Nzlinalg.eigvalsr   rz   r   r<   rs   rJ   r   r   r   )r   complex_dtypes     r,   meta__linalg_eigvalsr  `  sj     e-... !%+..	<.u{;; 
 ??5;ss+=?AAAr.   c                    t          | d           t          j        | j                  r| j        nt          j        | j                  }|                     | j        d d         |          }|                     | j        |          }||fS )Nz
linalg.eigr   rz   r  )r   r  r   vectorss       r,   meta_linalg_eigr  l  s     e\*** !%+..	<.u{;; 
 __U["-]_CCFooeko??G7?r.   r   c                 t    | j                             t          j                                      dd          S )Nr   r  r   )mTclonerB   r   	transpose)r   s    r,   cloneBatchedColumnMajorr  z  s+    6<<e&=<>>HHRPPPr.   r  c                      t          |           S r)   )r  )r   r  r  s      r,   _cholesky_solve_helperr  ~  s     #4(((r.   c                      t          j         j        dk     fd           t          j        j        dk    fd           t           d          \  }}t	          |||          S )Nr   c                      d j          dS )Nz-b should have at least 2 dimensions, but has  dimensions insteadr   r   s   r,   rQ   z cholesky_solve.<locals>.<lambda>  s    ^	^^^ r.   c                      d j          dS )Nz-u should have at least 2 dimensions, but has r  r  r  s   r,   rQ   z cholesky_solve.<locals>.<lambda>  s    [[[[ r.   cholesky_solve)rB   rS   r   !_linalg_broadcast_batch_dims_namer  )r   r  r  self_broadcastedA_broadcasteds   ``   r,   r  r    s     
L	Q^^^^   
L	![[[[   'Ha!' '#m ""2M5IIIr.   c                     |                                  dk    r t          j        | t          j                  S t	          | d           t          |           S )Nr   r   cholesky)r   rB   r   legacy_contiguous_formatr   r  r   r  s     r,   r  r    sN     zz||qE4RSSSSdJ'''"4(((r.   c                 @    t          | d           t          |           S )Ncholesky_inverse)r   r  r  s     r,   r  r    s#     d.///"4(((r.   check_errorsc                 D   t          | d           t          | d           | j        }t          |          }t	          |d          }|                     |          }|                    ||           |                     |d|dz
           t          j                  }||fS )Nzlinalg.choleskyFr   r   rz   )	r   r   r   r   r   r   r  rB   r  )r  r  r  A_shaper   	L_stridesr  infoss           r,   linalg_cholesky_exr    s    a*+++1/000gGw<<D ,GU;;I	GAMM'9%%% KKD1H-U[KAAEe8Or.   tauc                     t          j         j        dk    d            t          j                             d                               d          k    d            t          j                             d                              d          k    d            t          j         j        j        z
  dk     fd            j        dk    r: j        d d         }j        d d         t          j        |k    fd	           t          j        j         j        k     fd
           t          d d           t          j         j        t           j        d           j         j	                  S )Nr   c                      dS )NzHtorch.linalg.householder_product: input must have at least 2 dimensions.r2   r2   r.   r,   rQ   z,linalg_householder_product.<locals>.<lambda>      Z r.   r  r   c                      dS )Nzbtorch.linalg.householder_product: input.shape[-2] must be greater than or equal to input.shape[-1]r2   r2   r.   r,   rQ   z,linalg_householder_product.<locals>.<lambda>  s    t r.   c                      dS )Nz`torch.linalg.householder_product: input.shape[-1] must be greater than or equal to tau.shape[-1]r2   r2   r.   r,   rQ   z,linalg_householder_product.<locals>.<lambda>  s    r r.   r   c                  (    dj          d j          S )Nzptorch.linalg.householder_product: Expected tau to have one dimension less than input, but got tau.ndim equal to  and input.ndim is equal to r  r   r  s   r,   rQ   z,linalg_householder_product.<locals>.<lambda>  0    \),\ \OTz\ \ r.   c                      d  S )Nzltorch.linalg.householder_product: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r2   actual_batch_tau_shapes   r,   rQ   z,linalg_householder_product.<locals>.<lambda>      O6LO O r.   c                  (    dj          d j          S )Nz,torch.linalg.householder_product: tau dtype z does not match input dtype rz   r  s   r,   rQ   z,linalg_householder_product.<locals>.<lambda>  s*    939 9 9+0;9 9 r.   z torch.linalg.householder_productr  Fr  r   r   rJ   rm   )
rB   rS   r   r   r   rJ   r  empty_stridedr   rm   )r   r  expected_batch_tau_shaper  s   `` @r,   linalg_householder_productr    s   
 
L
aZZ   
L

2%**R..(tt   
L

2#((2,,&rr  
 
L
SX"	
 	
 	
 	
 	
   zA~~#(;ss#3 !$3B3"&>>   	
 	
 	
 
L	U[ 	
 	
 	
 	
 	
   6UEJJJ[*5;%HHHk|	   r.   c                 <   t          | d           t          | dd           |                     | j                  }|                    | j        t          | j        d                     |                     | j        d d         t          j                  }||fS )Nzlinalg.inv_exF)r  r  r  rz   r   r   r   r   r  r   rB   r  )r  r  r  r  s       r,   linalg_inv_ex_metar    s    a)))1o%PPPP	AGAMM!'6qw%PPPQQQKKEKK88Ee8Or.   LDpivotsinfo)	hermitianr  r  c                z   t          | d           t          | d           t          j        | j        t          | j        d          | j        | j                  }|                     | j        d d         t          j	                  }|                     | j        d d         t          j	                  }|||fS )Nztorch.linalg.ldl_factor_exFr  r  r   rz   r  )
r   r   rB   r  r   r   rJ   rm   r   int)r   r  r  r  r  r  s         r,   linalg_ldl_factor_ex_metar    s     d89994!=>>>		Z*4:GGGj{	
 
 
B ^^DJssO59^==F>>$*SbS/>;;Dvtr.   )r  c                @    t           d           t           d           t           d           t          j        j        dk    fd            j        d d         }t          j        |j        k    fd           t          j        t          j        j	                  fd           t          j         j	        j	        k     fd           t                     \  }}t          j        |t          |d	          j	        j        
          S )Nztorch.linalg.ldl_solver   c                      d j          dS )NzMtorch.linalg.ldl_solve: Expected B to have at least 2 dimensions, but it has r  r  )r  s   r,   rQ   z'linalg_ldl_solve_meta.<locals>.<lambda>  !    6&6 6 6 r.   r   c                      d j          dS )Nzjtorch.linalg.ldl_solve: Expected LD.shape[:-1] and pivots.shape to be the same, but got pivots with shape  insteadr   r  s   r,   rQ   z'linalg_ldl_solve_meta.<locals>.<lambda>%  %    @)/@ @ @ r.   c                      d j          S )Nz<torch.linalg.ldl_solve: Expected pivots to be integers. Got rz   r  s   r,   rQ   z'linalg_ldl_solve_meta.<locals>.<lambda>,  s    ]v|]] r.   c                  (    dj          d j          S )Nz!torch.linalg.ldl_solve: LD dtype z does not match b dtype rz   )r  r  s   r,   rQ   z'linalg_ldl_solve_meta.<locals>.<lambda>0  s    _BH__VWV]__ r.   Fr  r  )r   r   r  rB   rS   r   r   r<   is_integer_dtyperJ   _linalg_broadcast_batch_dimsr  r   rm   )r  r  r  r  expected_pivots_shapeB_broadcast_sizer?   s   ```    r,   linalg_ldl_solve_metar    sc    b2333278881b":;;;	L	!	
 	
 	
 	
   HSbSM	L-	
 	
 	
 	
   
Lv|,,]]]]   
L
AG_____   7q"==a*+;uMMMgx	   r.   Pr  )pivotr  c                    t          j         j        dk     fd           t           j                  }|d         }|d         }t          ||          }||d<   |r                     |          }n                     dg          }||d<                        |          }||d<   ||d<                        |          }|||fS )Nr   c                      d j          dS )Nz@linalg.lu: Expected tensor with 2 or more dimensions. Got size: r  r  r  s   r,   rQ   z linalg_lu_meta.<locals>.<lambda>@  s    dSTSZddd r.   r  r   r   )rB   rS   r   r   r   r}  r   )	r  r  sizesr2  r   r1  r  r  r  s	   `        r,   linalg_lu_metar  ;  s     
L	!dddd  
 MMEb	Ab	AAq		AE"I KKKKE"I	EAE"IE"I	EAa7Nr.   LU)r  r  c                    t          j         j        dk     fd           t           j                  }|d         }|d         }t          j        |t          |d           j         j                  }|	                                 t          ||          |d<                        |t           j                  }|	                                                      |t           j                  }|||fS )	Nr   c                      d j          dS )NzFtorch.lu_factor: Expected tensor with 2 or more dimensions. Got size: r  r  r  s   r,   rQ   z*linalg_lu_factor_ex_meta.<locals>.<lambda>a  s    jYZY`jjj r.   r  r   Fr  r  rz   )rB   rS   r   r   r   r  r   rJ   rm   r  r}  r   r   )	r  r  r  r  r2  r   r  r  r  s	   `        r,   linalg_lu_factor_ex_metar  W  s     
L	!jjjj  
 MMEb	Ab	A		*5EBBBgx	
 
 
B 
IIKKKAq		E"I[[ei[00F 
IIKKK;;uEI;..Dvtr.   )r   adjointr  c                    t           d           t          j         j        j        k     fd           t          j        j        t          j        k    d            t           d           t           |d           t          j                             d                              d          k    d            t          j         j        d d         j        k    fd           t                     \  }}t          j
        |t          ||           j        j        	          }|                                d
k    r*|s(|                                r|                                }|S )Nztorch.linalg.lu_solvec                  *    dj          d j          dS )NzPlinalg.lu_solve: Expected LU and B to have the same dtype, but found LU of type  and B of type r  rz   )r  r  s   r,   rQ   z&linalg_lu_solve_meta.<locals>.<lambda>  s6    O$&HO O=>WO O O r.   c                      dS )NzElinalg.lu_solve: pivots should be a Tensor of scalar type torch.int32r2   r2   r.   r,   rQ   z&linalg_lu_solve_meta.<locals>.<lambda>  s    W r.   zlinalg.lu_solver   c                      dS )NzYlinalg.lu_solve: Number of pivots per batch should be same as the dimension of the matrixr2   r2   r.   r,   rQ   z&linalg_lu_solve_meta.<locals>.<lambda>  s    k r.   c                      d j          dS )Nzclinalg.lu_solve: Expected LU.shape[:-1] and pivots.shape to be the same, but got pivots with shape r  r  r  s   r,   rQ   z&linalg_lu_solve_meta.<locals>.<lambda>  r	  r.   r  r  r   )r   rB   rS   rJ   r   r   r  r   r   r  r  r   rm   r   r   conj)r  r  r  r   r  r  r?   r4  s   ```     r,   linalg_lu_solve_metar#  {  s    26777	L
AG	
 	
 	
 	
 	
   
L	!WW   b1222b!T#4555	L
v{{2&kk   
L
"%	
 	
 	
 	
   7q"==a *+;4xPPPgx	  F ||~~4 	#[[]]FMr.   unpack_dataunpack_pivotsc                 D    t          j         j        dk     fd           |r)t          j        |j        t           j        k    d            t           j                  }|d         }|d         }t          ||          }||d<   |r                     |          }n                     dg          }|r:||d<                        |          }	||d<   ||d<                        |          }
n,                     dg          }	                     dg          }
||	|
fS )Nr   c                      d j          dS )NzFtorch.lu_unpack: Expected tensor with 2 or more dimensions. Got size: r  r  )r  s   r,   rQ   z lu_unpack_meta.<locals>.<lambda>  s    kY[Yakkk r.   c                      dS )Nztorch.lu_unpack: LU_pivots is expected to be a contiguous tensor of torch.int32 dtype.
Note: this function is intended to be used with the output produced by torch.linalg.lu_factorr2   r2   r.   r,   rQ   z lu_unpack_meta.<locals>.<lambda>  s    p r.   r  r   r   )	rB   rS   r   rJ   r  r   r   r}  r   )r  r  r$  r%  r  r2  r   r1  r  r  r  s   `          r,   lu_unpack_metar)    s?    
L
1kkkk    
LEK' 	
 	
 	
 NNEb	Ab	AAq		AE"I LLLL! b	LLb	b	LLLL!LL!a7Nr.   modec                 ~      dk    rd}d}n. dk    rd}d}n# dk    rd}d}nt          j        d fd           ||fS )NreducedTcompleteFrc                      d  dS )Nzqr received unrecognized mode 'z=' but expected one of 'reduced' (default), 'r', or 'complete'r2   )r*  s   r,   rQ   z _parse_qr_mode.<locals>.<lambda>  s"    O$ O O O r.   rB   rS   )r*  	compute_qr,  s   `  r,   _parse_qr_moder2    s~    y							   	
 	
 	
 gr.   QRr,  c                 T   t          | d           t          | d           t          |          \  }}| j        d         }| j        d         }t	          ||          }|rXt          | j                  }|r|n||d<   |                     |          }|                    |t          |d                     n|                     dg          }t          | j                  }	|s|s|n||	d<   |                     |	          }
|
                    |	t          |	d                     ||
fS )Nz	linalg.qrr  r   Fr  r   )	r  r   r2  r   r}  r   r   r  r   )r  r*  r1  reduced_moder2  r   r1  Q_shaper3  R_shaper4  s              r,   linalg_qr_metar9    s+    ![!!!1k***,T22I|	A	AAq		A qw--'.aaQKK  	g:7eTTTUUUUKK 17mmG#;9;!!!GBK	GAMM'6w%PPPQQQa4Kr.   sign	logabsdetc                    t          | d           t          | dd           | j        }|                     |d d                   }|                     |d d         t	          | j                            }t          j        |t          |d          | j        | j	                  }|                     |d d         t          j
                  }||||fS )Nzlinalg.slogdetFr  rz   r  r   )r   r   r   r   rL   rJ   rB   r  r   rm   r  )r  r   r:  r;  r  r  s         r,   _linalg_slogdetr=    s     a)***1.666GE;;uSbSz""DE#2#Joag.F.FGGI		*5%88gx	
 
 
B [[ss5;[77FB&&r.   full_matrices
compute_uvdriverc                    t          | d           t          | d           t          | j        d d                   }| j        d         }| j        d         }t	          ||          }|r|||r|n|gz   }|                     |          }	|	                    |t          |d                     ||r|n||gz   }
|                     |
          }t          |           dk    }|                    |
t          |
|                     n,|                     dg          }	|                     dg          }|                     ||gz   t          | j
                            }|	||fS )	Nz
linalg.svdr  r   Fr  cudar   rz   )r  r   r   r   r}  r   r  r   device_hintrL   rJ   )r  r>  r?  r@  r   r2  r   r1  U_shaper  V_shapeVis_cudaSs                 r,   _linalg_svd_metarI  #  sl    !\"""1l+++agcrcl##J	A	AAq		A #<111==KK  	g:7eTTTUUU] 91==KK  
 a..F*	g:7gVVVWWWW KKKK 	
J!$OAG,D,DEEAa7Nr.   arg1arg2c                 V   | j         d d         }|j         d d         }t          ||          }t          |          }||                     d          |                     d          gz  }t          |          }||                    d          |                    d          gz  }||fS )Nr  r   )r   r    r   r   )rJ  rK  arg1_batch_sizesarg2_batch_sizesexpand_batch_portionarg1_expand_sizearg2_expand_sizes          r,   r  r  I  s    
 z#2#z#2#,-=?OPP0112		"660112		"66---r.   c                     |rt          | ||           t          | |          \  }}|| j        k    r| n|                     |          }||j        k    r|n|                    |          }||fS r)   )r  r  r   expand)rJ  rK  r  rP  rQ  arg1_broadcastedarg2_broadcasteds          r,   r  r  Z  s      1tT4000)EdD)Q)Q&& !DJ..DKK@P4Q4Q  !DJ..DKK@P4Q4Q  ---r.   r   c                 v    | j         d d         }|j        dk    p| j        dz
  |j        k    o
|j         |k    }|S )Nr   r   )r   r   )r   r   expected_batched_rhs_shapevector_cases       r,   linalg_solve_is_vector_rhsrY  n  sK    !&SbS!1*/ 
Q%*$R8R)R  r.   )r   r  r4  r  r  r  c                    t           d           t          j         j        j        k     fd           t	                     }|r                    d          n}	t           |	|d           t          |	           \  }
}t          j        |p| d            |r
|
d d         n|
}t          j        |t          ||           j        j
                  } j        } j        }t          j        |t          |d           j         j
                  }                     |d d         t          j                  }                     |d d         t          j                  }||||f}||||f}t          d	 |D                       rjt!          ||          D ]Y\  }}t#          ||j                   |                    |j        |                                           t)          ||d
           Z|S )Nzlinalg.solvec                  *    d j          dj          dS )NzKlinalg.solve: Expected A and B to have the same dtype, but found A of type r  r  rz   )r  r  s   r,   rQ   z"_linalg_solve_ex.<locals>.<lambda>  s0    9w9 9'(w9 9 9 r.   r   c                      dS )Nzlinalg.solve: Vector broadcasting of the left hand side is not supported for left=False. In this case linalg.solve is equivalent to B / A.squeeze(-1)r2   r2   r.   r,   rQ   z"_linalg_solve_ex.<locals>.<lambda>  s    K r.   r  Frz   r  c              3      K   | ]}|d uV  	d S r)   r2   )r8   r9   s     r,   r_   z#_linalg_solve_ex.<locals>.<genexpr>  s&      
&
&Q1D=
&
&
&
&
&
&r.   )	copy_fromcopy_toexact_dtype)r   rB   rS   rJ   rY  	unsqueezer  r  r  r   rm   r   r   r   r  allzipr   r  r   r   )r  r  r   r  r4  r  r  r  rX  B_B_broad_shaper?   result_shaperesult_r   r   LU_pivots_info_r   resr.  os   ``                     r,   _linalg_solve_exrm  v  s<    1n---	L	17	
 	
 	
 	
 	
   -Q22K'	.RQBaT>2223B::M1	LK	
 	
   *5G="%%-L!*<TBBgx	  G GE6D

*5%88gx	  C kk%*EKk88GKKcrc
%+K66E2vt
$CC%
(C

&
&#
&
&
&&& FSMM 	F 	FDAqa)))MM!'188::...QuEEEEEJr.   )r   unitriangularr   rn  r   c                
   ||                      dg          }t          |t                    sJ t          | ||d           t	          || d           \  }}|                    dd                                          o|                                }|rt          ||j	                  }nYt          ||j	                  rD|                    |                    dd          j	                   |                    dd           |S )Nr   zlinalg.solve_triangularr  r   )r   r\   r   r  r  r  is_contiguousis_conjr   r   r   resize_
transpose_)	r  r  r  r   rn  r   rd  A_avoid_copy_As	            r,   linalg_solve_triangular_metarv    s     {kk1#c:&&&&&aD";<<<.q!T::FB<<B''5577HBJJLLL #RX..  RX.. 	#KKR,,2333NN2r"""Jr.   solutioncloned_coefficientr  c                     t          j         j        dk     fd           t          j        j        dk    fd           t           d           j        t           j        k    rvt                     \  }}t          j        |t          |d           j	         j
                  }t          j        |t          |d          j	        j
                  }nkj        t           j        k    sj        t           j        k    r+t          j                   }                     dg          }nt          j        dd	            ||fS )
Nr   c                      d j          dS )NzMtorch.triangular_solve: Expected b to have at least 2 dimensions, but it has r  r  r   s   r,   rQ   z'triangular_solve_meta.<locals>.<lambda>  s!    9)9 9 9 r.   c                      d j          dS )NzMtorch.triangular_solve: Expected A to have at least 2 dimensions, but it has r  r  r  s   r,   rQ   z'triangular_solve_meta.<locals>.<lambda>  r  r.   triangular_solveFr  r  r   c                      dS )Nz+triangular_solve: Got an unexpected layout.r2   r2   r.   r,   rQ   z'triangular_solve_meta.<locals>.<lambda>  s    $Q r.   )rB   rS   r   r  rl   stridedr  r  r   rJ   rm   
sparse_csr
sparse_bsrr   r   )	r   r  r  r  rn  self_broadcast_sizeA_broadcast_sizerw  rx  s	   ``       r,   triangular_solve_metar    s    
L	Q	
 	
 	
 	
   
L	!	
 	
 	
 	
   4$6777x5=  0LTST0U0U--&$./BeTTT*;	
 
 
 #0!./?5QQQ'8	
 
 
 
U%	%	%U5E)E)E#D))!^^QC00UQQRRR'''r.   c                 ~   t          | d           t          | d           |                     | j        d d                   }|                     | j                  }|                    | j        t          | j        d                     |                     | j        d d         t          j                  }|||fS )Nz
linalg.detr  Fr  r   rz   r  )r  detr  r  s       r,   _linalg_det_metar    s    a&&&1l+++
++agcrcl
#
#C	
QW		BNN1775QQQRRR[["U[[99FF?r.   c                     t          j         j        dk    d            t          j        j        dk    d            |rdndt          j        j                 j        d         k    fd           t          j        j                  j        d         k    fd           t          j        j        d          j        d         k    d            t          j         j        j        z
  d	k     fd
           t          j         j        j        k     fd            j        dk    re j        d d         }j        d d         t          j        |k    fd           j        d d         t          j        |k    fd           t          j        j         j        k     fd           t          j        j         j        k     fd           t          d d           t          d d           t          j        j        t          j        d          j        j                  S )Nr   c                      dS )Nz3torch.ormqr: input must have at least 2 dimensions.r2   r2   r.   r,   rQ   zormqr.<locals>.<lambda>      !V r.   c                      dS )Nz3torch.ormqr: other must have at least 2 dimensions.r2   r2   r.   r,   rQ   zormqr.<locals>.<lambda>  r  r.   r  r   c                      d  dS )Ntorch.ormqr: other.shape[z0] must be greater than or equal to tau.shape[-1]r2   left_size_conditions   r,   rQ   zormqr.<locals>.<lambda>  s    q,?qqq r.   c                      d  dS )Nr  z"] must be equal to input.shape[-2]r2   r  s   r,   rQ   zormqr.<locals>.<lambda>"  s    c,?ccc r.   c                      dS )NzHtorch.ormqr: tau.shape[-1] must be less than or equal to input.shape[-1]r2   r2   r.   r,   rQ   zormqr.<locals>.<lambda>'  r  r.   r   c                  (    dj          d j          S )Nz[torch.ormqr: Expected tau to have one dimension less than input, but got tau.ndim equal to r  r  r  s   r,   rQ   zormqr.<locals>.<lambda>,  r  r.   c                  (    dj          d j          S )Nzhtorch.ormqr: Expected other to have the same number of dimensions as input, but got other.ndim equal to r  r  r   r   s   r,   rQ   zormqr.<locals>.<lambda>3  s1    `+0:` `SXS]` ` r.   c                      d  S )NzWtorch.ormqr: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r2   r  s   r,   rQ   zormqr.<locals>.<lambda>>  r  r.   c                      d  S )NzYtorch.ormqr: Expected batch dimensions of other to be equal to input.shape[:-2], but got r2   )actual_batch_other_shapes   r,   rQ   zormqr.<locals>.<lambda>G  s    Q6NQ Q r.   c                  (    d j          dj          S )NzPtorch.ormqr: Expected input and tau to have the same dtype, but input has dtype z and tau has dtype rz   r  s   r,   rQ   zormqr.<locals>.<lambda>O  s0    O#(;O OCF9O O r.   c                  (    d j          dj          S )NzRtorch.ormqr: Expected input and other to have the same dtype, but input has dtype z and other has dtype rz   r  s   r,   rQ   zormqr.<locals>.<lambda>V  s0    S#(;S SEJ[S S r.   ztorch.ormqrr  r   Fr  r  )	rB   rS   r   r   rJ   r  r  r   rm   )	r   r  r   r   r  expected_batch_shaper  r  r  s	   ```   @@@r,   ormqrr    s    
L
aVV   
L
aVV   !%,"""	L'(CIbM9qqqq   
L'(EKO;cccc  
 
L	"R(ZZ  
 
L
SX"	
 	
 	
 	
 	
   
L
ej 	
 	
 	
 	
 	
   zA~~${3B3/!$3B3"&::   	
 	
 	
 $);ss#3 $(<<   	
 	
 	
 
L	U[ 	
 	
 	
 	
 	
   
Lu{"	
 	
 	
 	
 	
   M3u555M5%999[*5;%HHHk|	   r.   c                    t          j        t                    dz  k    fd            j        }|dz   k    }|}| }|r/t	          d|          D ]}|o                     |          dk    }n.t	          d|          D ]}|o                     |          dk    }t          j        |p| fd           d S )Nr   c                  4    dd z   dt                     S )Nzpadding size is expected to be r   z, but got: r   )rp   paddings   r,   rQ   z,_padding_check_valid_input.<locals>.<lambda>j  s"    T!c'TTc'llTT r.   r   r   c                  0    d dz    d dz    dj          S )Nz	Expected r   zD or r   zcD (batch mode) tensor with possibly 0 batch size and other non-zero dimensions for input, but got: r  )rp   r   s   r,   rQ   z,_padding_check_valid_input.<locals>.<lambda>  sC    Oa O OcAg O OAFO O r.   )rB   rS   r   r   r   r   )r   r  rp   	input_dimis_batch_modevalid_batch_modevalid_non_batch_moder   s   ```     r,   _padding_check_valid_inputr  g  s#   	LGCTTTTT  
 
I#'*M$,, Oq)$$ 	G 	GA/FEJJqMMQ4F	G q)$$ 	O 	OA#7#NEJJqMMQ<N   
L00	
 	
 	
 	
 	
    r.   c                   	
 d}dd} j         dk    r                     d          }dz  |dz  }t           |d           |\  	
                     |          }                               	z   
z   |r%t          j        	k     o
k      	
fd           t          j        dk    fd            j         dk    r                     |f          S                      ||f          S )Nr   r   r   r   c                  *    d d d  dj          S NzcArgument #4: Padding size should be less than the corresponding input dimension, but got: padding (rc   ) at dimension 
 of input r  dim_wr   pad_lpad_rs   r,   rQ   z_pad1d_common.<locals>.<lambda>  M    c%*c c.3c cDIc cUZU`c c r.   c                      d  d S )Nz
input (W: z%) is too small. Calculated output W: r2   )input_woutput_ws   r,   rQ   z_pad1d_common.<locals>.<lambda>  s    UWUU8UU r.   r   )r   r   r  rB   rS   r   )r   r  is_reflection	dim_planenbatchnplaner  r  r  r  r  s   `     @@@@@r,   _pad1d_commonr    sI   IEFzQA
Q	ug15555LE5ZZ	""FjjG&H 
GO/      	
 	
 	
 
LAUUUUU  
 zQ12229:::r.   c                 &    t          | |d          S NTr  r  r   r  s     r,   meta_reflection_pad1dr         t<<<<r.   c                 &    t          | |d          S NFr  r  r  s     r,   meta_replication_pad1dr         u====r.   c                    d|s't          j        t          |          dk    d            j        dk    rdz  |\                                }|z   z   |r%t          j        |k     o|k     fd           t          j                                       k     fd                               j                  S )Nr   r   c                      dS )Nz padding size is expected to be 2r2   r2   r.   r,   rQ   z(_pad1d_backward_common.<locals>.<lambda>  s    0R r.   r   c                  *    d d d  dj          S r  r  r  s   r,   rQ   z(_pad1d_backward_common.<locals>.<lambda>  r  r.   c                  :    d d                                 S Nz(grad_output width unexpected. Expected: , Got: r   r  grad_outputr  s   r,   rQ   z(_pad1d_backward_common.<locals>.<lambda>  '    e8eeKL\L\]bLcLcee r.   rB   rS   r   r   r   r   r   )	r  r   r  r  r  r  r  r  r  s	   ``   @@@@r,   _pad1d_backward_commonr    s   E TS\\Q&(R(RSSSzQ
LE5jjG&H 
GO/      	
 	
 	
 
LK$$U+++eeeeee  
 ??5;'''r.   
grad_inputc                 (    t          | ||d          S r  r  r  r   r  s      r,   meta_reflection_pad1d_backwardr    s     "+ugTRRRRr.   c                 (    t          | ||d          S r  r  r  s      r,   meta_replication_pad1d_backwardr    s     "+ugUSSSSr.   c                   	
 ddd}d}t           |d            j        }|dk    r$                     d          }dz  dz  |dz  }|\                       |          }                               	                               
	z   z   
z   z   |rJt          j        
k     o
k      fd           t          j        	k     o	k      fd           t          j        dk    pdk    	
fd            j        d	k    r                     |f          S                      ||f          S )
Nr   r   r   r      c                  *    d d d  dj          S r  r  r  s   r,   rQ   z_pad2d_common.<locals>.<lambda>  r  r.   c                  *    d d d  dj          S NzcArgument #6: Padding size should be less than the corresponding input dimension, but got: padding (rc   r  r  r  dim_hr   pad_bpad_ts   r,   rQ   z_pad2d_common.<locals>.<lambda>  r  r.   c                       d  d d d S )Nz
input (H:  W: z%) is too small. Calculated output H: r2   )input_hr  output_hr  s   r,   rQ   z_pad2d_common.<locals>.<lambda>  s?    = = =g = =$,= =2:= = r.   r   r  r   r   rB   rS   r   )r   r  r  
dim_slicesr  r   r  r  r  r  r  r  r  r  r  r  r  s   `      @@@@@@@@@@r,   _pad2d_commonr    s   EEJFug15555:DqyyA

a
!(E5%ZZ
##FjjGjjG&H&H 
GO/      	
 	
 	
 	GO/      	
 	
 	
 
LA&Q	
 	
 	
 	
 	
 	
 	
   zQ(;<<<(CDDDr.   c                 &    t          | |d          S r  r  r  s     r,   meta_reflection_pad2dr    r  r.   c                 &    t          | |d          S r  r  r  s     r,   meta_replication_pad2dr    r  r.   c                     ddd}d}|j         }|                                dk    r|d         }dz  dz  |dz  }|\  }}}}	||         }
|         }|         }||z   |	z   ||z   |z   t          j                                       k     fd           t          j                                       k     fd           |                    |j                   S )Nr   r   r   r  c                  :    d d                                 S r  r   r  s   r,   rQ   z%meta_pad2d_backward.<locals>.<lambda>C  r  r.   c                  :    d d                                 S Nz)grad_output height unexpected. Expected: r  r   r  r  r  s   r,   rQ   z%meta_pad2d_backward.<locals>.<lambda>G  '    fHff[M]M]^cMdMdff r.   )r   rp   rB   rS   r   r   )r  r   r  r  r  rP   r  r  r  r  r  r  r  r  r  r  r  s   `            @@@@r,   meta_pad2d_backwardr  #  s1    EEIFJxxzzQA

Q	!(E5%	"FGG&H&H	LK$$U+++eeeeee   
LK$$U+++ffffff   >>$*%%%r.   c          	      R   	
 d	ddd}t           |d            j        dk    }|r)                     d          }	dz  	dz  dz  |dz  }|\                       |          }                               
                                                    	          
z   z   z   z   z   z   |rot          j        k     ok     	 fd           t          j        k     ok      fd           t          j        
k     o
k      fd	           t          j        dk    pdk    pdk    
fd
           |r                     ||f          S                      |f          S )Nr   r   r   r   r      c                  *    d d d  dj          S r  r  r  s   r,   rQ   z_pad3d_common.<locals>.<lambda>i  r  r.   c                  *    d d d  dj          S r  r  r  s   r,   rQ   z_pad3d_common.<locals>.<lambda>p  r  r.   c                  *    d d d  dj          S )NzcArgument #8: Padding size should be less than the corresponding input dimension, but got: padding (rc   r  r  r  )dim_dr   pad_bkpad_fs   r,   rQ   z_pad3d_common.<locals>.<lambda>w  sM    d%*d d.4d dEJd dV[Vad d r.   c                  ,    d  d d d d d S )Nz
input (D:  H: r  z%) is too small. Calculated output D: r2   )input_dr  r  output_dr  r  s   r,   rQ   z_pad3d_common.<locals>.<lambda>  se    K K Kg K K7 K K$,K K2:K K@HK K r.   r  )r   r  r  r  
batch_moder  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s   `      @@@@@@@@@@@@@@@r,   _pad3d_commonr  L  s   EEEIug15555qJ A


Q	07-E5%vZZ	""FjjGjjGjjG'H&H&H 
GO/      	
 	
 	
 	GO/      	
 	
 	
 	GO0 0      	
 	
 	
 
LA7Q7(a-	
 	
 	
 	
 	
 	
 	
 	
 	
    G(HMNNN(HEFFFr.   c                 &    t          | |d          S r  r  r  s     r,   meta_reflection_pad3dr     r  r.   c                 &    t          | |d          S r  r  r  s     r,   meta_replication_pad3dr    r  r.   c                     t          j        t          |          dk    d            |j        dk    sJ  j        |j        k    sJ ddd|j        dk    rdz  dz  dz  |\  }}}}}}|                              }	|                              }
|                              }|	|z   |z   |
|z   |z   ||z   |z   t          j                                       k     fd           t          j                                       k     fd           t          j                                       k     fd	           |                    |j                  S )
N   c                      dS )Nz padding size is expected to be 6r2   r2   r.   r,   rQ   z%meta_pad3d_backward.<locals>.<lambda>  s    ,N r.   r   r   r   r  c                  :    d d                                 S r  r   r  s   r,   rQ   z%meta_pad3d_backward.<locals>.<lambda>  r  r.   c                  :    d d                                 S r  r   r  s   r,   rQ   z%meta_pad3d_backward.<locals>.<lambda>  r  r.   c                  :    d d                                 S )Nz(grad_output depth unexpected. Expected: r  r   )r  r  r  s   r,   rQ   z%meta_pad3d_backward.<locals>.<lambda>  r  r.   r  )r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s   `           @@@@@@r,   meta_pad3d_backwardr	    s    
LW"$N$NOOO:>>>>uz))))EEEzQ


07-E5%vjjGjjGjjG'H&H&H	LK$$U+++eeeeee   
LK$$U+++ffffff   
LK$$U+++eeeeee  
 ??5;'''r.   r   pc                 j   t          j        |                                 d            |                     d          }|dk    r4|                     dg                              t           j                  S |                     ||dz
  z  dz  f                              t           j                  S )Nc                      dS )Nz(_pdist_forward requires contiguous inputr2   r2   r.   r,   rQ   z%meta__pdist_forward.<locals>.<lambda>  s    &P r.   r   r   r   r   )rB   rS   rp  r   r   r   r  )r   r
  r   s      r,   meta__pdist_forwardr    s     
LPP   			!AAvv~~qc""%%E4R%SSS~~qAE{a/122558 6 
 
 	
r.   gradpdistc                     t          j        |                                d            t          j        |                                d            t          j        |t           j                  S )Nc                      dS )Nz._pdist_backward requires self to be contiguousr2   r2   r.   r,   rQ   z&meta__pdist_backward.<locals>.<lambda>  s    &V r.   c                      dS )Nz/_pdist_backward requires pdist to be contiguousr2   r2   r.   r,   rQ   z&meta__pdist_backward.<locals>.<lambda>  s    'X r.   r   )rB   rS   rp  r   r  )r  r   r
  r  s       r,   meta__pdist_backwardr    ss     
LVV   
LXX   D0NOOOOr.   )r"  r!  c                   	
                      d          }                     d          }                     d          }                     |||f           t          j                                        dk    d            t          j                                        dk    d            t          j         j        j        cxk    o
j        k    nc  fd           j        }j        	|d         
|d         t          j        	d         
k    o	d         k    	
fd                                                                            S )	Nr   r   r   r   c                      dS Nzbatch1 must be a 3D tensorr2   r2   r.   r,   rQ   zmeta_baddbmm.<locals>.<lambda>      ,H r.   c                      dS Nzbatch2 must be a 3D tensorr2   r2   r.   r,   rQ   zmeta_baddbmm.<locals>.<lambda>  r  r.   c                  8    dj          d j          dj          S )Nz+Input dtypes must be the same, got: input: z
, batch1: z
, batch2: rz   )batch1batch2r   s   r,   rQ   zmeta_baddbmm.<locals>.<lambda>  s*    zdjzzTZT`zzlrlxzz r.   c            	      :    d d d d          d d          d	S Nz@Expected size for first two dimensions of batch2 tensor to be: [rc   z] but got: [r   r   ].r2   batch2_sizesbscontraction_sizes   r,   rQ   zmeta_baddbmm.<locals>.<lambda>  sY    [[ [&[ [4@O[ [GSTU[ [ [ r.   )r   rS  rB   rS   rp   rJ   r   r   )r   r  r  r"  r!  dim1dim2dim3batch1_sizesr!  r"  r#  s   ```      @@@r,   meta_baddbmmr(    sl    ;;q>>D;;q>>D;;q>>D;;dD)**D	L"$H$HIII	L"$H$HIII	L
fl2222fl2222zzzzzz   <L<L	aB#A	LQ2E,q/5E"E	
 	
 	
 	
 	
 	
   >>$))++&&&r.   c                N    t          j        |                                           S r)   rB   r   r   r   r   s     r,   meta_bernoullir,    s!     D!!,,...r.         ?c                     | S r)   r2   r   r
  r   s      r,   meta_bernoulli_r0        Kr.   c                 N    t          j        |                                           S r)   r*  r/  s      r,   meta_bernoulli_pr3  	  s!     D!!,,...r.   c                 *    t          j        |           S r)   rB   r   r+  s     r,   meta_poissonr6    s     D!!!r.   c                     t          j        |
|                                 k     d            t          j        | t           j                  }t          j        |           |fS )Nc                      dS )NzJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r2   r2   r.   r,   rQ   z6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>'      \ r.   rz   )rB   rS   rp   r   bool)r   observer_onfake_quant_onrunning_minrunning_maxscale
zero_pointaveraging_const	quant_min	quant_maxch_axisper_row_fake_quantsymmetric_quantmasks                 r,   $meta__fused_moving_avg_obs_fq_helperrH    s]      
L$((**\\   D
333DT""D))r.   c                 L   t          j        |                                 dk    d            t          j        |                                dk    d            | j        \  |j        \  t          j        k    fd           |                               S )Nr   c                      dS )Nza must be 2Dr2   r2   r.   r,   rQ   zmeta_mm.<locals>.<lambda>0      ~ r.   c                      dS )Nzb must be 2Dr2   r2   r.   r,   rQ   zmeta_mm.<locals>.<lambda>1  rK  r.   c            	      "    d d  d d d	S )Nz/a and b must have same reduction dim, but got [rc   z] X [r  r2   )M1M2Nr  s   r,   rQ   zmeta_mm.<locals>.<lambda>6  s-    [![[r[[PR[[VW[[[ r.   )rB   rS   rp   r   r   )abrN  rO  rP  r  s     @@@@r,   meta_mmrS  -  s     
LA55666	LA55666GEArGEB	L
b[[[[[[[   ;;q!r.   c                      |r.t           fdt           j                  D                       S t          j         j                  S )Nc              3   >   K   | ]}|vrj         |         nd V  dS )r   Nr  )r8   r   dimsr   s     r,   r_   z+_compute_reduction_shape.<locals>.<genexpr>=  s5      UUqatmmTZ]]UUUUUUr.   )rR   r   r   r<   compute_reduction_output_shaper   )r   rV  r[  s   `` r,   rY  rY  ;  sR     VUUUUUE$)DTDTUUUUUU/
DAAAr.   strc                 \    t          | t          j        j                  r| j        j        S dS )NrB  )r\   rB   _subclasses
FakeTensorfake_devicere   )r  s    r,   rC  rC  F  s+    &%+677 !&&vr.   input_tensorr   r  dilationis_transposedgroupsoutput_paddingc                    dt           dt           dt           dt           dt           dt           fd}dt           dt           dt           dt           dt           dt           dt           fd	}	|j        d
d          }
| j        d
d          }|r||j        d         z  }n;|j        d         }|j        d         |z  | j        d         k    rt          d          | j        d         |g}t          |t                    r|gt          |          z  }n,t          |          dk    r|d         gt          |          z  }t          |t                    r|gt          |          z  }n,t          |          dk    r|d         gt          |          z  }t          |t                    r|gt          |          z  }n,t          |          dk    r|d         gt          |          z  }d }|rXt          |t                    r|gt          |          z  }n/t          |          dk    r|d         gt          |          z  }n|}t          t          |                    D ]}|rH|                     |	||         ||         ||         |
|         ||         ||                              L|                     |||         ||         ||         |
|         ||                              |S )Nlnr
  r   r1  rs  r9  c                 6    | d|z  z   ||dz
  z  z
  dz
  |z  dz   S )a  
        Formula to apply to calculate the length of some dimension of the output

        See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
        Returns:
            The output length
        r   r   r2   )rc  r
  r   r1  rs  s        r,   _formulaz+calc_conv_nd_return_shape.<locals>._formulaW  s.     QU
Q!a%[(1,2Q66r.   r*   c                 <    | dz
  |z  d|z  z
  ||dz
  z  z   |z   dz   S )a  
        Formula to apply to calculate the length of some dimension of the output
        if transposed convolution is used.
        See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
            op: output padding in that dim

        Returns:
            The output length
        r   r   r2   )rc  r
  r   r1  rs  r*   s         r,   _formula_transposedz6calc_conv_nd_return_shape.<locals>._formula_transposedh  s2    " Q!|a!e#a1q5k1B6::r.   r   r   r   zInvalid channel dimensions)r   r   r   r\   r   r   r   r   )r]  r  r   r  r^  r_  r`  ra  re  rg  kernel_sizerV  out_channels	ret_shapeoutput_padding_listr   s                   r,   calc_conv_nd_return_shaperl  M  s.   7S 7S 7S 7S 7S 7S 7 7 7 7"; ; ; ; ; ; ;QT ; ; ; ;& ,qrr"Kabb!D =Q/|A<?V#|'9!'<<<;<<<#A&5I&'"" )CII%	V		)s4yy('7## +)c$ii'	W		1:,T*(G$$ -:D		)	X!		QK=3t99,/3 1ng.. 	1#1"2SYY">  A%%#1!#4"5D		"A"03t99   	##GAJQKN1I'* 	 	 	 	 a'!*hqk;q>6RS9UU    r.   c                 \    t           j                            |           t           j        k    S r)   rB   _prims_commonsuggest_memory_formatchannels_lasttens    r,   is_channels_lastrt    s!    44S99U=PPPr.   c	           
            fd}	t           ||||||r|nd           }
d}d}                     |          dk    rd|
|<                        |
          }|                     |	                      }|S )Nc                  j   t                     dk    r+t                     st                    rt          j        S nt                     rt          j        S                      t          j                  rt          j        S                      t          j                  rt          j        S d S NrB  r   )rC  rt  rB   rq  rp  r   preserve_format)r]  r  s   r,   pick_memory_formatz%meta_conv.<locals>.pick_memory_format  s    |$$..-- +1A&1I1I +**+  -- +**%%E4K%LL 	)**''e6K'LL 	)((	) 	)r.   r   r   r   )rl  r   r   r   )r]  r  r  r   r  r^  r_  ra  r`  ry  	shape_outinput_channels_dimoutput_channels_dimr   s   ``            r,   	meta_convr}    s    
) 
) 
) 
) 
) 
) *'1T	 	I +,,11)*	%&

 
 
+
+C
&&1133&
4
4CJr.   mkldnnc
           
          t          | ||||d|g           }
|                     |
          }t          j        }|                                 dk    rt          j        }|                    |          }|S )NFr  r   )rl  r   rB   rq  rp   channels_last_3dr   )r]  r  r  r  r   r^  r`  attrscalars	algorithmrz  r   out_memory_formats                r,   meta_mkldnn_convolution_defaultr    s{     .&&'8UFB
 
	 $$Y//!/"" % 6ff#4f55
r.   c                 f    |                      g | j        d d         |j        d         R           S Nr   r   r   r   )r]  r  r  r  r  r  s         r,   meta_linear_pointwise_defaultr    s8     %%&Q(:3B3(?&Qa&Q&QRRRr.   mklc                 f    |                      g | j        d d         |j        d         R           S r  r  )r]  packed_weightorig_weightr  r   s        r,   meta_mkl_linearr  
	  s>    ))@,$SbS)@;+<Q+?@@  r.   onednnc           
          t          | ||||	d|
d           }|t          j        t          j        fv sJ |                     ||          }|                    t          j                  }|S )NFrz   r   )rl  rB   r,  r.  r   r   rq  )r9   x_scalex_zpww_scalew_zpr  r   r  r^  r`  output_scaleoutput_zero_pointoutput_dtyper  r  r  rz  r   s                      r,   meta_qconv2d_pointwiser  	  sw    ( .	
 	
	 u~>>>>>kk)<k88ff5#6f77
r.   c                     t          | j                  }|j        d         |d<   |	t          j        t          j        fv sJ |                     ||	          }|S )Nr   r   rz   )r   r   rB   r,  r.  r   )r9   r  r  r  r  r  r  r  r  r  post_op_namepost_op_argspost_op_algorithmr3  r   s                  r,   meta_qlinear_pointwiser  7	  sU    " AG}}71:Ru~>>>>>kk,lk;;
r.   	quantizedr2   r   r   c                 4   t          | |||||          \  }}}|                                 dk    r|                     d          nd}	t          j        }
|                                 dk    r|||g}n|	|||g}t          j        || j        | j        |
          S Nr  r   r   rK  )#max_pool2d_checks_and_compute_shaperp   r   rB   rq  ru   rJ   rm   r   rh  r   r  r^  	ceil_modenInputPlaneoutputHeightoutputWidthr  r   r   s               r,   meta_quantized_max_pool2dr  S	  s     0;9
 
		
 $)99;;!#3#3B+99;;!{;DDK{CD{+<'	
 
 
 	
r.   c                      t          j                                         k    o j                 k     fd           d S )Nc                  n    d  d d dd                                  d dj                  z   S )NzExpected a tensor of dimension z and tensor.size[z] == rc   zbut got : dimension z] = rp   r   )rp   dim_sizer   r  s   r,   rQ   z check_dim_size.<locals>.<lambda>u	  sU    _#____W[___
f
f
f
f
ffl[cNd
f
fg r.   )rB   rS   rp   r   )r  rp   r  r   s   ````r,   check_dim_sizer  r	  so    	L

>X 6$ >	g 	g 	g 	g 	g 	g 	g    r.   c                 z   d } |d|          \  }}	t          j        t          |          dv d            t          |          dk    r||	}}
n3t          |          dk    r|d         |d         }}
n |d|          \  }
} |d|          \  }}t          j        |d u p|dk    d	            |                                 d
k    r|                     d          nd}|                     d          }|                     d          }|                     d          }t          ||||
d|          }t          ||	||d|          }t          j        |           }t          | ||	|
|||dd||||||           |                                 dk    r|||g}n||||g}t          j	        || j
        | j        |          S )Nc                      t          j        t          |          dv  fd           |d         }t          |          dk    r|n|d         }||fS )Nr   r   c                      d  dS )Nzavg_pool2d: 4 must either be a single int, or a tuple of two intsr2   r  s   r,   rQ   z1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>	      ]4]]] r.   r   r   rB   rS   r   r  rl  HWs   `   r,   unpackzmeta_avg_pool2d.<locals>.unpack	  `    HH]]]]	
 	
 	
 FSQAACF!tr.   rh  r   r   r   c                      dS NzOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr2   r2   r.   r,   rQ   z!meta_avg_pool2d.<locals>.<lambda>	      a r.   r   r   r   r  c                      dS Nzdivisor must be not zeror2   r2   r.   r,   rQ   z!meta_avg_pool2d.<locals>.<lambda>	      * r.   r  r  r  r   r   rK  )rB   rS   r   rp   r   pooling_output_shaper<   rp  pool2d_shape_checkru   rJ   rm   )r   rh  r   r  r  count_include_paddivisor_overrider  kHkWdHdWpadHpadWr  r  inputHeight
inputWidthr  r  r   r   s                         r,   meta_avg_pool2dr  z	  s      VM;//FB	LFy aa   6{{aRB	V		F1IB&))B	7++JD$	LD 9$4$9**  
  %yy{{a//UZZ^^^QF**R..K**R..KBJ'Rr1iPPL&z2tRINNK/66M



		  $ yy{{a\;7\;?;k|#	   r.   c                     t          | ||||||dd|	|
||||           |                                 }|	}t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           d S )Nr   r   r   )r  rp   r  )r   
gradOutputr  r  r  r  r  r  r  r  r  r  r  r  
mem_formatr   nOutputPlanes                    r,   avg_pool2d_backward_shape_checkr  	  s    " 



		  $ 99;;DL:tTAX|<<<:tTAX|<<<:tTAX{;;;;;r.   c                 j   t          j        t          |          dk    pt          |          dk    d            |d         }t          |          dk    r|n|d         }	t          j        t          |          dk    p%t          |          dk    pt          |          dk    d            t          |          dk    r|n|d         }
t          |          dk    r|	nt          |          dk    r|
n|d         }t          j        t          |          dk    pt          |          dk    d            |d         }t          |          dk    r|n|d         }t          j        |d u p|dk    d            |j        }|                                dk    r|d	         nd}|d
         }|d         }|d         }t          ||||
d|          }t          ||	||d|          }t          j        |          }t          || |||	|
|||||||||           t          j	        ||j
        |j        |          S )Nr   r   c                      dS )NzKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr2   r2   r.   r,   rQ   z*meta_avg_pool2d_backward.<locals>.<lambda>
  s    ] r.   r   c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool2d_backward.<locals>.<lambda>
  r  r.   c                      dS )NzGavg_pool2d: padding must either be a single int, or a tuple of two intsr2   r2   r.   r,   rQ   z*meta_avg_pool2d_backward.<locals>.<lambda>
  s    Y r.   c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool2d_backward.<locals>.<lambda>
  r  r.   r  r  r  r  r   rK  )rB   rS   r   r   rp   r  r<   rp  r  ru   rJ   rm   )gradOutput_r   rh  r   r  r  r  r  r  r  r  r  r  r  
input_sizer  r  r  r  r  r  r  s                         r,   meta_avg_pool2d_backwardr  	  sz    
LKA6[!1!1Q!6]]   
QB;1$$+a.B	LFq@CKK1,@Fq0@aa   6{{aVAYB6{{a3v;;!+;+;RRB	LG.S\\Q.YY   1:Dw<<1$$44'!*D	LD 9$4$9**  
 J$yy{{a//Z^^QFR.KR.KBJ'Rr1iPPL&z2tRINNK,U33J#



  $ ;k| 	   r.   c                    t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }	t          j        | pt          |          dv d            |s|n|d         }
|s|nt          |          dk    r|
n|d         }|s|	nt          |          dk    r|
n|d         }t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        | j        dv d	            t          j        | p|dk    d
            |                     d          }|                     d          }|                     d          }|                     d          }|                     d          }t          ||||
d|          }t          ||||d|          }t          ||	||d|          }t          | ||||	|
|||||ddd||||||dd           | j        dk    r|                     ||||f          S |                     |||||f          S )Nr   r   c                      dS NzFavg_pool3d: kernel_size must be a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z!meta_avg_pool3d.<locals>.<lambda>J
      X r.   r   r   r   c                      dS NzJavg_pool3d: stride must be omitted, a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z!meta_avg_pool3d.<locals>.<lambda>R
  r9  r.   c                      dS NzBavg_pool3d: padding must be a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z!meta_avg_pool3d.<locals>.<lambda>Z
      T r.   r  r  c                      dS Nz9non-empty 4D or 5D (batch mode) tensor expected for inputr2   r2   r.   r,   rQ   z!meta_avg_pool3d.<locals>.<lambda>b
      K r.   c                      dS r  r2   r2   r.   r,   rQ   z!meta_avg_pool3d.<locals>.<lambda>g
  r  r.   r  r  r  r   zavg_pool3d()T)check_input_sizer  )rB   rS   r   r   r   r  pool3d_shape_checkr   )r   rh  r   r  r  r  r  kTr  r  dTr  r  padTr  r  r  nslicesitimeiheightiwidthotimeoheightowidths                           r,   meta_avg_pool3dr  =
  s    
LKF"XX   
QB;1$$+a.B;1$$+a.B	L
+c&kkV+\\   	(vayB	Fc&kkQ&6&6F1IB	Fc&kkQ&6&6F1IB	LGTT   1:Dw<<1$$44'!*Dw<<1$$44'!*D	L
fKK  
 
L5 0A 5**  
 ZZ]]FjjnnGJJrNNEjjnnGZZ^^F D"aCCE"7Bb!YGGG!&"dB9EEF





			-   2 zQ@AAAHIIIr.   c                    t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }	t          |          dk    r|n|d         }
t          j        | pt          |          dv d            |s|n|d         }|s|	nt          |          dk    r|n|d         }|s|
nt          |          dk    r|n|d         }t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        |j        dv d	            t          j        | p|dk    d
            |                    d          }|                    d          }|                    d          }|                    d          }t          ||||d|          }t          ||	||d|          }t          ||
||d|          }t          || |||	|
||||||||||||d           |                    |j                  S )Nr  c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool3d_backward.<locals>.<lambda>
  r  r.   r   r   r   c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool3d_backward.<locals>.<lambda>
  r9  r.   c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool3d_backward.<locals>.<lambda>
  r  r.   r  c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool3d_backward.<locals>.<lambda>
  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z*meta_avg_pool3d_backward.<locals>.<lambda>
  r  r.   r  r  r  r   zavg_pool3d_backward())	rB   rS   r   r   r   r  avg_pool3d_backward_shape_checkr   r   )r  r   rh  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  otime_for_shape_checkoheight_for_shape_checkowidth_for_shape_checks                           r,   meta_avg_pool3d_backwardr  
  s    
LKF"XX   
QB;1$$+a.B;1$$+a.B	L
+c&kkV+\\   	(vayB	Fc&kkQ&6&6F1IB	Fc&kkQ&6&6F1IB	LGTT   1:Dw<<1$$44'!*Dw<<1$$44'!*D	L
fKK  
 
L5 0A 5**  
 jjnnGJJrNNEjjnnGZZ^^F0D"aSS27Bb!YWW1&"dB9UU#





'  , ??5;'''r.   c                     t          j         j        dk    p
 j        dk     fd            j        d d         t	          |          z   }t          j                   }t          j        | j         j	        |          S )Nr   r  c                      d j          S )Nz"Expected 3D or 4D tensor, but got r  r   s   r,   rQ   z*meta_adaptive_avg_pool2d.<locals>.<lambda>
      ATZAA r.   r  rK  )
rB   rS   r   r   rR   r<   rp  ru   rJ   rm   )r   output_sizer3  r   s   `   r,   meta_adaptive_avg_pool2dr
  
  s    	L	Q($)q.AAAA   :crc?U;%7%77L/55M ;j{#	   r.   c                      t          j         j        dk    p
 j        dk     fd                                 j        d d         t          |          z             S )Nr  r  c                      d j          S )Nz"Expected 4D or 5D tensor, but got r  r   s   r,   rQ   z*meta_adaptive_avg_pool3d.<locals>.<lambda>
  r  r.   r  )rB   rS   r   r   r   rR   )r   r	  s   ` r,   meta_adaptive_avg_pool3dr  
  sb    	L	Q($)q.AAAA   >>$*SbS/E+,>,>>???r.   c                      j         }t          d|          D ]2t          j                                       dk     fd           3t          j        |dk    p|dk    fd           t          j        j         j        k     fd           t          j        }t                    rt          j        }	                    j
                                      |          S )	Nr   r   c                       d j          d dS )Nz{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions,  with dimension  being emptyr  )grad_outr   s   r,   rQ   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>  s1     f6>nf fVWf f f r.   r   r  c                      d j          S )NzBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got r  r   s   r,   rQ   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>	  s    aUYU_aa r.   c                  (    dj          d j          S Nzexpected dtype z! for `grad_output` but got dtype rz   )r  r   s   r,   rQ   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>  s    _$*__x~__ r.   r   )r   r   rB   rS   r   rJ   r   rt  rq  r   r   r   )r  r   r   r   r   s   ``  @r,   "meta__adaptive_avg_pool2d_backwardr  
  s   =D1d^^ 
 
MM!q f f f f f	
 	
 	
 	

 
L	TQYaaaa   
L
hn$_____   +M ,+>>$*%%((}(EEEr.   c                 b    t          | d           t          j        |t          j                  S )Nadaptive_avg_pool3d_backwardr   )!_adaptive_pool_empty_output_checkrB   r   r  r  r   s     r,   "meta__adaptive_avg_pool3d_backwardr    s.     &k3QRRRD0NOOOOr.   r  c                       j         }t          d|          D ]3t          j                                       dk     fd           4d S )Nr   r   c                  $      dj          d dS )Nzc(): Expected grad_output to have non-zero size for non-batch dimensions, but grad_output has sizes r  r  r  )r  r  r   s   r,   rQ   z3_adaptive_pool_empty_output_check.<locals>.<lambda>!  s>     ` `-8->` `PQ` ` ` r.   )r   r   rB   rS   r   )r  r  r   r   s   `` @r,   r  r    s{    D1d^^ 
 
Q!#     	
 	
 	
 	

 
r.   c                 8     j         }t          j        |dv  fd           t          d|          D ]2t          j                                       dk     fd           3t          j        t          |          dk    d            d}d}d} j         dk    r                     d          }|dz  }                     |dz
            }|\  }} j         d	k    r?|||f}                     |          }	                     |t          j        
          }
|	|
fS ||||f}t          j	                   }                     |          
                    |          }	                     |t          j        
          
                    |          }
|	|
fS )Nr   r  c                      d j          S )Nz:adaptive_max_pool2d(): Expected 3D or 4D tensor, but got: r  r   s   r,   rQ   z*meta_adaptive_max_pool2d.<locals>.<lambda>.      ZU[ZZ r.   r   r   c                       dj          d  dS )Nzjadaptive_max_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r  r  r  r   r   s   r,   rQ   z*meta_adaptive_max_pool2d.<locals>.<lambda>3  4    T',{T TDET T T r.   r   c                      dS )NzCadaptive_max_pool2d(): internal error: output_size.size() must be 2r2   r2   r.   r,   rQ   z*meta_adaptive_max_pool2d.<locals>.<lambda>;      U r.   r  r   rz   r   )r   rB   rS   r   r   r   r   r   r<   rp  r   )r   r	  r   dimHsizeBsizeDosizeHosizeWr   r   r   r   r   s   `           @r,   meta_adaptive_max_pool2dr-  (  s    :D	LZZZZ   1d^^ 
 
JJqMMA    	
 	
 	
 	
 
LKAUU  
 DEEzQ

1	JJtax  E NFFzQFF+	ooi((//)5;/??G|E662	3E::ooi((++-+HH//)5;/??BB' C 
 
 G|r.   c                 :     j         }t          j        |dv  fd           t           d           t          j        j         j        k     fd           t          j                  }                    j                  	                    |          S )Nr  c                      d j          S )NzKadaptive_max_pooling2d_backward(): Expected 3D or 4D grad_output, but got: r  r  s   r,   rQ   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>^  s    q^i^oqq r.   adaptive_max_pool2d_backwardc                  (    dj          d j          S r  rz   )r  r   s   r,   rQ   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>e  s    c%+ccP[Pacc r.   r   )
r   rB   rS   r  rJ   r<   rp  r   r   r   )r  r   r   r   r   s   ``   r,   !meta_adaptive_max_pool2d_backwardr3  X  s     D	Lqqqq  
 &k3QRRR	L{((ccccc  
 /66M??5;''***GGGr.   c                 :     j         }t          j        |dv  fd           t          d|          D ]2t          j                                       dk     fd           3t          j        t          |          dk    d            d}d}d}|dk    r                     d          }|dz  }                     |          }|\  }}}|d	k    r||||f}	n|||||f}	                     |	          }
                     |	t          j        
          }|
|fS )Nr  c                      d j          S )Nz:adaptive_max_pool3d(): Expected 4D or 5D tensor, but got: r  r!  s   r,   rQ   z*meta_adaptive_max_pool3d.<locals>.<lambda>r  r"  r.   r   r   c                       dj          d  dS )Nzjadaptive_max_pool3d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r  r  r  r$  s   r,   rQ   z*meta_adaptive_max_pool3d.<locals>.<lambda>w  r%  r.   r   c                      dS )NzCadaptive_max_pool3d(): internal error: output_size.size() must be 3r2   r2   r.   r,   rQ   z*meta_adaptive_max_pool3d.<locals>.<lambda>  r'  r.   r  r  rz   )r   rB   rS   r   r   r   r   r   )r   r	  r   dimDr)  r*  osizeTr+  r,  r   r   r   r   s   `           @r,   meta_adaptive_max_pool3dr:  l  sg    :D	LZZZZ   1d^^ 
 
JJqMMA    	
 	
 	
 	
 
LKAUU  
 DEEqyy

1	JJtE(FFFqyyFFF3		E666:	
//)
$
$Cooiu{o;;G<r.   c                 V    t          | d           |                    |j                  S )Nadaptive_max_pool3d_backward)r  r   r   )r  r   r   s      r,   !meta_adaptive_max_pool3d_backwardr=    s)     &k3QRRR??5;'''r.   c                 N    |t          d          |                     |          S )Nz:cannot repeat_interleave a meta tensor without output_size)r   r   )repeatsr	  s     r,   meta_repeat_interleave_Tensorr@    s+    WXXX[)))r.   c                     | j         j        sJ |j         j        sJ t          | j        |j                  }|                     |t          | j                             S rV  )rJ   r   r    r   r   r   )realimagr   s      r,   meta_complexrD    sZ     :'''':''''!$*dj99I>>)+Ftz+R+R>SSSr.   )
fill_valuer   rE  c                l    |                      ||                                 ft          j                  S rV  )r   rp   rB   r~   )r   r   rE  s      r,   nonzero_staticrG    s)     >>4,EJ>???r.   c           
          t          j        t                    d            g }t                    D ]d\  Ft          j        j        t           j        t           j        t           j        t           j        fv d            j        t           j        t           j        fv rǉ                                }t          |          t          j
        j        z    j        k     fd           t          j                  D ]dt          j
        j                  j        z            k     fd           |                    |                    d                     e8|                               O|                               f|t          j        t                     j        k     fd           dd lm} t%           |j                   t                     j        k     r-                    d            t                     j        k     -d}d}D ]|dk    rd}|dk    rd	} nd
}|sg }g }t                    D ]1\  *|                               |                               2t                    D ]1\  *|                               |                               2                     |           |g }	g }
g }t                    D ]_\  }D|r!|
                     j        |                    *|	                     j        |                    Kt%          j                  }`                     |	|z   |
z             S )Nc                      dS )Nz#at least one index must be providedr2   r2   r.   r,   rQ   z#meta_index_Tensor.<locals>.<lambda>  s    (M r.   c                      dS )Nz?tensors used as indices must be long, int, byte or bool tensorsr2   r2   r.   r,   rQ   z#meta_index_Tensor.<locals>.<lambda>  s    Y r.   c                      d j          S )N)too many indices for tensor of dimension r  r   s   r,   rQ   z#meta_index_Tensor.<locals>.<lambda>  s    S	SS r.   c            	      :    dj          d  dj          dz    S )NzThe shape of the mask z
 at index z0 does not match the shape of the indexed tensor r  )r   r|   jr1  r   s   r,   rQ   z#meta_index_Tensor.<locals>.<lambda>  sP     !h !h !hPQ !h !hJN*!h !h`ade`e!h !h r.   r   c                  :    dj          dt                      dS )NrL  z (got rd   )r   r   r   r   s   r,   rQ   z#meta_index_Tensor.<locals>.<lambda>  s$    \DI\\SQX\\\\\ r.   r   Fr   T)rB   rS   r:  	enumeraterJ   r~   r   r  nonzeror   r   r   r   r   r   selecttorch._refs_refsr   r!   r   r   )r   r   r4  rR  refsstatehas_contiguous_subspacerV  transposed_indicesbefore_shapeafter_shapereplacement_shaperp   r   r|   rN  r1  s   ``           @@@@r,   meta_index_Tensorr]    s   	Lg M MNNN &(Fg&& ! !5L
EIuz5:NNYY   {uz5:666--//KK"
Ndi/SSSS   uz** 8 8A&A$*QU*;;h h h h h h h h  
 MM'..A"6"677778 e$$$$MM%    G	LG	!\\\\\  
 (4('233G
g,,
"
"t g,,
"
" E# ' 'A:: aZZ}  ! #'
 # %!'** 	1 	1HAu A"))%000!'** 	1 	1HAu}A"))%000||D!!$ !LK#%(( 2 2
U=  5""4:c?3333##DJsO4444 $U[ 1 1>>,)::[HIIIr.   c                    d }d }d }|
d         r'|                      |                                          }|
d         r'|                      |                                          }|
d         r|                      |          }|||fS )Nr   r   r   r   r   )grad_output_input_weight_bias_sizes_optr   r  r^  
transposedra  r`  output_maskbackend_grad_inputbackend_grad_weightbackend_grad_biass                 r,   meta_convolution_backwardri    s      1~ C)33FKKMMBB1~ E*44W\\^^DD1~ C(22>BB 35FGGr.   c                                        d          }                     d          }|                     ||f          } t          j                                        dk    d            t          j                                        dk    d            t          j                             d                               d          k    fd           t          j                             d                               d          k    fd           t          j        |                      d          |k    o|                      d          |k    d	            |                     |                                            S )
Nr   r   r   c                      dS r  r2   r2   r.   r,   rQ   zmeta_addbmm.<locals>.<lambda>=  r  r.   c                      dS r  r2   r2   r.   r,   rQ   zmeta_addbmm.<locals>.<lambda>>  r  r.   r   c                  `    d                      d           d                     d           S )Nz8batch1 and batch2 must have same number of batches, got r   r   r   r  r  s   r,   rQ   zmeta_addbmm.<locals>.<lambda>A  s3    p6;;WX>>pp`f`k`klm`n`npp r.   c            
          d                      d           d                      d           d                     d           d                     d           d	S )Nz#Incompatible matrix sizes for bmm (r   r9   r   r   rd   r   rn  s   r,   rQ   zmeta_addbmm.<locals>.<lambda>E  sl    6&++a.. 6 66;;q>> 6 6;;q>>6 6$*KKNN6 6 6 r.   c                      dS )Nz.self tensor does not match matmul output shaper2   r2   r.   r,   rQ   zmeta_addbmm.<locals>.<lambda>L  s    @ r.   )r   rS  rB   rS   rp   r   )r   r  r  r"  r!  r$  r%  s    ``    r,   meta_addbmmrq  7  sg    ;;q>>D;;q>>D;;d|$$D	L"$H$HIII	L"$H$HIII	LA&++a..(ppppp   
LA&++a..(	
 	
 	
 	
 	
   
L		!51!5@@   >>$))++&&&r.   )
grad_scale	found_infc       	         t    | |||||fD ]-t          j        t          t                    fd           .d S )Nc                  (    dt                      S Nz'exponent must be a tensor list but got re   ls   r,   rQ   z#meta__fused_adam_.<locals>.<lambda>g      Gd1ggGG r.   rB   rS   r\   r   )r   gradsexp_avgsexp_avg_sqsmax_exp_avg_sqsstate_stepslrbeta1beta2weight_decayepsamsgradmaximizerr  rs  ry  s                  @r,   meta__fused_adam_r  Q  s]    & E8[/;O 
 
q$GGGG	
 	
 	
 	

 
r.   c       	             | |||||fD ]-t          j        t          t                    fd           .d } ||            ||           ||           ||           ||          fS )Nc                  (    dt                      S rv  rw  rx  s   r,   rQ   z"meta__fused_adam.<locals>.<lambda>  rz  r.   c                     d | D             S )Nc                 6    g | ]}t          j        |          S r2   r5  )r8   r  s     r,   r;   z=meta__fused_adam.<locals>.empty_like_list.<locals>.<listcomp>  s#    999 ##999r.   r2   )tensor_lists    r,   empty_like_listz)meta__fused_adam.<locals>.empty_like_list  s    99[9999r.   r{  )r   r|  r}  r~  r  r  r  r  r  r  r  r  r  rr  rs  r  ry  s                   @r,   meta__fused_adamr  k  s    & E8[/;O 
 
q$GGGG	
 	
 	
 	

: : : 	!!$$(( r.   c                 r    t          j                                         dk    d            t          j                                        dk    d            t          j         j        t           j        u  fd           t          j        j        t           j        u fd           t          j                             d                              d          k     fd                                                     d                              d          ft           j        	          S )
Nr   c                      dS )Nza must be a 2D tensorr2   r2   r.   r,   rQ   zmeta__int_mm.<locals>.<lambda>      '> r.   c                      dS )Nzb must be a 2D tensorr2   r2   r.   r,   rQ   zmeta__int_mm.<locals>.<lambda>  r  r.   c                      d j          S )Nzexpected self to be int8, got rz   )rQ  s   r,   rQ   zmeta__int_mm.<locals>.<lambda>      ::: r.   c                      d j          S )Nzexpected mat2 to be int8, got rz   )rR  s   r,   rQ   zmeta__int_mm.<locals>.<lambda>  r  r.   r   r   c            
          d                      d           d                      d           d                     d           d                     d           d	S )Nz'Incompatible matrix sizes for _int_mm (r   r9   r   r   rd   r   rQ  rR  s   r,   rQ   zmeta__int_mm.<locals>.<lambda>  sl    ,affQii , ,!&&)) , ,66!99, , vvayy, , , r.   rz   )rB   rS   rp   rJ   r  r   r   r  r  s   ``r,   meta__int_mmr    s"    
LA>>???	LA>>???	L	5:::::   
L	5:::::   
L	q		QVVAYY	
 	
 	
 	
 	
   ;;q		166!99-U[;AAAr.   c                 j    t          j                                         dk    d            t          j         j        t           j        u  fd                                d          }                     d          dz  }                     |dz  ||dz  z  d|dz  ft           j        	          S )
Nr   c                      dS Nzw must be a 2D tensorr2   r2   r.   r,   rQ   z2meta__convert_weight_to_int4pack.<locals>.<lambda>  r  r.   c                      d j          S )Nzexpected w to be uint8, got rz   r  s   r,   rQ   z2meta__convert_weight_to_int4pack.<locals>.<lambda>      8qw88 r.   r   r      r+      rz   )rB   rS   rp   rJ   uint8r   r   r  )r  inner_k_tilesr   r1  s   `   r,    meta__convert_weight_to_int4packr    s    	LA>>???	L	5;8888   	
q		A	q		AA;;F-"$%Q		
 k    r.   c                     t          j                                         dk    d            t          j                                        dk    d            t          j         j        t           j        t           j        t           j        fv  fd           t          j        j        t           j        u fd                                 	                    d          	                    d          dz   j        	          S )
Nr   c                      dS Nzx must be a 2D tensorr2   r2   r.   r,   rQ   z*meta__weight_int4pack_mm.<locals>.<lambda>  r  r.   r  c                      dS )Nzw must be a 4D tensorr2   r2   r.   r,   rQ   z*meta__weight_int4pack_mm.<locals>.<lambda>  r  r.   c                      d j          S Nz#expected x to be f32/f16/bf16, got rz   r9   s   r,   rQ   z*meta__weight_int4pack_mm.<locals>.<lambda>      ?ag?? r.   c                      d j          S )Nzexpected w to be int32, got rz   r  s   r,   rQ   z*meta__weight_int4pack_mm.<locals>.<lambda>  r  r.   r   r  rz   )
rB   rS   rp   rJ   r,  r-  r.  r  r   r   )r9   r  q_group_sizeq_scale_and_zeross   ``  r,   meta__weight_int4pack_mmr    s    	LA>>???	LA>>???	L	EM5=%.AA????   
L	5;8888   ;;qvvayy!&&))a-qw;???r.   c                     t          j                                         dk    d            t          j         j        t           j        t           j        t           j        fv  fd           t          j                                        dk    d            t          j        j        t           j        u fd                                 	                    d          	                    d           j                  S )Nr   c                      dS r  r2   r2   r.   r,   rQ   z*meta__weight_int8pack_mm.<locals>.<lambda>  r  r.   c                      d j          S r  rz   r  s   r,   rQ   z*meta__weight_int8pack_mm.<locals>.<lambda>  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z*meta__weight_int8pack_mm.<locals>.<lambda>  r  r.   c                      d j          S )Nzexpected w to be int8, got rz   r  s   r,   rQ   z*meta__weight_int8pack_mm.<locals>.<lambda>  s    7ag77 r.   r   rz   )
rB   rS   rp   rJ   r,  r-  r.  r  r   r   )r9   r  q_scaless   `` r,   meta__weight_int8pack_mmr    s    	LA>>???	L	EM5=%.AA????   
LA>>???	L	5:7777   ;;qvvayy!&&))17;;;;r.   c                     t          j                                         dk     fd           t          j                                        dk    fd           t          j                             d                              d          k     fd           t          j        t	          j         j                  d            t          j        t	          j        j                  d            t          j        |dk    d	            t          j        d
v fd                                d          }                    d          } j        d d         }j        d d         }t          t          j	        ||                    }|
                    ||g                                |          S )Nr   c                  4    d                                   dS )Nz1cdist only supports at least 2D tensors, X1 got: Dr   )x1s   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>      OBFFHHOOO r.   c                  4    d                                   dS )Nz1cdist only supports at least 2D tensors, X2 got: r  r   )x2s   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>  r  r.   r   c                  `    d                      d           d                     d           S )Nz4X1 and X2 must have the same number of columns. X1: r   z X2: r   )r  r  s   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>  s2    frwwr{{ffY[Y`Y`acYdYdff r.   c                      dS )Nz=cdist only supports floating-point dtypes, X1 got: {x1.dtype}r2   r2   r.   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>  r   r.   c                      dS )Nz=cdist only supports floating-point dtypes, X2 got: {x2.dtype}r2   r2   r.   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>  r   r.   r   c                      dS )Nz)cdist only supports non-negative p valuesr2   r2   r.   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>  s    !L r.   Nr   r   c                      d  S )Nz%possible modes: None, 1, 2, but was: r2   )compute_modes   r,   rQ   z$meta_cdist_forward.<locals>.<lambda>  s    FFF r.   r  )rB   rS   rp   r   r<   is_float_dtyperJ   r   r   broadcast_shapesextendr   )	r  r  r
  r  r1r2batch_tensor1batch_tensor2r3  s	   `` `     r,   meta_cdist_forwardr    s   	L
AOOOO   
L
AOOOO   
L
rwwr{{"fffff   
LRX&&OO   
LRX&&OO   
LaLLMMM	L$FFFF   
B	BHSbSMMHSbSMM.}mLLMMLR!!!<<%%%r.   c                 @   |j         d         }|j         d         }|j         d         }|j         d d         }|j         d d         }	t          t          j        ||	                    }
|
                                }|                    ||g           t          j        |
          }|dk    s|dk    s|dk    s|dk    rt          j        |          S |t          |j                   k    r|	                    |          }t          j
        |t          j                  S )Nr   r  r   r   )r   r   rB   r  copyr  mathprod
zeros_likerS  r   r   )r  r  r  r
  cdistc1r  r  r  r  rO  tensor1_expand_sizebatch_products                r,   meta_cdist_backwardr    s
    
"B	"B	"BHSbSMMHSbSMM 6}m T TUU.3355Bx(((I233M	Qww"''R1WW(:(:###d28nn,,YY*++Be.EFFFFr.   c	                     t          j        j        t           j        t           j        fv fd           t          j        j        t           j        t           j        fv fd           t          j        t          j         j                   fd                               d          }	|rt          j        |	dk    d            |	dz  }	                     |	                     d                    }
t          d          \  }}}t          j        ||k    d            t          j        j         j        k     fd	           t          j        j
        dk    fd
           t          j                                                                        k    fd           fdd fd}t                    dk    r                                        d                    }                                                              }||k    r*                    |	                     d                    }n                    d          }n | |
|          }|||fv s|s)                                        d                    }n                    d          }                    |	          }j        d         }||k    rC|rt          j        |dk    d            |dz  }                    | j        d                   }n'                    |                                          }|
|||fS )Nc                      d j          S )Nz(expected indices to be long or int, got rz   r   s   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>"      J7=JJ r.   c                      d j          S )Nz(expected offsets to be long or int, got rz   )rD  s   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>&  r  r.   c                      d j          S )Nz/expected weight to be floating point type, got rz   )r  s   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>*  s    P&,PP r.   r   r   c                      dS Nz1include_last_offset: numBags should be at least 1r2   r2   r.   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>1  s    G r.   r   c                      dS )Nz@embedding_bag: per_sample_weights only supported with mode='sum'r2   r2   r.   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>;  s    V r.   c                  *    dj          d j          dS )Nzexpected weight (z) and per_sample_weights (z) to have same dtyperz   )per_sample_weightsr  s   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>?  s!    ~~~PbPh~~~ r.   c                      d j          dS )Nz1expected per_sample_weights to be 1D tensor, got r  r  )r  s   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>C  s    bHZH_bbb r.   c                  ^    d                                  d                                   dS )Nz%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (rd   r   )r   r  s   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>G  sF    I8J8P8P8R8R I I6=mmooI I I r.   c                 P     | ||          o|                     d          dk    S Nr   r   r   )r   r?  r   padding_idxis_fast_path_index_selects       r,   is_fast_path_index_select_scalez;meta_embedding_bag.<locals>.is_fast_path_index_select_scaleM  s0    %%c6;??XELLQROOWXDX	
r.   c                     | j         t          j        k    s| j         t          j        k    o7|                     d          dk    o|                    d          dk    o|dk     S Nr   r   )rJ   rB   rF   rD   r   )r   r   r  s      r,   r  z5meta_embedding_bag.<locals>.is_fast_path_index_selectR  s_    Y%+%@ej)@  

1" a  A%  a		
r.   c                 >    | | |||          S  | ||          S r)   r2   )r   r?  r   r  r  r  s       r,   is_fast_pathz(meta_embedding_bag.<locals>.is_fast_pathZ  s5    223v{SSS,,S&+FFFr.   cpuc                      dS r  r2   r2   r.   r,   rQ   z$meta_embedding_bag.<locals>.<lambda>t  s    O r.   )rB   rS   rJ   r~   r   r<   r  r   r   r   r   r   rC  r   )r  r   rD  scale_grad_by_freqr*  sparser  include_last_offsetr  num_bagsr   MODE_SUM	MODE_MEANMODE_MAXr  
offset2bagbag_sizemax_indicesfast_path_sumnumBagsr  r  s   ```   `             @@r,   meta_embedding_bagr    s    
L%*ei00JJJJ   
L%*ei00JJJJ   
LV\**PPPP  
 ||AH MGG	
 	
 	
 	AhA77F$)!HH!Hi%HVV	
 	
 	
 	$4~~~~~	
 	
 	
 	#q(bbbb	
 	
 	
 	$$&&'--//9    	
 	
 	

 
 
 
 


 
 
G G G G G G 7u$$&&w||A77
$$W\\^^448!++Hfkk!nnEEKK!++A..KK$V-?UUIx(((( **7<<??;;JJ **1--J$$X..-"8" qLOO   1!++GV\!_EEKK!++HMMOO<<K:x44r.   c                     t          | ||g|R  \  }}}}t          |          dk    r'|                    |                                          }||||fS )Nr  )r  rC  r   r   )r  r   rD  r>   r   r  r  r  s           r,   meta_embedding_bag_forward_onlyr   }  sm    0B1#'1 1 1-FJ+ 7u$$$$W\\^^44:x44r.   c                 r    |r|S | j         j        s| j         j        r| j         S |rt          j        S | j         S r)   )rJ   r   r   rB   r~   )r   rJ   promote_int_to_longs      r,   _get_reduction_dtyper    sH     {$ (> {	 z;r.   rz   c                    t          | |d          }t          j        | j        |          }t	          | ||          }|                     ||          S )NT)r  rz   )r  r<   rX  r   rY  r   )r   rV  r[  rJ   r  r3  s         r,   meta_nansumr    sS     (u$OOOLT22D+E4AAL??<|?<<<r.   c           	          t          j        | j        t          t	          |                                                               }|                     |          S r)   )r<   rW  r   rR   r   rp   r   )r   r3  s     r,   meta_medianr    sG    7U5--.. L ??<(((r.   c                    t          |           dk    rt          j        d           t          j        | j        |f          }t          | ||          }|                     |          |                     |t          j                  fS )NrB  zmedian CUDA with indices outputrz   )	rC  r<   alert_not_deterministicrX  r   rY  r   rB   r~   )r   rp   r[  r3  s       r,   meta_median_mode_dimr
    s~     5V##%&GHHH

u{SF
3
3C+E3@@L%%EJ77 r.   c                     | S r)   r2   r   s    r,   meta_logical_not_r    r1  r.   c                 p   t          j        t                    |                                 k    d            t                    |                                 z
  }d|z  t	          | j                  z   fdt          t                              D             }|                     |          S )Nc                      dS )NzZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr2   r2   r.   r,   rQ   zmeta_repeat.<locals>.<lambda>  s    l r.   r  c                 2    g | ]}|         |         z  S r2   r2   )r8   r   padded_sizer?  s     r,   r;   zmeta_repeat.<locals>.<listcomp>  s&    LLL1;q>GAJ.LLLr.   )rB   rS   r   rp   rR   r   r   r   )r   r?  num_new_dimensionstarget_sizer  s    `  @r,   meta_repeatr    s    	LG

"ll   W

2++eDJ.?.??KLLLLLc'll8K8KLLLK>>+&&&r.   c                     | S r)   r2   r   s    r,   
meta_zero_r    r1  r.   c                 n    t          |t          j                  rt          | j        |j                   | S r)   r\   rB   r   rU   r   r   r   s     r,   meta_binop_inplacer    s1     %&& 9
EK888Kr.   c                 n    t          |t          j                  rt          | j        |j                   | S r)   r  )r   r   r!  s      r,   meta_binop_inplace_alphar    s1     %&& 9
EK888Kr.   c                 8    t          | t          j                  S Nr4   )r@   r   r=   )r   kwargss     r,   
meta_roundr     s"    <D   r.   c                 \    t          j        t          j        j                   fd           t          t           j                  r2t          j        t          j        j                   fd           d S t          j        t          t                     fd           d S )Nc                        dj          S )Nz7: Expected input tensor to have an integral dtype. Got rz   )r  r   s   r,   rQ   z#shift_dtype_check.<locals>.<lambda>  s    7__SWS]__ r.   c                        dj          S )Nz6: Expected shift value to have an integral dtype. Got rz   r  rl  s   r,   rQ   z#shift_dtype_check.<locals>.<lambda>  s    waaVYV_aa r.   c                        d S )Nz): Expected shift value to be an int. Got r2   r$  s   r,   rQ   z#shift_dtype_check.<locals>.<lambda>  s    wNNNN r.   )rB   rS   r<   r  rJ   r\   r   r   )r  r   rl  s   ```r,   shift_dtype_checkr&    s    	Ltz**_____   #u|$$ 	
"39--aaaaa	
 	
 	
 	
 	

 	sG$$NNNNN	
 	
 	
 	
 	
r.   c                 \    t          d| |           t          | |t          j                  S )Nrshiftr  r&  r@   r   r=   r  s     r,   meta_rshiftsr*    6    he,,,e$C$K   r.   c                 \    t          d| |           t          | |t          j                  S )Nlshiftr  r)  r  s     r,   meta_lshiftsr.    r+  r.   c                 6    |                      | j                  S r)   r  r   s    r,   	meta_zeror0    s    >>$*%%%r.   c                     | S r)   r2   r   rl  s     r,   
meta_fill_r3     r1  r.   c                 *    t          j        |           S r)   r5  r2  s     r,   	meta_fillr5  %      D!!!r.   c                     | S r)   r2   r   s    r,   
meta_relu_r8  *  r1  r.   c                 *    t          j        |           S r)   r5  r   r   r   
accumulates       r,   meta_index_putr<  /  r6  r.   c                 :    t          | j        |j                   | S r)   )rU   r   )r   rG  values      r,   meta_masked_fill_r?  4  s    DJ
333Kr.   c                     |                      |                                                               t          j        |                     }|S r;  )r   r   r   r<   rp  )r   rG  r?  masked_scales       r,   meta__masked_scalerB  :  sF    >>$))++..111$77 2  L r.   c                     t          j        |j        t           j        t           j        fv d            t          j        | j        |j        k    d            | S )Nc                      dS )NzMask must be bool or uint8r2   r2   r.   r,   rQ   z&meta_masked_scatter_.<locals>.<lambda>E  s    9U r.   c                      dS )Nzdmasked_scatter: expected self and source to have same dtypes but got {self.dtype} and {source.dtype}r2   r2   r.   r,   rQ   z&meta_masked_scatter_.<locals>.<lambda>I  s	     9 r.   )rB   rS   rJ   r:  r  )r   rG  r7  s      r,   meta_masked_scatter_rF  B  sb    	L
uz5;//1U1U   
L
fl"	9 	9  
 Kr.   c                     t          | |          \  } }t          j        | t          j                  }t	          |||          S r;  )r!   rB   r   r   rF  )r   rG  r7  r   s       r,   meta_masked_scatterrH  O  sA     "$--JD$d%2IJJJFf555r.   c                 ,    |                      |          S r)   rR  )r   rG  r  s      r,   meta_masked_scatter_backwardrJ  W  s    >>%   r.   c                     | S r)   r2   r:  s       r,   meta_index_put_rL  \  r1  r.   c                 6    |                      | j                  S r)   )viewr   r   s    r,   
meta_aliasrO  a  s    99TZ   r.   c                   	
 t          j        |                                 dk    d            t          j        |                                dk    d            |                                 }|                                |d         	|d         
|d         }d         }	||ft          j        d         	k    od         
k    	
fd           |                              }|s][t          j                                        dk    d            t          j                                        k    fd	           |S )
Nr   c                      dS r  r2   r2   r.   r,   rQ   z)common_meta_baddbmm_bmm.<locals>.<lambda>g  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z)common_meta_baddbmm_bmm.<locals>.<lambda>h  r  r.   r   r   r   c            	      :    d d d d          d d          d	S r  r2   r   s   r,   rQ   z)common_meta_baddbmm_bmm.<locals>.<lambda>u  sU     RSU R RR R+7?R R>J1oR R R r.   c                      dS )Nzself must be a 3D tensorr2   r2   r.   r,   rQ   z)common_meta_baddbmm_bmm.<locals>.<lambda>~  s    6P r.   c                  8    d  d                                  S )Nz*Expected an input tensor shape with shape z but got shape: r   )r	  self_baddbmms   r,   rQ   z)common_meta_baddbmm_bmm.<locals>.<lambda>  s%    sss^j^o^o^q^qss r.   )rB   rS   rp   r   r   )r  r  is_bmmrV  r'  res_rowsres_colsr   r!  r"  r#  r	  s      `    @@@@r,   common_meta_baddbmm_bmmrZ  f  sr   	L"$H$HIII	L"$H$HIII;;==L;;==L	aB#AAHAHx*K	LQ2E,q/5E"E	R 	R 	R 	R 	R 	R   k**F 
l.\%%''1,.P.PQQQ;.sssss	
 	
 	

 Mr.   c                 $    t          | |d          S )NT)rZ  )r   r  s     r,   meta_bmmr\    s    "4t444r.   c                     | |z  }| |z  }|dk    r-t          |dk               t          |dk               k    r|dz  }|S r  )r:  )r9   yqr.  s       r,   div_rtnr`    sK    	QA	AA 	Avv4A;;$q1u++--	QHr.   c                     t          | |z   |z   ||dz
  z  z
  dz
  |r|dz
  ndz   |          dz   }|r|dz
  |z  | |z   k    r|dz  }|S r  )r`  )	inputSize
kernelSizer  r  r   r^  r  
outputSizes           r,   pooling_output_shape_pad_lrre    s     	 *q.)* 	
 '-vzzA/ 	
 	
 		   Nf$	E(999!OJr.   c           	          t          j        |dk    d            t          j        dk    fd           t          j        dz
  z  dz   dz  k    fd           t          | ||          S )Nr   c                      dS )Nzstride should not be zeror2   r2   r.   r,   rQ   z&pooling_output_shape.<locals>.<lambda>  s    &A r.   c                      d  S )Nz'pad must be non-negative, but got pad: r2   )pads   r,   rQ   z&pooling_output_shape.<locals>.<lambda>  s    #RS#R#R r.   r   r   c                      d d d  S )NzApad should be at most half of effective kernel size, but got pad=z, kernel_size=z and dilation=r2   )r^  rc  ri  s   r,   rQ   z&pooling_output_shape.<locals>.<lambda>  s;    @PS @ @%@ @5=@ @ r.   )rB   rS   re  )rb  rc  ri  r   r^  r  s    `` ` r,   r  r    s    	L1AABBB	LRRRRSSS	La8+a/A55	
 	
 	
 	
 	
 	
   ':sC9  r.   c           	      L   	
                                   }	t          j        dk    odk    d            t          j        |dk    o|dk    d            t          j        |dk    o|dk    d                                 d          dk    o                     d          dk    }|t          j        k    r6t          j        |dk    o|o                     d          dk    d	            nXt          j        |dk    r                     d          dk    r|p |dk    o|o                     d          dk     fd
           t          j        dz  k    odz  k    fd           t          j        dk    odk    
	fd           d S )Nr   c                      dS )NzCkernel size should be greater than zero, but got kH: {kH}, kW: {kW}r2   r2   r.   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  r'  r.   c                      dS )Nz>stride should be greater than zero, but got dH: {dH}, dW: {dW}r2   r2   r.   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  s    P r.   c                      dS )Nz\dilation should be greater than zero, but got dilationH: {dilationH}, dilationW: {dilationW}r2   r2   r.   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  s    n r.   r   r   r  r   c                      dS )NzExpected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: {input.size()}r2   r2   r.   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  s
     Q r.   c                  2    d                                   S )NzYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: r   r!  s   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  s    ~pupzpzp|p|~~ r.   c                       d d d d  S )NzKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r2   )r  r  r  r  s   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  s>     >> >!%> >.0> >9;> > r.   c                  .    d d  d d d d dS NzGiven input size: (r9   z). Calculated output size: (z). Output size is too smallr2   )r  r  r  r  r  r  s   r,   rQ   z$pool2d_shape_check.<locals>.<lambda>  sZ     #k # #K # #* # #$0# #3?# #BM# # # r.   )rp   rB   rS   r   rq  )r   r  r  r  r  r  r  	dilationH	dilationWr  r  r  r  r  r   r   
valid_dimsr  s   ```  ``  `````   @r,   r  r    s8   " 99;;DL	L
Q26UU   
L
Q26PP   
LA')a-nn  
 A!#:

1(:J+++AI;*;A!);Q Q	
 	
 	
 	
 	QY<5::a==A-<* A	?j?UZZ]]a-?~~~~	
 	
 	
 
L
a4+B!GtO	> 	> 	> 	> 	> 	> 	>   
Lq.\Q.	# 	# 	# 	# 	# 	# 	# 	# 	#    r.   r  r  r  r  r  r  r  pTpHpW	dilationTrt  ru  r  r  r  r  r  r  r  c           
      8   	
  j         }t          j        dk    odk    odk    fd           t          j        dk    odk    odk    fd           t          j        dk    odk    odk    fd           t          j        |dv  fd           t          |          D ]@|dk    rdk    rt          j                                       dk     fd           A|r-t          j        k    ok    ok    fd	           t          j        d
z  k    od
z  
k    od
z  	k    	
fd           t          j        dk    odk    odk    fd           d S )Nr   c                      d d  d S )Nz5kernel size should be greater than zero, but got kT: z, kH: z, kW: r2   )r  r  r  s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>  4    ,, ,, ,'), , r.   c                      d d  d S )Nz0stride should be greater than zero, but got dT: z, dH: z, dW: r2   )r  r  r  s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>!  r}  r.   c                      d d  d S )Nz9dilation should be greater than zero, but got dilationT: z, dilationH: z, dilationW: r2   )rt  rz  ru  s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>(  s;    V#V V2;V VJSV V r.   r  c                        dj          S )Nz/: Expected 4D or 5D tensor for input, but got: r  )r  r   s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>0  s    7XX5;XX r.   r  c                  J      dj          d                               dS )NzZ: Expected input's non-batch dimensions to have positive length, but input has a shape of z and non-batch dimension z has length zero!)r   r   r  r   r   s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>9  sG     M M-2[M M,1JJqMMM M M r.   c                  .    d d  d d d d dS )Nzinput image (T: r  r  z ) smaller than kernel size (kT:  kH:  kW: rd   r2   )r  r  r  r  r  r  s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>C  s]    <5 < <g < <6 < <$&< <-/< <68< < < r.   r   c                  ,    d d d  d d d S )NzHpad should be smaller than or equal to half of kernel size, but got kT: r  r  z padT: z padW: z padH: r2   )r  r  r  rx  rw  ry  s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>K  sh    KK KK K%'K K02K K;=K KFHK K r.   r   c                  :    d d d  d d d d d dS rs  r2   )r  r  r  r  r  r  r  s   r,   rQ   z$pool3d_shape_check.<locals>.<lambda>S  sv    (' ( (E ( (G ( (f ( ((/( (27( (:A( (DJ( ( ( r.   )r   rB   rS   r   r   )r   r  r  r  r  r  r  r  rw  rx  ry  rz  rt  ru  r  r  r  r  r  r  r  r  r   r   s   `````````````````````  @r,   r  r    s   0 :D	L
Q$26$b1f	
 	
 	
 	
 	
 	
   
L
Q$26$b1f	
 	
 	
 	
 	
 	
   
LA9)a-9IM	
 	
 	
 	
 	
 	
   
LXXXXX  
 4[[ 
 
199aJJqMMA     	
 	
 	
 	
  
RK:GrM:fl        	
 	
 	
 
L
Q"6a26"q&B,	
 	
 	
 	
 	
 	
 	
 	
 	
   
L
3v{3w!|	
 	
 	
 	
 	
 	
 	
 	
 	
 	
    r.   c                    | j         }t          | |||||||	|
||||||||||||           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           d S )Nr  r   r   r   r   r  r  )r   r  r   r  r  r  r  r  r  r  rw  rx  ry  rz  rt  ru  r  r  r  r  r  r  r  r   s                           r,   max_pool3d_backward_shape_checkr  [  s$   2 :D








+  0 ;dQh888;dQh666;dQh888;dQh7777D$(G4447D$(E2227D$(G4447D$(F33333r.   c                    | j         }t          | ||||||||	|
|ddd|||||||d           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           t          |||dz
  |           d S )Nr   Tr  r   r   r  )r   r  r  r  r  r  r  r  r  rw  rx  ry  r  r  r  r  r  r  r  r   s                       r,   r  r    s    * :D








			-  2 ;dQh888;dQh666;dQh888;dQh77777r.   c                 P   d } |d|          \  }}t          j        t          |          dv d            t          |          dk    r||}
}	n |d|          \  }	}
 |d|          \  }} |d|          \  }}|                     d	          }|                     d
          }|                     d          }t	          j        |           }|t           j        k    r-t          j        |                                 dk    d            nQ|t           j        k    r+t          j        |                                 dv d            nt          j        dd            t          ||||	||          }t          ||||
||          }t          | |||	|
||||||||||           |||fS )Nc                      t          j        t          |          dv  fd           |d         }t          |          dk    r|n|d         }||fS )Nr  c                      d  dS )Nzmax_pool2d: r  r2   r  s   r,   rQ   zEmax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>  r  r.   r   r   r  r  s   `   r,   r  z3max_pool2d_checks_and_compute_shape.<locals>.unpack  r  r.   rh  r  c                      dS )NzOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr2   r2   r.   r,   rQ   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>  r  r.   r   r   r  r^  r  r  r   r  c                      dS )NzMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr2   r2   r.   r,   rQ   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>  s    c r.   r  c                      dS )Nz9non-empty 3D or 4D (batch mode) tensor expected for inputr2   r2   r.   r,   rQ   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>      O r.   Fc                      dS )Nz?Unsupport memory format. Supports only ChannelsLast, Contiguousr2   r2   r.   r,   rQ   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>   s    U r.   )rB   rS   r   r   r<   rp  rq  rp   r   r  r  )r   rh  r   r  r^  r  r  r  r  r  r  r  r  rt  ru  r  r  r  r   r  r  s                        r,   r  r    s      VM;//FB	LFy aa   6{{aRB&))B	7++JD$!6*h77Iy**R..K**R..KBJ/66M+++IIKK1cc	
 	
 	
 	
 
%1	1	1IIKK6!OO	
 	
 	
 	

 	UU	
 	
 	

 (Rr9iXXL&z2tRIVVK



  $ k11r.   c                 P    t          |||||          \  }t          j        j         j        k     fd           |j        fd}	 |	             |	|           t          j                  }
t          j        j        j        j	        |
          S )Nc                  (    dj          d j          S )NzExpected dtype z  for `gradOutput` but got dtype rz   r  s   r,   rQ   z7meta_max_pool2d_with_indices_backward.<locals>.<lambda>0  s    a$*aakN_aa r.   c                     t          | dz
             t          | dz
             t          | dz
             d S )Nr   r   r   )r  )r  r  r   r  r  s    r,   _check_dim_sizez>meta_max_pool2d_with_indices_backward.<locals>._check_dim_size6  sQ    q$q,777q$q,777q$q+66666r.   rK  )
r  rB   rS   rJ   r   r<   rp  ru   r   rm   )r  r   rh  r   r  r^  r  r   r  r  r   r  r   r  r  s   ``         @@@@r,   %meta_max_pool2d_with_indices_backwardr    s     	,k67Hi	 		
 
L
k''aaaaa  
 L9D7 7 7 7 7 7 7 7
 OK   OG/55M;
j{#	   r.   c                    t          | |||||          \  }}}|                                 dk    r|                     d          nd}	t          j        |           }
|                                 dk    r|||g}n|	|||g}t          j        || j        | j        |
          t          j        |t
          j	        | j        |
          fS r  )
r  rp   r   r<   rp  rB   ru   rJ   rm   r   r  s               r,   meta_max_pool2d_with_indicesr  G  s     	,{FGXy	 		
  %yy{{a//UZZ^^^QF/66Myy{{a\;7\;?+<'		
 	
 	
 	+<'		
 	
 	
 r.   c           	         
 t          j         j        dv  fd            j        }t          |dz
  |          D ]G
t          j                             
          dk    d                                  d
 d           Ht          j        t                    dk    d	            t          j        t          |          dk    d
                                 d          }                     d                               d          |dk    r                     d          }nd}t          j         j        j        k    d            t          j        j        dk    fd                               d          }                    d          }                    d          
t          j        ||k    d           t          j        ||k    d            t          j        
dk    
fd           t          j        |d         d         z   dz
  k    fd           t          j        |d         d         z   dz
  k    fd                                            dk    r|||d         |d         g}	n||d         |d         g}	t          j        |	 j         j	                  t          j        |	t           j
         j	                  fS )Nr  c                      d j          S )Nz:fractional_max_pool2d: Expected 3D or 4D tensor, but got: r  r   s   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>r  s    XTYXX r.   r   r   z^fractional_max_pool2d: Expected input to have non-zero  size for non-batch dimenions, but got r  z emptyr   c                      dS )NzNfractional_max_pool2d: kernel_size musteither be a single int or tuple of Intsr2   r2   r.   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>  	     2 r.   c                      dS )NzOfractional_max_pool2d: output_size must either be a single int or tuple of Intsr2   r2   r.   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>  r  r.   r  r  r   r  r   c                      dS )Nz6Expect _random_samples to have the same dtype as inputr2   r2   r.   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>      H r.   c                      d j          S )Nz1Expect _random samples to have 3 dimensions got, r  )random_sampless   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>  s    YNDWYY r.   z=Expect _random_samples.size(0) no less then input batch size.c                      dS )Nz<Expect _random_samples.size(1) equals to input channel size.r2   r2   r.   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>      N r.   c                      d  dS )Nz/Expect _random_samples.size(2) equals to 2 got .r2   )r   s   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>  s    !WST!W!W!W r.   c                       dd          d  S )Nz%fractional_max_pool2d: kernel height r   z' is too large relative to input height r2   )input_heightrh  s   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>  s    }A}}o{}} r.   c                       dd          d  S )Nz$fractional_max_pool2d: kernel width r   z& is too large relative to input width r2   )input_widthrh  s   r,   rQ   z,meta_fractional_max_pool2d.<locals>.<lambda>  s    z{1~zzmxzz r.   rJ   rm   )rB   rS   r   r   r   r   rJ   rp   ru   rm   r   )r   rh  r	  r  r   input_channelsinput_batchr   cr   r   r  r  s   `` `      @@@r,   meta_fractional_max_pool2dr  n  sb   	L	VXXXX   9D4!8T"" 
 
IIaLL1]6:iikk] ]ST] ] ]	
 	
 	
 	
 
LKA	2 	2  
 
LKA	2 	2   YYr]]N99R==L))B--Kqyyiill	L
n**HH   
Lq YYYY  
 	AAAAAA	L	[G   
L	^NN   
LaWWWWXXX	LAQ'!+|;}}}}}   
LAQ'!+{:zzzzz  
 xxzzQ^[^[^LAA? 	*;	
 	
 	

 	+;	
 	
 	
 r.   c                 $    t          j        d           t          j        j        t          j        k    fd           t          j        t                    dk    fd           \  }}t          j         j        dv  fd           t          j         j        j        k     fd           t          d j                  D ]2t          j         
                              d	k     fd
           3                                   j        dk    r. 
                    d	          }                     |||f          }nC 
                    d	          } 
                    d          }                     ||||f          }|S )Nmax_unpooling2d_forward_outc                      d j          S )Nz2elements in indices should be type int64 but got: rz   r  s   r,   rQ   z#meta_max_unpool2d.<locals>.<lambda>  s    TW]TT r.   r   c                  *    dt                      dS )NzMThere should be exactly two elements (height, width) in output_size, but got 
 elements.r  r	  s   r,   rQ   z#meta_max_unpool2d.<locals>.<lambda>  '    4;''4 4 4 r.   r  c                      d j          dS )NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with  dimensions.r  r   s   r,   rQ   z#meta_max_unpool2d.<locals>.<lambda>  s!    =%)Y= = = r.   c                  (    dj          d j          S NzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: r  rP  s   r,   rQ   z#meta_max_unpool2d.<locals>.<lambda>  s1    BQUQ[ B B29-B B r.   r   r   c                       dj          d  dS )NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got r   being empty.r  )r   r   s   r,   rQ   z#meta_max_unpool2d.<locals>.<lambda>  s4    H:H H78H H H r.   r   )r<   r	  rB   rS   rJ   r   r   r   r   r   r   r   r   )	r   r   r	  r  r  	nchannelsr4  r  r   s	   ```     @r,   meta_max_unpool2dr    s    
!"?@@@	L$TTTT   
LKA	
 	
 	
 	
   "OGV	L	V	
 	
 	
 	
   
L
gm#	
 	
 	
 	
 	
   1di   
 
IIaLL1    	
 	
 	
 	
 ??DyA~~IIaLL	GV <==1IIaLL	GV DEEMr.   c                     t          j        j        t           j        k    d            t          j         j        dv  fd           t          j        t                    dk    fd           t          j        t                    dk    fd           t          j        t                    dk    fd           t          j         j        j        k     fd           t          d	 j                  D ]3t          j                                       d
k     fd           4t          j        d
         d
k    od	         d
k    od         d
k    fd           d S )Nc                      dS )Nz(elements in indices should be type int64r2   r2   r.   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  s    .X r.   r  c                      d j          dS )NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with r  r  r!  s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  s    w_d_iwww r.   r   c                  *    dt                      dS )NzVThere should be exactly three elements (depth, height, width) in output_size, but got r  r  r  s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  r  r.   c                  *    dt                      dS )NzRThere should be exactly three elements (depth, height, width) in stride, but got: r  r  r  s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  s    |ehioepep||| r.   c                  *    dt                      dS )NzSThere should be exactly three elements (depth, height, width) in padding, but got: r  r  )r  s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  s    ~fijqfrfr~~~ r.   c                  (    dj          d j          S r  r  )r   r   s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  s1    BQVQ\ B B29-B B r.   r   r   c                  $      dj          d dS )NzI: Expected input to have non-zero size for non-batch dimensions, but got r  r  r  r  s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>  s=     I I ;I I89I I I r.   r   c                      d  S )Nz5strides should be greater than zero, but got stride: r2   r  s   r,   rQ   z._max_unpooling3d_shape_check.<locals>.<lambda>'  s    PPP r.   )	rB   rS   rJ   r   r   r   r   r   r   )r   r   r	  r   r  r  r   s   ``````@r,   _max_unpooling3d_shape_checkr    s   	L$&X&X   
L
fwwww   
LKA	
 	
 	
 	
   
LFq||||   
LG~~~~   
Lw}$	
 	
 	
 	
 	
   1ej!! 
 
JJqMMA     	
 	
 	
 	
 
Lq	A9&)a-9F1IMPPPP    r.   c                    t          j        d           t          | ||||d           |                                 } |\  }}}| j        dk    r/|                     d          }|                     ||||f          }	nD|                     d          }
|                     d          }|                     |
||||f          }	|	S )Nmax_unpooling3d_forward_outzmax_unpooling3d()r  r   r   )r<   r	  r  r   r   r   r   )r   r   r	  r   r  odepthr  r  r  r4  r  s              r,   meta_max_unpool3dr  +  s     
!"?@@@ g{FG5H   ??D)FGVyA~~IIaLL	FGV DEE1IIaLL	FGV LMMMr.   c                 T   t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        | pt          |          dv d            |s|n|d         }	|s|nt          |          dk    r|	n|d         }
|s|nt          |          dk    r|	n|d         }t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        | j        d	v d
            | j        dk    r|                     d          nd}|                     d          }|                     d          }|                     d          }|                     d          }t          ||||	||          }t          ||||
||          }t          ||||||          }t          | |||||	|
|||||||||||||d           | j        dk    o!t          j        |           t           j	        k    }| j        dk    rQ| 
                    d          }|                                 o|                    t           j	                  }||||f}n|||||f}|                     |          }|                     |t           j                  }|r@|                    t           j	                  }|                    t           j	                  }||fS )Nr  c                      dS NzMmax_pool3d: kernel_size must either be a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z.meta_max_pool3d_with_indices.<locals>.<lambda>O      _ r.   r   r   r   c                      dS NzQmax_pool3d: stride must either be omitted, a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z.meta_max_pool3d_with_indices.<locals>.<lambda>W      c r.   c                      dS NzImax_pool3d: padding must either be a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z.meta_max_pool3d_with_indices.<locals>.<lambda>_      [ r.   c                      dS NzJmax_pool3d: dilation must be either a single int, or a tuple of three intsr2   r2   r.   r,   rQ   z.meta_max_pool3d_with_indices.<locals>.<lambda>g  r9  r.   r  c                      dS r  r2   r2   r.   r,   rQ   z.meta_max_pool3d_with_indices.<locals>.<lambda>o  r  r.   r  r  r  r  r   zmax_pool3d_with_indices()r  r   rz   )rB   rS   r   r   r   r  r  r<   rp  r  ra  rp  r   r   r   )r   rh  r   r  r^  r  r  r  r  r  r  r  rw  rx  ry  rz  rt  ru  r  r  r  r  r  r  r  r  rq  input_channels_last_checkr   r   r   s                                  r,   meta_max_pool3d_with_indicesr  C  s    
LKF"__   
QB;1$$+a.B;1$$+a.B	L
+c&kkV+cc   	(vayB	Fc&kkQ&6&6F1IB	Fc&kkQ&6&6F1IB	LG[[   
B7||q  gajB7||q  gajB	LH\\   I ]]a//		Xa[I ]]a//		Xa[I	L
fKK  
  %zQUZZ^^^AFjjnnGJJrNNEjjnnGZZ^^F BIyIIE"7BB	9MMG!&"b"iKKF








#+  2 	
aXE7>>%BXX  zQ$)OOA$6$6!)77999
'550 6 
 
 	
 eWf5		WeWf=	
//)
$
$Cooiu{o;;G Cff5#9f::**5+A*BB<r.   c                    t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }	t          |          dk    r|n|d         }
t          j        | pt          |          dv d            |s|n|d         }|s|	nt          |          dk    r|n|d         }|s|
nt          |          dk    r|n|d         }t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        t          |          dv d            |d         }t          |          dk    r|n|d         }t          |          dk    r|n|d         }t          j        |j        d	v d
            |                    d          }|                    d          }|                    d          }|                    d          }|                     d          }|                     d          }|                     d          }t          || ||||	|
|||||||||||||||d           |j        dk    o!t          j        |          t           j        k    }|j        dk    rJ|	                    d          }|
                                 o|
                    t           j                  }|                    |j                  }|r |                    t           j                  }|S )Nr  c                      dS r  r2   r2   r.   r,   rQ   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  r  r.   r   r   r   c                      dS r  r2   r2   r.   r,   rQ   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  r9  r.   r  c                      dS r  r2   r2   r.   r,   rQ   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  r  r.   r  r  r  r   z"max_pool3d_with_indices_backward()r  r  r   )rB   rS   r   r   r   r  r<   rp  r  ra  rp  r   r   r   )r  r   rh  r   r  r^  r  r   r  r  r  r  r  r  rw  rx  ry  rz  rt  ru  r  r  r  r  r  r  r  rq  r  r  s                                 r,   %meta_max_pool3d_with_indices_backwardr    s    
LKF"__   
QB;1$$+a.B;1$$+a.B	L
+c&kkV+cc   	(vayB	Fc&kkQ&6&6F1IB	Fc&kkQ&6&6F1IB	LG[[   
B7||q  gajB7||q  gajB	LH\\   I ]]a//		Xa[I ]]a//		Xa[I	L
fKK  
 jjnnGJJrNNEjjnnGZZ^^FR  Er""Gb!!F#








,/  6 	
aXE7>>%BXX  zQ$)OOA$6$6!)77999
'550 6 
 
 	 --J I]]1G]HH
r.   gridc                 "    t          j         j        j        k     fd           t          j         j        t           j        k    oj        t           j        k     fd           t          j         j        d         j        d         k     fd           t          j        j        d          j        dz
  k     fd           t          d j                  D ]*t          j         j                 dk     fd           +d S )	Nc                  (    dj          d j          S )NzNgrid_sampler(): expected input and grid to be on same device, but input is on z and grid is on rw  r  r   s   r,   rQ   z+check_grid_sampler_common.<locals>.<lambda>  s0    A\A A37;A A r.   c                  (    dj          d j          S )NzTgrid_sampler(): expected input and grid to have torch.strided layout, but input has z and grid has )rl   r  s   r,   rQ   z+check_grid_sampler_common.<locals>.<lambda>  s0    CC C59[C C r.   r   c                  (    dj          d j          S )NzZgrid_sampler(): expected grid and input to have same batch size, but got input with sizes  and grid with sizes r  r  s   r,   rQ   z+check_grid_sampler_common.<locals>.<lambda>$  s0    O %O OBF*O O r.   r   r   c                  .    dj         dz
   d j         S )Nz+grid_sampler(): expected grid to have size r   z, in last dimension, but got grid with sizes )r   r   r  s   r,   rQ   z+check_grid_sampler_common.<locals>.<lambda>+  s.    ?%*q. ? ?26*? ? r.   c                       dj          d  dS )NzYgrid_sampler(): expected input to have non-empty spatial dimensions, but input has sizes r  r  r  r$  s   r,   rQ   z+check_grid_sampler_common.<locals>.<lambda>4  r%  r.   )rB   rS   rm   rl   r~  r   r   r   )r   r  r   s   ``@r,   check_grid_sampler_commonr    st   	L#	
 	
 	
 	
 	
   
L%F$+*F	
 	
 	
 	
 	
   
LA$*Q-'	
 	
 	
 	
 	
   
L
2%*q.(	
 	
 	
 	
 	
   1ej!! 
 
KNQ    	
 	
 	
 	

 
r.   c                       e Zd ZdZdZdZdS )GridSamplerInterpolationr   r   r   N)rf   
__module____qualname__BILINEARNEARESTBICUBICr2   r.   r,   r  r  ;  s        HGGGGr.   r  interpolation_modec                      t          j         j        dk    o j        j        k     fd           t          j         j        dk    o|t          j        j        k     d            d S )Nr  c                  (    dj          d j          S )Nzdgrid_sampler(): expected 5D input and grid with same number of dimensions, but got input with sizes r  r  r  s   r,   rQ   z'check_grid_sampler_3d.<locals>.<lambda>D  s+    149K1 1$(J1 1 r.   c                      dS )Nz<grid_sampler(): bicubic interpolation only supports 4D inputr2   r2   r.   r,   rQ   z'check_grid_sampler_3d.<locals>.<lambda>O  r  r.   )rB   rS   r   r  r  r>  )r   r  r  s   `` r,   check_grid_sampler_3dr  A  s    	L
a3EJ$)3	
 	
 	
 	
 	
   
LJ!O M"&>&F&LL	
 	ON    r.   c                     |d         }|r!t          j        |t           j                  }nd }t          j        |t           j                  }	||	fS Nr   r   )rB   r  r   r   
r  r   r  r  padding_modealign_cornersre  input_requires_gradr  	grad_grids
             r,   grid_sampler_2d_backward_metar  S  sX     &a. %e5;RSSS


 U5LMMMI	""r.   c                     t          | |           t          | ||           | j        d         }| j        d         }|j        d         }|j        d         }|j        d         }	|                     |||||	f          S )Nr   r   r   r   )r  r  r   r   )
r   r  r  r  r  rP  Cout_Dout_Hout_Ws
             r,   grid_sampler_3dr  f  s{     eT***%'9:::AAAAJqMEJqMEJqME??Aq%6777r.   r  c                     t          ||           t          |||           |d         }|r!t          j        |t          j                  }nd }t          j        |t          j                  }	||	fS r  )r  r  rB   r  r  r   r  s
             r,   grid_sampler_3d_backwardr
  y  s     eT***%'9:::%a. %!?
 
 


 
 U5STTTIy  r.   c                     |                     dd           }|st          j        |          }||d<   t          j        | g|R i |S )NrJ   )rI   r<   	get_dtyperB   ru   )r   rE  r>   r  rJ   s        r,   fullr    sU    JJw%%E ,
++F7O;t-d---f---r.   c                 n   |t           j        k    rt          j        |d u d            t          j        d|| j        n|||| j        n||          }| j        rN|                    |                                 | 	                                | 
                                           n;|                    |                                 |                                 d           |                    d           |S t          j                            | |||||          }|                    d           |S )Nc                      dS )Nz9memory format option is only supported by strided tensorsr2   r2   r.   r,   rQ   zzeros_like.<locals>.<lambda>  r  r.   r   r   Tru  )rB   
sparse_coorS   ru   rJ   rm   	is_sparsesparse_resize_and_clear_r   
sparse_dim	dense_dimrp   _coalesced_r#   r   r   fill_)r   rJ   rl   rm   rn   r   rk  s          r,   r  r    sA    !!!T!OO	
 	
 	

 k %$**5"(.4;;f!
 
 
 > 	E((		T__..0@0@    ((dhhjj!DDD

/
!
!# "  C IIaLLLJr.   c                                                       }t          j        |dk    d            dk    rn|z                                  }t          j         |k    p|k      fd           dk    rn|z   t	                                                     }t	                                                     }                                 |         z  z   }|= |=                      |||          S )Nr   c                      dS )Nz-select() cannot be applied to a 0-dim tensor.r2   r2   r.   r,   rQ   zmeta_select.<locals>.<lambda>  s    ? r.   c                  >    d d                                  d  S )Nzselect(): index z! out of range for tensor of size z at dimension r   rp   r|   r   s   r,   rQ   zmeta_select.<locals>.<lambda>  s8     ,5 , ,99;;, ,&), , r.   )rp   rB   r   r   r   r   r   r   )r   rp   r|   r   r   new_sizer	  new_storage_offsets   ```     r,   meta_selectr    s(   88::D		??  
 ##sTzC99S>>D	Vd]+etm,	, 	, 	, 	, 	, 	,   aZZEEUT\EDIIKK  Hdkkmm$$J,,..C1HH3??8Z1CDDDr.   c                 *    t          j        |           S r)   r<   clone_preserve_strides)r   r   rp   r|   s       r,   meta_select_scatterr!        '---r.   c                 *    t          j        |           S r)   r  )r   r   rp   rh   rg   steps         r,   meta_slice_scatterr%    r"  r.   dim_post_exprwrap_scalarc                     |dk    r|sJ d}| }|dz
  }| |k     s| |k    rJ d|  d| d| d            | dk     r| |z  } | S )Nr   r   zdim z out of bounds (rc   rd   r2   )rp   r&  r'  r}  r~  s        r,   r   r     s~    .C
!
Cc		S3YYY)R)R)RS)R)RC)R)R)RYYY
Qww}Jr.   c                 P    |                                  dk    rdn| j        |         S r  r  )r  rp   s     r,   ensure_nonempty_sizer*    s"    111!'#,.r.   c                 p    t                                           d          }t                                          d          }t          j        ||k    d            t	          |          D ]Ck    r;t          j        t                    t                     k     fd           Dd S )Nr   c                      dS )NzDIndex tensor must have the same number of dimensions as input tensorr2   r2   r.   r,   rQ   z$gather_shape_check.<locals>.<lambda>  s    V r.   c                  :    d dj          dj          d  z   S )Nz!Size does not match at dimension z expected index  to be smaller than self  apart from dimension r  )rp   r   r|   r   s   r,   rQ   z$gather_shape_check.<locals>.<lambda>  s6    \A\\u{\\UdjUUPSUUV r.   )r~  rp   rB   rS   r   r*  )r   rp   r|   	self_dims
index_dimsr   s   ```  @r,   gather_shape_checkr2    s    DHHJJ""IUYY[[!$$J	LZVV   9  88L$UA..2FtQ2O2OOV V V V V V V   r.   c                 F   ddl m} t          ||                                           } |                                dk              }|s<t          j        j        t
          j        k    fd           t          | |           | 
                    j                  S )Nr   guard_size_obliviousc                      d j          S )Nz2gather(): Expected dtype int64 for index, but got rz   r{   s   r,   rQ   zmeta_gather.<locals>.<lambda>  s    VVV r.   )r   r5  r   rp   r   rB   rS   rJ   r~   r2  r   r   )r   rp   r|   sparse_gradr5  wrapped_dimis_index_emptys     `    r,   meta_gatherr:    s    JJJJJJ dhhjj11K))%++--1*<==N 5K5:%VVVV	
 	
 	
 	4e444>>%+&&&r.   c                     |r@| dk    rdS | dk    rdS | dk    rdS | dk    rdS | d	k    rd
S t          j        dd            d S | dk    rdS | dk    rdS t          j        dd            d S )Nsum
REDUCE_ADDr  REDUCE_MULTIPLYmeanREDUCE_MEANamaxREDUCE_MAXIMUMaminREDUCE_MINIMUMFc                      dS )Nz=reduce argument must be either sum, prod, mean, amax or amin.r2   r2   r.   r,   rQ   z#get_operator_enum.<locals>.<lambda>/  s    S r.   addmultiplyc                      dS )Nz/reduce argument must be either add or multiply.r2   r2   r.   r,   rQ   z#get_operator_enum.<locals>.<lambda>7  s    $U r.   r0  )reduce_use_new_optionss     r,   get_operator_enumrK  !  s     e<$$ =####SS	
 	
 	
 	e<
""$$UUUVVVr.   c                      ddl m}  ||                                dk              r+t          j        |j        t          j        k     fd           |(t          j        |j        |j        k     fd           d S d S )Nr   r4  c                        dS )Nz"(): Expected dtype int64 for indexr2   method_names   r,   rQ   z,scatter_gather_dtype_check.<locals>.<lambda>B  s    {FFF r.   c                        dS )Nz0(): Expected self.dtype to be equal to src.dtyper2   rN  s   r,   rQ   z,scatter_gather_dtype_check.<locals>.<lambda>H  s    {TTT r.   )r   r5  r   rB   rS   rJ   r~   )rO  r   r|   src_optr5  s   `    r,   scatter_gather_dtype_checkrR  <  s    JJJJJJEKKMMQ.// 
K5:%FFFF	
 	
 	

 J'-'TTTT	
 	
 	
 	
 	
 r.   c                 "    t          | d          S r  )r~  r   s    r,   ensure_nonempty_dimrT  L  s    sA;;r.   c                 z    ddl m}  |                                dk              rd S t          j        t                                                     t                                                    k    d            d}t                                                     }t          |          D ]1}t          |          }|k    r|t           |          k    rd} n2|s<:t          |          D ]*}t          |          }|t          |          k    rd} n+vt          j        t                                                     t                                                    k    d            t          j        |  fd           d S t          j        |  fd           d S )	Nr   r4  c                      dS NzCIndex tensor must have the same number of dimensions as self tensorr2   r2   r.   r,   rQ   z%scatter_shape_check.<locals>.<lambda>X  r'  r.   FTc                      dS rW  r2   r2   r.   r,   rQ   z%scatter_shape_check.<locals>.<lambda>r  s    Y r.   c                  D    dj          dj          d  dj          z   S )NExpected index r.  r/  z and to be smaller than src r  )rp   r|   r   rQ  s   r,   rQ   z%scatter_shape_check.<locals>.<lambda>v  s7    XekXXDJXXWsWWWWX r.   c                  4    dj          dj          d  z   S )NrZ  r.  r/  r  r  s   r,   rQ   z%scatter_shape_check.<locals>.<lambda>|  s,    XekXXDJXX,s,,- r.   )	r   r5  r   rB   rS   rT  rp   r   r*  )	r   rp   r|   rQ  r5  is_wrong_shaper0  r   index_d_sizes	   ````     r,   scatter_shape_checkr^  Q  s.   JJJJJJEKKMMQ.// 	LDHHJJ''+>uyy{{+K+KKUU  
 N#DHHJJ//I 9  +E15588.tQ7777!NE 8
  g1y!! 	 	A/q99L27A>>>>!% ? 

++/B599;;/O/OOYY	
 	
 	
 	X X X X X X X	
 	
 	
 	
 	
 	- - - - - -	
 	
 	
 	
 	
r.   c                     t          ||                                           }t          d| ||           t          | |||           |t	          ||           d S d S )Nscatter)r   rp   rR  r^  rK  )r   rp   r|   r   rI  rJ  r8  s          r,   scatter_meta_implra    se     dhhjj11Ky$s;;;k5#666'?33333 r.   c                 \    t          | |||d           |                     | j                  S NrF  ra  r   r   r   rp   r|   r   s       r,   meta_scatter_addrf    s,    dCU333>>$*%%%r.   c                 ,    t          | |||d           | S rc  ra  re  s       r,   meta_scatter_add_ri    s    dCU333Kr.   c                     t          |t          j                  r|nd }t          | ||||           |                     | j                  S r)   )r\   rB   r   ra  r   r   r   rp   r|   src_or_valuer8  r   s         r,   meta_scatterrm    sH     %\5<@@
J,,dCdCV444>>$*%%%r.   c                 h    t          |t          j                  r|nd }t          | ||||           | S r)   )r\   rB   r   ra  rk  s         r,   meta_scatter_ro    s9     %\5<@@
J,,dCdCV444Kr.           queryr   r>  	dropout_p	is_causalreturn_debug_maskr?  c                 *   |                      d          }|                      d          }|                      d          }	|                      d          }
|                     d          }|                     dd          }t          j        |                              dd          }t          j        |||	ft          j        | j                  }|rX|
dk    rdnd}t          j        |	|z            }|dk    rd}n|dk    rd}t          j        |||	|f| j	        | j                  }n!t          j        d| j	        | j                  }||d d |	|t          j        d	t          j
        d
          t          j        d	t          j
        d
          |f	S )Nr   r   r   r   r  @         r2   rk   )r   r  rB   r   ru   rF   rm   r  ceilrJ   r~   )rq  r   r>  rr  rs  rt  r?  r   	num_headsmax_seqlen_batch_qhead_dimmax_seqlen_batch_kquery_t	attention	logsumexpblocksize_cmax_seqlen_k
debug_masks                     r,   (meta__scaled_dot_product_flash_attentionr    s    AJ

1IAzz!}}H!ooa##G ))33Aq99I	Y 23k|  I  L%]]ccy!3k!ABB$$LL3&&L[$6E+<
 
 


 [%+elKKK
 	Bej888Bej888
 
r.   	attn_biascompute_log_sumexpc	           	      4   |                      d          }	|                      d          }
|                      d          }|                     d          }|                      d          }|                     d          }t          j        |	|
||f| j        | j                  }t          j        |	|
|ft          j        | j                  }t          j        dt          j        d          }t          j        dt          j        d          }||d d ||||d f	S )Nr   r   r   r   r  r2   rk   )r   rB   ru   rJ   rm   rF   r~   )rq  r   r>  r  r  rr  rs  rt  r?  r  r  S_QS_KVD_QKD_Vrk  
logsum_expseedoffsets                      r,   (meta__scaled_dot_product_cudnn_attentionr    s    	

1A

1A
**Q--C88A;;D::b>>D
**R..C
+q!S#&ek%,
O
O
OC	
Ask|  J ;rF;;;D[5:f===F 	
 
r.   r  r  	cum_seq_q	cum_seq_kmax_qmax_kphilox_seedphilox_offsetc                 t   t          j        |                    dd                                        dd          }t          j        |                    dd                                        dd          }t          j        |                    dd                                        dd          }|||fS r  )rB   r   r  )r  rq  r   r>  r   r  r  r  r  r  rr  rs  r  r  r?  grad_qgrad_kgrad_vs                     r,   'meta__scaled_dot_product_flash_backwardr    s    , eooa3344>>q!DDFcmmAq1122<<QBBFeooa3344>>q!DDF66!!r.   	attn_maskc                 T   |                      d          }|                      d          }|                      d          }	|                      d          }
t          j        |           }t          j        ||	|ft          j        | j                                      dd          }||fS )Nr   r   r   r   r  )r   rB   r   ru   rF   rm   r  )rq  r   r>  rr  rs  r  r?  r   rz  r{  r|  r  r  s                r,   0meta__scaled_dot_product_flash_attention_for_cpur  4  s     AJ

1IAzz!}}H ''I	

 k|   i1oo  	 r.   c
                    |                     d          }
|                     d          }|                     d          }|                     d          }|                     d          }t          j        |
|||fd|j        |j                  }t          j        |
|||fd|j        |j                  }t          j        |
|||fd|j        |j                  }|||fS )Nr   r   r   r   r   r   r   r   r  )r   rB   empty_permutedrJ   rm   )r  rq  r   r>  r   r  rr  rs  r  r?  r   rz  r|  len_qlen_kr  r  r  s                     r,   9meta__scaled_dot_product_flash_attention_for_cpu_backwardr  W  s    & AJ

1Izz!}}HJJqMMEHHQKKE!	Yx0k|	  F !	Yx0iz	  F !	Yx0k|	  F 66!!r.   c                    |                      dd          } |                     dd          }|                     dd          }|                     d          }|                     d          }	|                    d          }
|                     d          }|                     d          }|                    d          }t          j        ||	||| j        | j                  }|rt          j        |	dz            dz  nd}t          j        |||ft          j        | j                  }|                     dd          }t          j        dt          j	        d	          }t          j        dt          j	        d	          }||||fS )
Nr   r   r   r  r   r  r  r2   rk   )
r  r   rB   ru   rJ   rm   r  ry  rF   r~   )rq  r   r>  r  r  rr  rs  r?  r  MrP  rz  KKvrk  logsumexp_dimr  r  r  s                      r,   ,meta__scaled_dot_product_efficient_attentionr    sY    OOAq!!E
--1

COOAq!!E

1A

1AA

2I

2A	BB
+aIrU\
R
R
RC.@GDIa"f%%**aM	
I}%k|  J --1

C ;rF;;;D[5:f===F
D&((r.   grad_input_maskc                    |                     d          }|                     d          }|                     d          }|                     d          }|                     d          }|                     d          }t          j        ||||fd|j        |j                  }t          j        ||||fd|j        |j                  }t          j        ||||fd|j        |j                  }d }||
d         r~|                     d          }|dz  dk    r|n
|dz   |dz  z
  }t          |                                           }||d<   t          j        ||j        |j                  }|d	d |f         }||||fS )
Nr   r   r   r   r  r  r   r+  .)r   rB   r  rJ   rm   r   ru   )r  rq  r   r>  r  r   r  r  r  rr  r  rs  r?  r   rz  r  r|  
head_dim_vr  r  r  r  	grad_biaslastDimlastDimAligned	new_sizess                             r,   +meta__scaled_dot_product_efficient_backwardr    s   ( AJ

1IJJqMMEzz!}}HAJHHQKKE!	Yx0k|	  F !	Yx0iz	  F !	Yz2k|	  F I!3..$$$+bLA$5$57R<'TV,;V))**	&	"KY_Y5E
 
 
	 c8G8m,	669,,r.   c                     t          j        |          }t          j        |          }t          j        |          }|||fS r)   r5  )r  rq  r   r>  r   r  r  r  r  r  r  r  r  rr  rs  r?  r  r  r  s                      r,   'meta__scaled_dot_product_cudnn_backwardr    sA    . e$$Fc""Fe$$F66!!r.   window_size_leftwindow_size_right	seqused_kalibi_slopesc                    ||                      d          n|                                dz
  }||                      d          n|}||                     d          n|}|                      d          }|                      d          }t          j        |           }t          j        |||ft          j        | j                  }|	rX|dk    rdnd}t          j        ||z            }|dk    rd}n|dk    rd}t          j        ||||f| j	        | j                  }n!t          j        d| j	        | j                  }||t          j        d	t          j
        d
          t          j        d	t          j
        d
          |fS )Nr   r   r  r   r  rv  rw  rx  r2   rk   )r   r   rB   r   ru   rF   rm   r  ry  rJ   r~   )rq  r   r>  r  r  r  r  rr  rs  rt  r?  r  r  r  r  r   r{  r}  rz  r|  r  r  r  r  r  s                            r,   meta__flash_attention_forwardr    s   4 #,"3A9J9JQ9NJ*3*;A(1(9!u

2Izz"~~H  ''I	Y 23k|  I  L%]]ccy!3k!ABB$$LL3&&L[$6E+<
 
 


 [%+elKKK
 	Bej888Bej888 r.   c                     t          j        |          }t          j        |          }t          j        |          }|||fS r)   r5  )r  rq  r   r>  r   r  r  r  r  r  rr  rs  r  r  r?  r  r  
grad_querygrad_key
grad_values                       r,   meta__flash_attention_backwardr  H  sA    0 !%((J$$H!%((Jx++r.   cu_seqlens_qcu_seqlens_kmax_seqlen_qr  custom_mask_typecausal_diagonalseqlen_kwindow_sizec                    |                      d          }|                      d          }|                     d          }|                      d          }|                      d          }|                     d          }t          j        ||||| j        | j                  }||                     d          dz
  n|}|}||J |}||n|}|
rt          j        |dz            dz  nd}t          j        |||ft          j        | j                  }t          j        dt          j        d          }t          j        dt          j        d          }||||||fS )	Nr   r   r  r   r  r  r2   rk   )	r   rB   ru   rJ   rm   r  ry  rF   r~   )rq  r   r>  r  r  r  r  r  rr  r  r  r?  r  r  r  r  r  rP  rz  r  r  rk  logsumexp_batch_dimactual_max_seqlen_qactual_max_seqlen_kr  r  r  r  s                                r,   !meta__efficient_attention_forwardr  g  si   , 	

1A

1AA

2I

2A	BB
+aIrU\
R
R
RC7C7O,++A..22VW'''**6*B,,4FM	%*++b00A  	i7k|  J ;rF;;;D[5:f===F
D&*=?RRRr.   bias_requires_gradnum_splits_keyshared_storage_dqdkdvc                    |rt          j        |j        d         |j        d         k    d            t          j        |j        d         |j        d         k    d            t          j        g |j        dd         d|j        d         |j        d         R |j        |j                  }|                    d	d          }|                    d	d          }|                    d	d
          }n<t          j        |          }t          j        |          }t          j        |          }||                    d          }|dz  dk    r|n
|dz   |dz  z
  }t          |                                          }||d<   t          j        ||j        |j                  }|dd |f         }nt          j        d|j                  }||||fS )Nr   c                      dS )Nz,seqlen must match for `shared_storage_dqdkdvr2   r2   r.   r,   rQ   z4meta__efficient_attention_backward.<locals>.<lambda>  s    B r.   r   c                      dS )Nz3embedding dim must match for `shared_storage_dqdkdvr2   r2   r.   r,   rQ   z4meta__efficient_attention_backward.<locals>.<lambda>  s    I r.   r   r  r   r  r  r   r+  .r2   rw  )
rB   rS   r   ru   rJ   rm   rS  r   r   r   )r  rq  r   r>  r  r  r  r  r  r  rr  r  r  r  r  r?  r  r  chunkr  r  r  r  r  r  r  s                             r,   "meta__efficient_attention_backwardr    s   2  -KNcil*BB	
 	
 	
 	KNcil*II	
 	
 	
 Eek!B$EEEKOEU[_EE+<
 
 

 \\"a((
<<A&&\\"a((

%e,,
#C((%e,,
))B--$+bLA$5$57R<'TV,;V%%	&	"K	DKPPP	c8G8m,		K5<888	xY66r.   scale_ascale_bscale_resultuse_fast_accumc                     d }d }	d }
t          j                                         dk    o                                dk     fd           t          j         |                                           d            t          j         |	                                          d            t          j                             d          d	z  d
k     fd           t          j                            d
          d	z  d
k    o                    d          d	z  d
k    fd           t          j         |
 j                  o |
j                   fd           t          j        j        t           j        k    oj        t           j        k    d             j        \  }                    d                                          dk    r                                dk    rnt          j                                        dk    o                                dk    fd                               d
          k    r                    d          dk    ro                    d
          dk    rV                    d          k    r=t          j        	                                o	                                d            nt          j        dfd           ||n j        }t          j
                             d
                              d          | j                  S )Nc                 >    | d         | d         k    o| d         dk    S r  r2   r  s    r,   is_row_majorz$meta_scaled_mm.<locals>.is_row_major  s"    ay6!9$7a7r.   c                 2    | d         dk    o| d         dk    S r  r2   r  s    r,   is_col_majorz$meta_scaled_mm.<locals>.is_col_major  s    ayA~/&)a-/r.   c                 b    | t           j        t           j        t           j        t           j        fv S r)   )rB   float8_e4m3fnfloat8_e5m2float8_e4m3fnuzfloat8_e5m2fnuzrz   s    r,   is_fp8_typez#meta_scaled_mm.<locals>.is_fp8_type  s,    !!	
 
 	
r.   r   c                  \    d                                  d                                   S )Nz%Inputs must be 2D but got self.dim()=z and mat2.dim()=r   r  r   s   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  s-    `

``TXT\T\T^T^`` r.   c                      dS )Nzself must be row_majorr2   r2   r.   r,   rQ   z meta_scaled_mm.<locals>.<lambda>      ( r.   c                      dS )Nzmat2 must be col_majorr2   r2   r.   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  r  r.   r   r+  r   c                  4    d                      d           S )NzBExpected self.size(1) to be divisible by 16, but got self.size(1)=r   r   r   s   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  s     cUYU^U^_`UaUacc r.   c                      d j          S )Nz>Expected both dimensions of mat2 to be divisble by 16 but got r  )r  s   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  s    ]QUQ[]] r.   c                  (    dj          d j          S )Nz8Expected both inputs to be fp8 types but got self.dtype=z and mat2.dtype=rz   r  s   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  s    s4:ssgkgqss r.   c                      dS )Nz6Both scale_a and scale_b must be float (fp32) tensors.r2   r2   r.   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  r  r.   c                  \    d                                  d                                 S )NzLFor non-tensorwise scaling, scale tensors must be 2D, but got scale_a.dim()=z and scale_b.dim()=r   )r  r  s   r,   rQ   z meta_scaled_mm.<locals>.<lambda>  s2    zU\U`U`UbUbzzjqjujujwjwzz r.   c                      dS )Nz@Both scale_a and scale_b must be contiguous for rowwise scaling.r2   r2   r.   r,   rQ   z meta_scaled_mm.<locals>.<lambda>$  s    Z r.   Fc                      d  d d                     d           d                     d           d                     d           d                     d           dS )	Nz}Invalid scaling configuration. For tensorwise scaling, both scales should be scalar. For rowwise scaling, scale_a should be (z, 1), scale_b should be (1, z). Got scale_a.size()=(r   rc   r   z) and scale_b.size()=(rd   r   )r2  r   r  r  s   r,   rQ   z meta_scaled_mm.<locals>.<lambda>*  s    Q?@Q Q^_Q Q ,3<<??Q Q ?Fll1ooQ Q ,3<<??	Q Q ?Fll1oo	Q Q Q r.   r  )rB   rS   rp   r   r   rJ   r,  r   r   rp  ru   rm   )r   r  r  r  r  r  r  r  r  r  r  r1  
_out_dtyper2  r   s   ````         @@r,   meta_scaled_mmr    s   8 8 80 0 0
 
 
 
L

a+DHHJJ!O`````   
LT[[]]##((   
LT[[]]##((   
L		!rQcccc   
L		!rQ9499Q<<"#4#9]]]]   
LDJ;KK
$;$;sssss   
L&I7=EM+IHH   :DAq		!A}}!1 4 4 	KKMMQ57;;==A#5zzzzz	
 	
 	
 LLOOq  Q1$$Q1$$Q1$$ L%%''CG,A,A,C,CZZ    L      	 	 	 (3J;tyy||TYYq\\DKXXXXr.   c                 `    t          | ||||d           |                     | j                  S NT)rJ  rd  r   rp   r|   r   r8  r6  s         r,   meta_scatter_reduce_twor  7  s3     dCVTJJJJ>>$*%%%r.   c                 0    t          | ||||d           | S r  rh  r  s         r,   meta_scatter_reduce__twor  >  s"    dCVTJJJJKr.   c                d    t          j        d                                 cxk     odk    nc  fd                                            dk    r&t          j        |t           j         j                  S t          j                             d          |t           j         j                  S )Nr   r   c                  2    d                                   S )Nz@The probabilty distributions dimensions must be 1 or 2, but got r   r!  s   r,   rQ   z"meta_multinomial.<locals>.<lambda>I  s    `SXS\S\S^S^`` r.   r   r  )rB   rS   rp   ru   r~   rm   r   )r   num_samplesreplacementr   s   `   r,   meta_multinomialr  D  s     
L	EIIKK1````   yy{{a{;ejNNNN;

1{%*U\   r.   c                     d}| D ]}||z  }|S r  r2   )vsr.  vs      r,   multiply_integersr  R  s&    	A  	QHr.   c                     t          j        t                    k    fd           dz   t          j        t                     k     fd           t          j        t          d  dd          D                       ot          d D                        fd            d d         \  }}||gR S )Nc                  .    d  dt                     S )Nz%It is expected output_size equals to , but got size r  )num_spatial_dimsr	  s   r,   rQ   z'upsample_common_check.<locals>.<lambda>\  s#    k8HkkY\]hYiYikk r.   r   c                  .    d  dt                     S )Nz$It is expected input_size equals to r  r  )expected_input_dimsr  s   r,   rQ   z'upsample_common_check.<locals>.<lambda>a  s#    l7Jll[^_i[j[jll r.   c              3   "   K   | ]
}|d k    V  dS r   Nr2   r8   rs  s     r,   r_   z(upsample_common_check.<locals>.<genexpr>e  s&      **aAE******r.   c              3   "   K   | ]
}|d k    V  dS r  r2   r  s     r,   r_   z(upsample_common_check.<locals>.<genexpr>e  s&      2N2NQ1q52N2N2N2N2N2Nr.   c                      d  d S )NzDInput and output sizes should be greater than 0, but got input size z and output size r2   )r  r	  s   r,   rQ   z'upsample_common_check.<locals>.<lambda>f  s)     A A A3>A A r.   )rB   rS   r   rb  )r  r	  r  r  channelsr  s   ```  @r,   upsample_common_checkr  Y  s   	LK,,kkkkk   +Q.	LJ..lllll  
 
L**:abb>*****Ns2N2N+2N2N2N/N/N	A 	A 	A 	A 	A   ""1"~FHH+{+++r.   c                 p    t          j                                         dk    p(t                                           dd                     fd           t                                           |d          }                     |                              t          j	                             S )Nr   r   c                  2    d                                   S )Nz>Non-empty 3D data tensor expected but got a tensor with sizes r   r!  s   r,   rQ   z$upsample_nearest1d.<locals>.<lambda>t      _QVQ[Q[Q]Q]__ r.   r  r   
rB   rS   r   r  r   r  r   r   r<   rp  )r   r	  scalesfull_output_sizes   `   r,   upsample_nearest1dr  n       
LA/

QRR0@AA____   -

kA   ??+,,//1%88 0   r.   c                     t          j                                         dk    p(t                                           dd                     fd           t                                           |d          }                     |          }t          j                   } j	        \  }}}} j
        j        dk    r|dk     rt           j        }|                    |          }|S )	Nr   r   c                  2    d                                   S Nz>Non-empty 4D data tensor expected but got a tensor with sizes r   r!  s   r,   rQ   z$upsample_nearest2d.<locals>.<lambda>  r  r.   r   r  rB  r  r   )rB   rS   r   r  r   r  r   r<   rp  r   rm   re   r   r   )	r   r	  scales_hscales_wr  r   r   r?   
n_channelss	   `        r,   upsample_nearest2dr  ~  s     
LA/

QRR0@AA____   -

kA   __-..F /66M  +Az1a|F""zA~~/];;FMr.   r	  r  r  r  c                 v    t          ||d          t          j         j        dk     fd           t	          d          D ]9t          j                                                k     fd           :                     |                              t          j	                             S )Nr   r  r  c                      d j          S )NzFExpected grad_output to be a tensor of dimension 4 but got: dimension r  r0  s   r,   rQ   z-upsample_nearest2d_backward.<locals>.<lambda>  s    kYdYikk r.   c            
      R    d d           d d                                S )NzCExpected grad_output to have the same shape as output; output.size(z) = z but got grad_output.size(r   )r  r  r   s   r,   rQ   z-upsample_nearest2d_backward.<locals>.<lambda>  s]    J !J J'7':J J-.J J4?4D4DQ4G4GJ J r.   r   )
r  rB   rS   r   r   r   r   r   r<   rp  )r  r	  r  r  r  r  r   s   `    @@r,   upsample_nearest2d_backwardr    s     -K!   
LAkkkk   1XX 
 
Q#3A#66     	
 	
 	
 	
   ,,//1+>> 0   r.   c                 p    t          j                                         dk    p(t                                           dd                     fd           t                                           |d          }                     |                              t          j	                             S )Nr   r   c                  2    d                                   S )Nz>Non-empty 5D data tensor expected but got a tensor with sizes r   r!  s   r,   rQ   z$upsample_nearest3d.<locals>.<lambda>  r  r.   r   r  r   r  )r   r	  scales_dr  r  r  s   `     r,   upsample_nearest3dr#    r  r.   c                    t          j        |           t          j        | t           j                  }}||t          |t                    sJ t          |t                    sJ |j        }|                                }	t          ||          }t          ||          }|                    ||	           |                    ||	           t          ||           t          ||           ||fS ||fS )Nrz   )r^  r_  )
rB   r   r   r\   r   r   r   r   r  r   )
r   stablerp   
descendingr   r   r  r   r   
out_strides
             r,   	meta_sortr(    s     D!!5#3D#L#L#LqAg1&*-----':..... G	XXZZ
"6955#GY779j111Iz222F3333G4444wa4Kr.   c           	          t          j         j        dk     fd           t          j         j        j        k     fd                                d          wt          j        j        dk    fd           t          j                                        k    fd           t          j        j        j        k    fd           t          j        j        dk    fd                                d	          z  z  t          j                                        k     fd
           t          j        t           fdfD                       d            d S )Nr   c                       j          dS Nz != 2r  )input_gatess   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>      ;3C0J0J0J r.   c                  &    j          d j          S N != r  )hidden_gatesr,  s   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>  s    ;$>>,*<>> r.   r   c                       j          dS )Nz != 1r  )
input_biass   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>  s    jo3L3L3L r.   c                  6                                      d  S r/  r  )
gates_sizer3  s   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>  s!    z''));;z;; r.   c                  &    j          d j          S r/  r  )hidden_biasr3  s   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>  s    z'@@[->@@ r.   c                       j          dS r+  r  )prev_hiddens   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>  r-  r.   r   c            
      p                                      d                    d           d d d  d
S )Nr0  r   z * z // z (aka rd   )r   r   )expected_prev_hidden_numelfactorr5  r,  r9  s   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>  sL    ;$$&&K,<,<Q,?,?JTZb| r.   c              3   8   K   | ]}|j         j         k    V  d S r)   rw  )r8   r9   r,  s     r,   r_   z&rnn_cell_checkSizes.<locals>.<genexpr>  sA       
 
 H**
 
 
 
 
 
r.   c                      dS )Nz%expected all inputs to be same devicer2   r2   r.   r,   rQ   z%rnn_cell_checkSizes.<locals>.<lambda>	  s    7 r.   )rB   rS   r   r   r   r   rb  )r,  r1  r3  r7  r<  r9  r;  r5  s   ``````@@r,   rnn_cell_checkSizesr?    s    
L!Q&(J(J(J(JKKK	L\//>>>>>   !!!$$JZ_)+L+L+L+LMMM*,;;;;;	
 	
 	
 	 11@@@@@	
 	
 	
 
L!Q&(J(J(J(JKKK!,!1!1!!4!4z!AV!K	L99   
L 
 
 
 
"J[I
 
 
 	
 	
 	87    r.   c                     t          | |||d|           t          j        | t          j                  }t          j        |t          j                  }t          j        |t          j                  }|||fS )Nr  r   )r?  rB   r   r   )r,  r1  cxr3  r7  	workspacehycys           r,   _thnn_fused_lstm_cell_metarE    sq     \:{ArRRR E<STTTI		"E,C	D	D	DB		"E,C	D	D	DBIr.   c                 t   t          |          dk    }|r%t          |          }|d         }| j        d         }n:|
r| j        d         n| j        d         }|
r| j        d         n| j        d         }d}|rdnd}|dk    r|n|}|r|||z  g}n|
r||||z  gn||||z  g}|                     |          }|	|z  ||g}|t          j        d| j                  }n|                    |          }|                    |	|z  ||g          }|rdnd}|                     |t          j                  }|||||fS )Nr   r   r   r   rw  rz   )r   r   r   rB   ru   rm   r  )r   r  weight_stride0
weight_bufhxrA  r*  hidden_size	proj_size
num_layersbatch_firstdropouttrainbidirectionalbatch_sizesdropout_stateis_input_packed
seq_length
mini_batchbatch_sizes_sumnum_directionsout_sizer   r   
cell_shaperD  rC  reserve_shapereserves                                r,   
_cudnn_rnnr\    s   & +&&!+O %%
 ^
+a.'2FU[^^A
'2FU[^^A
'.QQQN%NNyyH 
$h&?@		 EZX%>??j(^*CD 	
 __Y''F~-z;GJ	z[5<000\\*%%	zN2JI	J	JB %AAAMoom5;o??G2r7J..r.   c                    |r| j         d         n| j         d         }|r| j         d         n| j         d         }|
}|r|||gn|||g}|                     |          }|t          j        d| j                  }n|                    |j                   }|t          j        d| j                  }n|                    |j                   }t          j        d| j        t          j                  }||||fS )Nr   r   rw  r   )r   r   rB   ru   rm   r  )r   w0w1w2w3hx_cx_r   rQ  r*  rJ  rL  
has_biasesrP  rM  rO  rT  rU  output_chanelsr   r   rC  rD  rB  s                           r,   mkldnn_rnn_layerrf  T  s    & $/BQEKNJ#.BQEKNJ N 	6Z00*n5 
 __Y''F
{[5<000]]39%%
{[5<000]]39%%Ael%+FFFI2r9$$r.   c                     | j         dk    r%t          j        dk    pdk    fd           d S t          j        |                               dk    fd           d S )Nr   r   c                       d  S )Nz4: Expected reduction dim -1 or 0 for scalar but got r2   rp   r  s   r,   rQ   z'zero_numel_check_dims.<locals>.<lambda>  s    wYYTWYY r.   c                       d  dS )Nz: Expected reduction dim z to have non-zero size.r2   ri  s   r,   rQ   z'zero_numel_check_dims.<locals>.<lambda>  s    wUUUUU r.   )r   rB   r   r   )r   rp   r  s    ``r,   zero_numel_check_dimsrk  |  s    yA~~1H!r	YYYYY	
 	
 	
 	
 	

 	IIcNNaUUUUU	
 	
 	
 	
 	
r.   c                      |5t          ||                                          }t          ||            d S t          j        |                                dk     fd           d S )Nr   c                        dS )Nz@: Expected reduction dim to be specified for input.numel() == 0.r2   r  s   r,   rQ   z%check_argmax_argmin.<locals>.<lambda>  s    t]]] r.   )r   rp   rk  rB   rS   r   )r  r   rp   s   `  r,   check_argmax_argminrn    sq    
S$((**--dC.....JJLLA]]]]	
 	
 	
 	
 	
r.   c                     t          d| |           t          j        | j        ||fnd           }t	          | ||          }|                     |t          j                  S )Nargmaxrz   )rn  r<   rX  r   rY  r   rB   r   )r   rp   r[  rV  r   s        r,   argmax_argmin_metarq    s[    $,,,
coSFF4PPD$T499E>>%u{>333r.   c                 4    t          j        d||||          S )Nr2   r   r   )rs  rJ   rl   rm   rn   s        r,   scalar_tensorrs    s&    ;
%v*   r.   c                    t          ||                                 d          }|                                 dk    rdn|                     |          }t          j        |dk    o||k    d            t          | j                  }t          |          dk    r|||<   |                     |          |                     |t          j	                  fS )NTr'  r   r   c                      dS )Nzk not in range for dimensionr2   r2   r.   r,   rQ   ztopk_meta.<locals>.<lambda>  s    4R r.   rz   )
r   rp   r   rB   rS   r   r   r   r   r   )r   r1  rp   largestsorted	sliceSizetopKSizes          r,   	topk_metar{    s     dhhjjd
;
;
;CXXZZ1__$))C..I	La*AN,R,RSSSDJH
8}}q>>(##T^^HEK^%P%PPPr.   c                    t          |                                 d          |                                 dk    r|                               nd}t          j        |dk    o||k    fd           t          | j        d          | j        dz   d          z             }|r.|                                 dk    r|                    d           |                     |          |                     |t          j	                  fS )NTru  r   r   c                      d  S )Nz9kthvalue(): selected number k out of range for dimension r2   r   s   r,   rQ   zkthvalue_meta.<locals>.<lambda>  s    QCQQ r.   rz   )
r   rp   r   rB   rS   r   r   r  r   r   )r   r1  rp   r[  dimSizer   s     `   r,   kthvalue_metar    s     dhhjjd
;
;
;C $

QdiinnnAG	L	Q1<QQQQ  
 DSD!DJsQwyy$99::E 488::>>S!>>%  $..ek."J"JJJr.   c                    | | n|}t          j        |                                dk    d            |                                }| ,t          j        |                                 |k    d            |,t          j        |                                |k    d            t          j        |                                |k    d            t          j        |                                |k    d            t          j        |                                dk    d            t          j        |                                |d         |d	         z  d
z  k    d            d S )Nr   c                      dS N r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>  s    " r.   c                      dS r  r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>       r.   c                      dS r  r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>       r.   c                      dS r  r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>  r  r.   c                      dS r  r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>  s    r r.   r   r   r  c                      dS r  r2   r2   r.   r,   rQ   z(checkLSTMBackwardSizes.<locals>.<lambda>  s    R r.   )rB   rS   rp   r   r   )grad_hygrad_cyrA  rD  rB  defined_gradexp_sizes          r,   checkLSTMBackwardSizesr    s2   %177wL	L!!##q(**555  ""HW\\^^x/<<<W\\^^x/<<<	Lh&

333	Lh&

333	LA%zz222	L""hqkHQK&?!&CCZZPPPPPr.   c                     | |dS t          | ||||           t          j        |t                    }t          j        |t                    }|r|                    dd          nd }|||fS )NNNNr   r   F)r[  )r  rB   r   legacy_contiguous_memory_formatr<  )	r  r  rA  rD  rB  has_bias
grad_gatesgrad_cxr  s	            r,   #_thnn_fused_lstm_cell_backward_implr    s    7?7GRY???!!@  J r1PQQQG4<F
q%000$Iw	))r.   c                 `   d }d }d }|d         r'|                     |                                           }|d         s|d         re|                     |                    d          |                     d          f          }|                     |                    d                    }|||fS )Nr   r   r   r   r_  )ra  r`  rb  re  r  grad_weightr  s          r,   linear_backwardr    s    JKI1~ ;!++FKKMM::
1~ BQ B",,l.?.?.C.CV[[QS__-UVV **<+<+<R+@+@AA	Y//r.   c                     t           j                  dk    r j        d         ||z  z  dk    sJ d j         d|             d  fd} j        d         ||z  z  } j        d         |z  } j        d	         |z  }g  j        d d         |||R }                     |          }|                     |            
          }|S )Nr   r  r   z'Invalid input shape for pixel_shuffle: z with upscale_factor = c                 \    t           j                            |           t           j        k    S r)   rn  rr  s    r,   rt  z,meta_pixel_shuffle.<locals>.is_channels_last  s!    "88==ATTTr.   c                  $               r+t                    dk    rt          j        S t          j        S                     t          j                  rt          j        S                     t          j                  rt          j        S d S rw  )rC  rB   r   rq  rp  rx  )rt  r   s   r,   ry  z.meta_pixel_shuffle.<locals>.pick_memory_format  s    D!! 	)4  F**..**e.EFF 	)**e.CDD 	)((	) 	)r.   r  r   r   )r   r   r   r   )	r   upscale_factorry  r  HrWrr   r   rt  s	   `       @r,   meta_pixel_shuffler    s    	DJ!
2.>2Q RVW W W WdddTbdd !X W WU U U	) 	) 	) 	) 	) 	) 	
2>N:;A	B.	(B	B.	(B-$*SbS/-1-b-"--I
..
#
#C
&&1133&
4
4CJr.   c                 L   |                      | j                  }|                     |j                  }|                     |j                  }|                     |j                  }|                     |j                  }|                     |j                  }|||||||fS r)   r  )r   weight0weight1weight2weight3rb  cx_tmpr   hy_cy_grad_output_r_optgrad_hy_r_optgrad_cy_r_optr   r*  rJ  rL  rd  rO  rP  rQ  rM  rB  diff_xdiff_hxdiff_cxdiff_w1diff_w2diff_bs                                r,   mkldnn_rnn_layer_backwardr    s    4 __U[))FmmCI&&Gv|,,G..G..Gw}--F7GVVWgEEr.   )	out_int32r   c                    t          j        | |rt           j        nt           j                                                  S rV  )rB   r   r  r   r   )r   
boundariesr  r   s       r,   meta_bucketizer  0  s9     9=EKK%+  jllr.   d   c                 B    dt                     dk    r*t          j                                          fd           t          j        t	          t
                    fd           t          j        dk    fd           t          j        t	          t                    fd           t          j        t	          t                    fd           t          j        k    d	            t          j         j         j	        
          S )Nzhistc()r  c                      d j          dS )Nz%"histogram_cpu" not implemented for ''rz   r!  s   r,   rQ   zmeta_histc.<locals>.<lambda>?  s    LekLLL r.   c                  ,     dt                      S )Nz#: argument 'bins' must be int, not rw  binsr  s   r,   rQ   zmeta_histc.<locals>.<lambda>C  s    7KKtDzzKK r.   r   c                       d  S )Nz: bins must be > 0, but got r2   r  s   r,   rQ   zmeta_histc.<locals>.<lambda>E  s    g#Q#Q4#Q#Q r.   c                  ,      dt                     S )Nz%: argument 'min' must be Number, not rw  )r  r}  s   r,   rQ   zmeta_histc.<locals>.<lambda>H      7LLcLL r.   c                  ,      dt                     S )Nz%: argument 'max' must be Number, not rw  )r  r~  s   r,   rQ   zmeta_histc.<locals>.<lambda>L  r  r.   c                      dS )Nz&{fn_name}: max must be larger than minr2   r2   r.   r,   rQ   zmeta_histc.<locals>.<lambda>N  s    %M r.   r   )
rC  rB   rS   r   r\   r   r   ru   rm   rJ   )r   r  r}  r~  r  s   ````@r,   
meta_histcr  8  s@    G5U""##%%LLLL	
 	
 	
 
L4!!KKKKK   
LQQQQQRRR	L3LLLLL   
L3LLLLL   
LMMNNN;tELDDDDr.   c                     t                                           |d          }t          j                                         dk    p2t          d                                  dd          D                        fd                                |                              t          j	                             S )Nr   r  r   c              3   "   K   | ]
}|d k    V  dS r  r2   )r8   r   s     r,   r_   z,meta_upsample_bimode2d_aa.<locals>.<genexpr>`  s&      !H!Ht$(!H!H!H!H!H!Hr.   r   c                  2    d                                   S r  r   r!  s   r,   rQ   z+meta_upsample_bimode2d_aa.<locals>.<lambda>a  r  r.   r   )
r  r   rB   rS   r   rb  r   r   r<   rp  )r   r	  r  r  r  r  s   `     r,   meta_upsample_bimode2d_aar  R  s     -

kA   
LHc!H!Huzz||ABB7G!H!H!HHH____   ??+,,//1%88 0   r.   c                 6   t          j        |                                dk    d            t          j        |                                dk    d            t          j        |j        j        d            t          j        |j        j        d            d S )Nr   c                      dS )Nz%found_inf must be a 1-element tensor.r2   r2   r.   r,   rQ   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>l      (O r.   c                      dS )Nz%inv_scale must be a 1-element tensor.r2   r2   r.   r,   rQ   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>o  r  r.   c                      dS )Nz!found_inf must be a float tensor.r2   r2   r.   r,   rQ   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>s      3 r.   c                      dS )Nz!inv_scale must be a float tensor.r2   r2   r.   r,   rQ   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>w  r  r.   )rB   rS   r   rJ   r   )r   rs  	inv_scales      r,   *_amp_foreach_non_finite_check_and_unscale_r  i  s    	LQ O O   
LQ O O   
L)33   
L)33    r.   c                 n    t          |                                           }|                     |          S r)   )r   r   r   )r   nanposinfneginfrA  s        r,   
nan_to_numr  |  s+     tyy{{##K>>+&&&r.   c                     | j         t          j        t          j        t          j        t          j        hvsJ d| j          d            | j        }t          ||          }t          ||          }||k    r| S t          | 	                                          }t          | 
                                          }||         ||         c||<   ||<   ||         ||         c||<   ||<   |                     ||           | S )Nz>torch.transpose_: in-place transposition is not supported for z layout)rl   rB   r  
sparse_cscr  
sparse_bscr   r   r   r   r   r  )r   dim0r$  ndimsr   r   s         r,   rs  rs    s	    		
	
 	
 	
 ]\\\	
 	
 	
 IE$&&D$&&Dt||		D$++--  F!'vd|F4L&,!$ZdDJT
T6"""Kr.   c                 (   | j         }| j        rF|                                 }|                                 }|dk    r|dk    sJ d| d| d            n&|                                 dk    sJ d| d            t          | d|dk     rdnd          S )	Nr   r   zEt_ expects a tensor with <= 2 sparse and 0 dense dimensions, but got z sparse and z dense dimensionsz6t_ expects a tensor with <= 2 dimensions, but self is r  r   )r   r  r  r  rp   rs  )r   r  r  r  s       r,   t_r    s    IE~ 	M__&&
NN$$	!OO	Q HS]  H  Hkt  H  H  H !/ HHJJ!OOOLELLL OO dAEAIIqq1555r.   )r  r   sidesorterc                    |rt           j        nt           j        }t          |t           j                  r(t          j        ||                                          S t          j        d|| j                  S )Nrz   r2   r  )	rB   r  r   r\   r   r   r   ru   rm   )sorted_sequencer   r  r   r  r  rJ   s          r,   meta_searchsortedr    sg     %5EKK%+E$%% KE222==???{2U?3IJJJJr.   c                      t          j         t           j        t           j        t           j        t           j        fv fd           d S )Nc                      d  S )Nz/Unsupported input type encountered for isin(): r2   rz   s   r,   rQ   z3_check_for_unsupported_isin_dtype.<locals>.<lambda>  s    I%II r.   )rB   rS   r:  r.  
complex128	complex64rz   s   `r,   !_check_for_unsupported_isin_dtyper    sG    	Lej%.%2BEOTTIIII    r.   c                 |    |	r#t                               | ||||||||
|
  
        S t          | ||||||||
|
  
        S r)   )r#   _embedding_bag_sparse_backward!meta_embedding_bag_dense_backward)r  r   rD  r  r  maximum_indicesnum_weightsr  r*  r  r  r  s               r,   meta_embedding_bag_backwardr    st      
22
 
 	
 1
 
 	
r.   c
                 P    t          j         j        t           j        t           j        t           j        t           j        fv  fd           t          d          \  }
}}||k    rt          j        |d u                                | 	                    d          f          }|S )Nc                      d j          S )Nz$Unsupported input type encountered: rz   )r  s   r,   rQ   z3meta_embedding_bag_dense_backward.<locals>.<lambda>  s    CtzCC r.   r   r   )
rB   rS   rJ   r-  r.  r,  float64r   r   r   )r  r   r  r  r  r  r  r*  r  r  r  r  r  index_grad_weights   `             r,   r  r    s     
L
u}enemU]SSCCCC   %*!HH!Hix_D0111TYYq\\'BCCr.   c                 0   t          d          \  }}}	|                     d          }
t          j        ||k    d           t          j        |                                 dk               t          j        |                                dk               |                    d          }t          j        |                                dk               t          j        |                    d          |
k               |                     |f          }|S )Nr   r   zHembedding_bag_backward: per_sample_weights only supported for mode='sum'r   r   )r   r   rB   rS   rp   r   )r  r  r   rD  r  r*  r  r  r  r  embedding_featuresr  r   s                r,   .meta_embedding_bag_per_sample_weights_backwardr    s     %*!HH!Hi1	LR   
Lq!!!	L!#$$$,,q//K	L"###	LQ#55666^^[N++FMr.   )assume_uniqueinvertc                   t          j        t          | t                    pt          |t                    d            t          | t                    st          j        | |j                  } t          |t                    st          j        || j                  }t          | j                   t          |j                   t          j        | t           j	                  S )Nc                      dS )Nz<At least one of elements and test_elements must be a Tensor.r2   r2   r.   r,   rQ   zmeta_isin.<locals>.<lambda>-  r  r.   rw  rz   )
rB   rS   r\   r   r  rm   r  rJ   r   r:  )elementstest_elementsr  r  s       r,   	meta_isinr  (  s     
L8V$$I
=&(I(INN   h'' G<1EFFFmV,, L]8?KKK%hn555%m&9:::HEJ7777r.   r   c                     t          j        | dk    d            t          |t          j                  \  }}t          j        ||          S )Nr   c                      dS )Nz,polygamma(n, x) does not support negative n.r2   r2   r.   r,   rQ   z meta_polygamma.<locals>.<lambda>=  s    !O r.   rc  rz   )rB   rS   r   r   rd  r   )r   r   r?   r:   s       r,   meta_polygammar  :  sW     
LaOOPPP(;H  OA| D5555r.   c                      t          d          )Nz.Tensor.item() cannot be called on meta tensors)r   r   s    r,   meta_local_scalar_denser   E  s    
G
H
HHr.   max_lengthspadding_valuec                     t          |          dk    sJ t          |          dk    sJ |d         j        d         dz
  }|d         }||g| j        dd          R }|                     |          S r  )r   r   r   )r   rD  r  r  r  rH  r3  s          r,   $meta__jagged_to_padded_dense_forwardr  J  s     w<<1{q    
aAAAq,6<+,,LL)))r.   paddedtotal_Lc                 b   t          |          dk    sJ |sqt          | t          j        j                  sJ | j        j        }|J |                                }t          j        j	        j
                            |dd            |g| j        dd          R }|                     |          S )Nr   r   r|  r   )r   r\   rB   rZ  r[  	fake_mode	shape_envcreate_unbacked_symintfxexperimentalsymbolic_shapesr  r   r   )r  rD  r  r	  r3  s        r,   $meta__padded_dense_to_jagged_forwardr  [  s     w<<1 
&%"3">?????$.	$$$2244-GG 	H 	
 	
 	
 /fl122.//LL)))r.   c                 ^    t          |           t                      d                         }|S )Nc                 8    t          | t          j                  S r  r@   r   rd  r  s    r,   _fz)_create_unary_float_meta_func.<locals>._fr  s$      =J
 
 
 	
r.   r3   r   funcr  s     r,   _create_unary_float_meta_funcr  q  9    4]]
 
 ] 

 Ir.   c                 ^    t          |           t                      d                         }|S )Nc                 :    t          | |t          j                  S r  r  )r9   r^  s     r,   r  z*_create_binary_float_meta_func.<locals>._f}  s&      q!@!M
 
 
 	
r.   r  r  s     r,   _create_binary_float_meta_funcr  |  r  r.   c                     i } dD ]#}t           |         }|D ]}|| vr||         | |<   $|                                 D ]\  }}t          |t          j        j                  r&t          |t                    sJ  |                    t          j        j	        j
                  |           t          j                            |                                d          r"|t           d         v rt          | d          |j        r|                                dv rd|                                v rt                              ||           d|                                v rt"                              ||           Hd|                                v rt$                              ||           {d	|                                v rt&                              ||           t(                              ||           d S )
N)rk   post_autogradpre_autogradCompositeImplicitAutogradrk   z is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.>   aten::cloneaten::copy_aten::rot90aten::_to_copyaten::empty_stridedaten::constant_pad_ndaten::as_strided_scatterzmkldnn::zmkl::zonednn::zquantized::)r   itemsr\   rB   _opsHigherOrderOperatorr   py_impl_CDispatchKeyr%   %_dispatch_has_kernel_for_dispatch_keyr  r   is_view2_meta_lib_dont_use_me_use_register_meta_for_mkldnnimpl/_meta_lib_dont_use_me_use_register_meta_for_mkl2_meta_lib_dont_use_me_use_register_meta_for_onednn5_meta_lib_dont_use_me_use_register_meta_for_quantized'_meta_lib_dont_use_me_use_register_meta)activate_meta_tablere   registryopoop_overloadr+   s         r,   activate_metar8    sb    : 9 9-d3 	9 	9C---+3C=#C(	9 /4466 6N 6NR
 k5:#ABB 	+z222226EH0566r:::899 ;
 
 +	N 8@@@"" ; ; ;   A   	N 	 	 [--////BGGUWXXXXK,,....?DD[RTUUUU{//1111BGGUWXXXX+"2"2"4"444EJJ    8<<["MMMMm6N 6Nr.   r   r  r)   )NNNF)NN)Tr  )r4  )r  T)FF)TT)r,  )FTN)TFF)TF)r   )r-  N)r9  rX  )r2   r   r  F)r2   r   FTN)Fr   FNFr   )NF)r   F)NNNNN)r   NNr   )NNF)rp  FFN)rp  FNN)rp  FN)FN)FNNNN)Nr   FNN)NNNN)r   TT)r  r   r   )r   )rp  (F  r  enumr   typingr   r   r   r   r   rB   torch._prims_commonro  r<   r	   r
   r   torch._decompr   r   r   r   
torch._opsr   torch._primsr   r   r   r   r   r   r   r   r   r   torch._prims_common.wrappersr   r   r   r   r   rT  r    r!   torch.utilsr"   r/   opsr#   libraryLibraryr3  r3   r@   rL   rU   linspacelogspacer~  rw   taker   r   r   r   r   cummaxcumminr   r   r   _fft_c2cr   _fft_r2cr   randpermgenerator_outr   r~   r   randintr   r   low_outr   randr   _fft_c2rr   rh  r  r
  
unsqueeze_r  _sparse_semi_structured_linearrX  rJ   r  _sparse_semi_structured_mmr   _sparse_semi_structured_addmmr$  _cslt_sparse_mmr:  r5  index_reducer   r=  index_reduce_r?  index_selectrB  segment_reducerO  r~  	unary_outrS  rp   r\  r}  r_  ra  ri  re  rj  _assert_asyncrm  msgrq  _printrt  _make_dep_tokenrx  r  _functional_sym_constrain_ranger  r  (_functional_sym_constrain_range_for_sizer  _functional_assert_asyncr  r   r  r   r  r  r  r  _linalg_eighr  r  _linalg_eigvalslinalg_eigvalsr  
linalg_eigr  r  r  r  r  r  r  r  linalg_inv_exr  linalg_ldl_factor_exr  linalg_ldl_solver  	linalg_lur  linalg_lu_factor_exr  linalg_lu_solver#  	lu_unpackr)  r2  	linalg_qrr9  r=  r:  _linalg_svdrI  r  r  rY  rm  linalg_solve_triangularrv  r|  r  _linalg_detr  r  r  r  reflection_pad1dr  replication_pad1dr  r  reflection_pad1d_backwardr  replication_pad1d_backwardr  r  reflection_pad2dr  replication_pad2dr  reflection_pad2d_backwardr  replication_pad2d_backwardr  r  reflection_pad3dr   replication_pad3dr  reflection_pad3d_backwardreplication_pad3d_backwardr	  _pdist_forwardrF   r  _pdist_backwardr  baddbmmr(  	bernoullir,  
bernoulli_r0  r
  r3  poissonr6  _fused_moving_avg_obs_fq_helperrH  mmrS  rY  rC  rl  rt  convolutionr}  r*  _has_mkldnnr.  r~  _convolution_pointwiser  _linear_pointwiser  has_mklr0  r  _mkl_linearr  r1  r  qconv2d_pointwiser  qlinear_pointwiser  r  r2  r  
max_pool2dr  r  
avg_pool2dr  r  avg_pool2d_backwardr  
avg_pool3dr  avg_pool3d_backwardr  _adaptive_avg_pool2dr
  _adaptive_avg_pool3dr  _adaptive_avg_pool2d_backwardr  _adaptive_avg_pool3d_backwardr  r  adaptive_max_pool2dr-  r1  r3  adaptive_max_pool3dr:  r<  r=  repeat_interleaver@  r]   rD  rG  r|   _unsafe_indexr]  convolution_backwardri  addbmmrq  _fused_adam__fused_adamw_r  _fused_adamr  _int_mmr  _convert_weight_to_int4packr  _weight_int4pack_mmr  _weight_int8pack_mmr  _cdist_forwardr  _cdist_backwardr  _embedding_bagr  _embedding_bag_forward_onlyr   r  nansumr  median	nanmedianr  
dim_valuesr*  r   r
  logical_not_r  repeatr  zero_r  mul_Scalardiv_logical_and_logical_or_logical_xor_r  add_sub_r  rounddecimalsr   r&  
__rshift__r*  
__lshift__r.  zeror0  r  r3  fillr5  relu_r8  	index_put_unsafe_index_putr<  masked_fill_r?  _masked_scalerB  masked_scatter_rF  masked_scatterrH  masked_scatter_backwardrJ  
index_put_rL  aliasrO  rZ  bmmr\  r`  re  r  r  r  r  r  r   max_pool2d_with_indices_backwardr  max_pool2d_with_indicesr  fractional_max_pool2dr  max_unpool2dr  r  max_unpool3dr  max_pool3d_with_indicesr   max_pool3d_with_indices_backwardr  r  r  r  grid_sampler_2d_backwardr  r  r
  r  r  rS  r  select_scatterr!  slice_scatterr%  r   r*  r2  gatherr:  rK  rR  rT  r^  ra  scatter_addrf  scatter_add_ri  r`  r   r>  r8  value_reducerm  scatter_ro  #_scaled_dot_product_flash_attentionr  #_scaled_dot_product_cudnn_attentionr  ,_scaled_dot_product_flash_attention_backwardr  +_scaled_dot_product_flash_attention_for_cpur  4_scaled_dot_product_flash_attention_for_cpu_backwardr  '_scaled_dot_product_efficient_attentionr  0_scaled_dot_product_efficient_attention_backwardr  ,_scaled_dot_product_cudnn_attention_backwardr  _flash_attention_forwardr  _flash_attention_backwardr  _efficient_attention_forwardr  _efficient_attention_backwardSymIntr  
_scaled_mmr  scatter_reducetwotwo_outr  scatter_reduce_r  multinomialr  r  r  r  _upsample_nearest_exact1dr  _upsample_nearest_exact2dr  "_upsample_nearest_exact2d_backwardr#  _upsample_nearest_exact3dr   r%  values_stabler(  r?  _thnn_fused_lstm_cellrE  r\  rf  rk  rn  rp  argminrq  rs  topkr{  kthvaluer  r   r  r  r  r  pixel_shuffler  r  	bucketize
Tensor_outr  histcr  _upsample_bilinear2d_aa_upsample_bicubic2d_aar  r  r  rs  r  searchsortedr  r  _embedding_bag_backwardr  _embedding_bag_dense_backwardr  *_embedding_bag_per_sample_weights_backwardr  isinr  	polygammar  _local_scalar_denser   _jagged_to_padded_dense_forwardr  _padded_dense_to_jagged_forwardr  r  r  special_airy_aispecial_bessel_y0special_bessel_y1special_modified_bessel_i0special_modified_bessel_i1special_modified_bessel_k0special_modified_bessel_k1!special_scaled_modified_bessel_k0!special_scaled_modified_bessel_k1special_chebyshev_polynomial_tspecial_chebyshev_polynomial_uspecial_chebyshev_polynomial_vspecial_chebyshev_polynomial_w&special_shifted_chebyshev_polynomial_t&special_shifted_chebyshev_polynomial_u&special_shifted_chebyshev_polynomial_v&special_shifted_chebyshev_polynomial_wspecial_hermite_polynomial_hspecial_hermite_polynomial_hespecial_laguerre_polynomial_lspecial_legendre_polynomial_ptorch._refs.nn.functionaltorch._refs.specialr8  r2   r.   r,   <module>r	     sN          9 9 9 9 9 9 9 9 9 9 9 9 9 9  # # # # # # + + + + + + + + + +            " ! ! ! ! ! U U U U U U U U	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	              < ; ; ; ; ; ; ; ) ) ) ) ) ) y~*/-*?*?PV*W*W '
 
 
3   (* * *   t}-..
 

=5 5 5  /.5p 	!49=122' '  32' !)4+<+@ABB%' % % % %  CB%$ t%&&I I  '&I 	[$+/4;+>P  Xy!!  "!  !)4+<+@ABB/ /  CB/1H 1H 1Hl %t}'89::   ;:" %t}'89::   ;: t}*++"& 3 3 3 3 ,+3 t}$%% *
 
 
 
 &%
 $dl&6788
 *     98  $,"6788 *     98 	!49=122%)$tPT      32 %t}'89::K K  ;:K tz!""   #"0( ( ( t&''  (' t233
 "%)'+   6
	
 c] $   43B t.//
 (,	 
  $	   0/0 t122 	
'+  
  	 $   32D t#$$ ""'+"! !,!\! 6
! F	!
 $! ! ! ! %$!H t ()) 	I 	I 	I
	I		I 	I L		I
 	I 	I 	I 	I 	I *)	I t!)** 	 	 	
			 	 L		
 	 	 	 	 	 +*	 t ())' ' *) ' t"*++
 !% $ $ W  W  W
 W W f	 W
 f W f W  W  W  W  W  W ,+ WF  $("4566   76 tx|     $("4566   76 tx|    tz!""6 6 #"6 tz~( ( (
 t!)**  +* t!%&&  '& t{"##  $# t#+,, ) ) ) ) -,) t'/00, , , 10, t3;<<   =<
 t08996 6 6 :96 t<DEE  FE
 t,011  21
sF sC s s s s   F  #        N (,
 


 !%
 
 
 
$ V S C    
 
F 
$ 
 
 
 
 
"  	   	   C     !)4+<+HIJJ]N++  s T    ,+ KJ" $,d.A.EFGGB B6 B B B  HGB  !!]N++	6 	 	 	 ,+ "!	Q QF Q Q Q Q t*++) )F )4 )F ) ) )  ,+) t"##J J JF J4 JF J J J  $#J t}) )6 )$ )6 ) ) )  ) t$%%) )6 )$ )6 ) ) )  &%) t&.// &  T    0/" 	$,d.M.QR  .f .6 .f . . .  .d t!)** &     +* )143L3PQRRT8V$$ 	  
  	
 666!"   %$ SR& %-t/D/HIJJ ' ' ''' '
 ' ' ' '  KJ'T &(:;<<S#s/3   f  fff>T8U     =<4 (0$2J2NOPPT8V$$ 	    	
 666!"   %$ QPD $,d.B.FGHH 4 4 444 4
 4 4 4 4 4  IH4n t~S#s 	$ $$$ $ 	$
 666!"$ $ $  $P tTz!2    * &(:;<<S# f C ffn8M     =<4 $,d.B.GHIIV[$11'v '%(F"G ' ' ' 21 JI'$ t'((   	" """ " SM	" " " )("J.
.
. 49d3i . . . .".
.
. 3-. 66>	. . . .(f V      t$%%
 ##!8 8 888 	8
 8 V8 	8 V8 6
8 6666)*8 8 8 &%8v ,4d6R6VWXX     	
   
&	    YX2 t$%%Z-.. +( +(
+(+( +( 	+(
 +( 66>+( +( +( /. &%+(^ t'((
 
 )(
 tz
 W WW	W W 	W
 W W W W  Wt  >#; #; #;L t$%%= =  &%= t%&&> >  '&>( ( (< t-..\S S  /.S t.//\T T  0/T2E 2E 2Ej t$%%= =  &%= t%&&> >  '&> &.&1'/'2	  \& &  &@<G <G <G~ t$%%= =  &%= t%&&> >  '&> &.&1'/'2	  \$( $(  $(N t"##

 

f 

 

v 

 

 

  $#

 t#$$Pv PV P Pf PQW P P P  %$P $dl&6788/0 ' ' ' '  98'2 &(:;<<&* / / / /  =</
 t$%%   &% t~  / / / ! /
 $dl&6788" " "  98" t3;<< * * * =<*. tw	 	  	B B B    7;b b,bLb $s)S.!b 49c>"	b
 DIsN#b b b U49c>23b b b bJQ Q Q t'((),)L) ,) I	)
 #Y) 3i) ) I) ) ) ) )()X 	8 K
9>9N9N&&: :6 ]59#:BCC  DC, ]59#5=>>S S ?>S
 x 	:?-:O:O66;
 ;
7 
uy}0	1	1	 	 
2	1	
 :?9N9N&&: :6 ]59#5=>>    ?> D ]59#5=>>]59#5<==  >= ?>, =BM<Q<QVV= =9 ]59&122 
 
 
 32
<   t&'' I I I ('IZ(< (< (<X t'/00E E 10EP t QJ QJ QJ   QJh t'((\K( K(  )(K(\ t(011  21" t(011@ @ 21@ t19::F F ;:F, t122\P P  32P
	
6 	
S 	
 	
 	
 	
 t'((UI+ +  )(+\ t011\H H  21H$ t'((UI' '  )('T t011\( (  21(
 t%,--* * * .-* $dl&6788T T  98T #+T-@-DEFF9; @ @ @# @3 @ @ @  GF@ 
!4#5#<=>>`J `J ?>`JF )1233H H 43H: #T[_566./q ' ' ' '  76'0 !)4+=+EFGG  !
 
 
 
 HG
2  ()**  !! ! ! ! +*!H ~B B  B* 0122  32& ()**@ @ +*@ ()**< < +*< t"*++ &  & ,+ &F t#$$G G  %$G* t"*++
 	
e5 e5 e5 ,+e5P t/7885 5 985
 
 
 
 #T[_566=$ = = = =  76= #T^%;<==) ) >=) !			 	 Xy!!   "!	 	 t ())  *) t{"##' ' $#' tz!""  #" 				! !
 
 
 
 					     
"DJ$7899  :9
 
 
" &(>?@@  A@ &(>?@@  A@ ty !!& & "!& 
!4:#4566  76 	 $)"2344" " 54" tz!""  #" &(>(FGHH" " " IH" t '((  )(
 t!)**  +* t#$$	 	 %$	 t"##6 6  $#6 t+,,! ! -,! t&''   (' tz!""! ! #"!   B tx  5 5 ! 5    6  ; ; ;h #-Z ZZZ 	Z 		Z
 	Z 	Z 	Z 	Z 	Z 	Z 	Z Z Z Z Z  !Z" #Z$ %Z& 'Z( )Z* +Z, -Z Z Z Zz;4 ;4 ;4|383838 38 		38
 	38 	38 	38 	38 	38 	38 	38 	38 38 38 38  !38" #38$ %38& '38 38 38 38lI2 I2 I2X t4<==( ( >=(V t+344 # # # 54#L t)122Q Q 32Qh t !!4 4  "!4n, , ,^ t !!   "!, t+,,UI d d d  -,dN t455\b b  65bJ%
V %
6 %
 %
 %
 %
P    t    v 3    $ t,455# # 65#$ t#$$8 8  %$8" t,--\;''! ! (' .-!, 	!"##. . $#. t&'' ) ) ) (')X t{E E  E8 t"*++. . ,+. t!)**. . . +*.
	 	 	C 	d 	 	 	 	/ / /
    t{"##' ' ' $#'   6
 
 
 
   
-
 -
 -
 -
b4 4 4 4 t'((& & )(&
 t !!  "!
 !	  & & &  & "	     89::
 #!5 55	5 5 	5
 5 5 E?5 5 5 ;:5p 89:: #!' ''	' ' 	'
 ' ' ' ' E?' ' ' ;:'T 9 ( "" """ 
" 	"
 
" " " " " " " " " " E?" " " 
". 8  "&! 	  	
   E?   
< A  #'!'" '"'"'" 
'" 	'"
 
'" '" '" '" '" E?'" '" '" 
'"T <=>> !$) $)$)	$) $) 	$)
 $) $) E?$) $) $) ?>$)N = " !4- 4-4-4- 
4- 	4-
 4- 
4- 4- 4- 4- 4- $Z4- 4- E?4- 4- 4- 
4-n 9 * "!" """ 
" 	"
 
" " " " " " " " " " "  E?!" " " 
"0 %   "&*'+"&%)9 99	9 9 	9
 9 9 9 9 9 9 E?9 sm9  }9 9 6"9 9 9 
9x & ( "&*'+#, ,,, 
, 	,
 
, , , , , , , , , , E?,  sm!,"  }#, , , 
,4 )    %!(,!%!%-S -S-S	-S -S 6
	-S
 6"-S 6"-S 3--S 3--S -S -S -S E?-S f%-S v-S #-S -S -S 
-S` * * "$("'%47 474747 
47 	47
 6
47 6"47 6"47 ,47 ,47 47 47 47 47 47 47  E?!47" SM#47$  %47 47 47 
47n '()) $(+/'+ [Y [Y
,[Y
,[Y \[Y \	[Y
 5<
 [Y 5<([Y $[Y [Y [Y [Y *)[Y| #')<)DEFF& & &  GF&
 t#'((   )(
  ($*:*>?@@	 	 	 	 	  A@	  , , ,* 	$d&D&LM 
 
 
 
 	$d&D&LM    . (0/7  !% $ %U\ 123 sEL012 uo	
 uo   : 	$d&D&LM 
 
 
 
 					    &$ $ $N t)122
    32 t&''4/ 4/ ('4/n t$,--$% $% .-$%N

 

 


 
 
 #T[%89::4 4 4 ;:4 t!)**   +* ty !!	Q 	Q 	Q "!	Q %t}';<==Xy!!K K K "! >=K #("9 Q Q Q t7?@@	* 	* A@	* t#+,,	0 	0 -,	0 t!)**  +*: t-566F F 76FD %t~'@ABB27u      CB 
|E E E  E0 	!)4+F+NO     ( t>FGG  HG$ ')<=>>' ' '  ?>'
 uy~())  *): uy~ !!6 6 "!6" t !!
 
	K K K K  "!K    t+,, '
 '
 '
 -,'
T t122    32. t>??    @?0 ty8=e 8 8 8 8  8  t~6c 6 6F 6 6 6  6 t'((I& I I I )(I t3;<<
 	* **&\* c* 	* * * =<*  t3;<< "* **&\* c]* * * =<**      d2 3 3 3  d4 5 5 5  d4 5 5 5  d= > > >  d= > > >  d= > > >  d= > > >  dD E E E  dD E E E  tB C C C  tB C C C  tB C C C  tB C C C  tJ K K K  tJ K K K  tJ K K K  tJ K K K  t@ A A A  tA B B B  tA B B B  tA B B B
                BN BN BNJ r.   