
    קgɢ             !       
r   U d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl mZmZ d dlm	Z	m
Z
 d dlmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmc mZ d dlm Z m!Z!m"Z" d dl#m$Z$ d dl%m&Z& d d	lm'Z'm(Z(m)Z)m*Z*m+Z+ d d
l,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z3 d dl4m5Z5 ej6        j7        Z7g Z8ee9         e:d<   ej;        j<        j=        Z= G d de          Z>	 ddedej?        de@fdZA eeAej?        jB        d          ZC eeAej?        jB                  ZD eeAej?        jE                  ZFde"deGde"fdZH e$e=jI                   e0d          eDde"de"fd                                    ZI e$e=jJ                   e0d          eDde"de"fd                                     ZJ e$e=jK                   e0d          eDde"de"d!eLd"eLfd#                                    ZK e$e=jM                   e0d          eDd$e"d%eLd&eLd'eLd(e@d)e"fd*                                    ZM e$e=jN        jO        g          d+             ZP e$e=jN        j"        g          d,e"fd-            ZQ e$e=jR                   e0            eDd.e"de"fd/                                    ZR e$e=jS                   e0d          eDd$e"d.e"fd0                                    ZS e$e=jT                   e0d          d$e"d.e"d1eLd2eLfd3                        ZT e$e=jU                   e0            eDd.e"de"fd4                                    ZU e$e=jV                   e0            eDd$e"d.e"de"fd5                                    ZV e$e=jW                   e0d          d$e"d.e"d"eLfd6                        ZW e$e=jX                   e0d          eDd$e"d.e"d7eLd8e@fd9                                    ZX e$e=jY                   e0d          eDdd;e"d.e"d<e9fd=                                    ZY e$e=jZ                  eDd$e"d>e"fd?                        ZZ e$e=j[                   e0            eDd.e"de"fd@                                    Z[ e$e=j\                   e0d          eDd$e"d.e"de"fdA                                    Z\ e$e=j]                  d.e"dBe"de"fdC            Z] e$e=j^                  d$e"d.e"dBe"dee"e"f         fdD            Z^ e$e=j_                  e=j_        j`        a                    e7jb                   e0            eD	 	 	 	 dd.e"dGe"dHeLdIeLdJe@dKeejc                 de"fdL                                                Z_ e$e=jd                  e=jd        j`        a                    e7jb                  eD	 	 	 	 dd.e"dGe"dHeLdIeLdJe@dKeejc                 de"fdM                                    Zd e$e=je                   e0            eDd$e"d.e"dGe"dHeLdIeLdJe@d8e@de"fdN                                    Ze e$e=jf                   e0d          eDd$e"d.e"dOe"de"fdP                                    ZfdQe"dReGfdSZgdTejh        fdUZi e$e=jj                   e0            eDe>jk        jl        fd.e"dVe"dReGde"fdW                                    Zj e$e=jm                   e0d          eDd$e"d>e"dVe"dReGfdX                                    Zm e$e=jn                  ddY            Zo e$e=jp                   e0            eDe>jk        jl        dZfd.e"dVe"dReGd!eLfd[                                    Zp e$e=jq        j`                  eDd$e"d.e"dVe"dReGd!eLf
d\                        Zq e$e=jq        jr                  eDd$e"d.e"dVe"dReGd!eLde"fd]                        Zs e$e=jt        j`                  eDd$e"d.e"dVe"dReGd^eLf
d_                        Zt e$e=jt        ju                  eDd$e"d.e"dVe"dReGd^eLde"fd`                        Zvd$e"d.e"dVe"dBee"         dReGdaeGdbe"de"fdcZw e$e=jx                   e0d          eDd$e"d.e"deGde"fdd                                    Zx e$e=jy                   e0d          d$e"d.e"dVe"dBee"         dReGdaeGdbe"de"fde                        Zy e$e=jz                   e0d          d$e"d.e"dVe"dBee"         dReGdaeGdbe"de"fdf                        Zz e$e=j{                   e0            eDde>jk        jl        fd.e"dVe"dBee"         dReGde"f
dg                                    Z{ e$e=j|                   e0d          eDde>jk        jl        fd$e"d.e"dVe"dBee"         dReGde"fdh                                    Z| e$e=j}                   e0            eDe>jk        jl        fd>e"dVe"dReGde"fdi                                    Z} e$e=j~                   e0d          eDe>jk        jl        fd$e"d.e"dVe"dReGde"f
dj                                    Z~ e$e=j                   e0            dd>e"dle"dmeLfdn                        Z e$e=j                   e0            doe"dpe"de"fdq                        Z e$e=j                   e0            d$e"dreeG         deGdseGdteGdueGfdv                        Z e$e=j        j"                  	 	 	 	 dd.e"deGdseeG         dteeG         dueGf
dx            Zde"deGdseeG         dteeG         deeGeGf         f
dyZ e$e=j                   e0            	 	 	 	 dd>e"dze"deGdseeG         dteeG         dueGfd{                        Z e$e=j                   e0            d$e"dreeG         deGd|eGfd}                        Z e$e=j                   e0            d$e"dreeG         d~eGdeGdeGf
d                        Zd$e"de"dejh        fdZ e$e=j                   e0d          eCd$e"de"deGdejh        fd                                    Z e$e=j                   e0            eCd$e"de"deGdejh        fd                                    Zd Z e$e=j                   e0            d>e"deeG         deeG         deeG         deeG         de"fd                        Z e$e=j                   e0            eDd>e"deeG         deeG         deeG         deeG         deeG         de"fd                                    Z e$e=j                   e0            d$e"de"d&eLfd                        Z e$e=j                   e0            d;e"deeG         deGdeGdueGde"fd                        Z e$e=j        j`                  eD	 dd$e"d.e"deeL         de"fd                        Z e$e=j                  e=j        j`        a                    e7j                  e=j        j`        a                    e7j                  d>e"dmeLdee@         fd                                    Z e$e=j                   e0dd          d>e"dmeLdee@         fd                        Z e$e=j                   e0            de"deGde@fd                        Z e$e=j                   e0            de"deGde@fd                        Z e$e=j                   e0            	 	 	 ddBe"de"deGde@de@de"fd                        Z e$e=j                   e0            d$e"de"deGdeGde@f
d                        ZdeeG         fdZdee"         deGdeGdee"         fdZdee"         fdZdee"         deGfdZdee"         deGdeGfdZ e$e=j        j`        e=j        ju        g          	 ddee"         deGdeGdee"         de"f
d            Z e$e=j                  	 dd.e"deeG         deGdee"         fd            Z e$e=j        j`        e=j        ju        g          	 	 dd.e"deeG         deGdeee"                  deee"                  f
d            Z e$e=j        j"                  dd>e"deGdeGdee"df         fd            Z e$e=j        j`                  	 dd>e"deeG         deGdee"df         fd            Z e$e=j        j"                  dd.e"deGdeGdee"df         fd            Ze=j        j        a                    e7j                  	 dd.e"de"deGdee"df         fd            Z e$e=j                   e0            eDdd.e"de"de"d!eGd%eGf
d                                    Z e$e=j                   e0            eD	 	 	 dd.e"de"de"d!eGd%eGde@fd                                    Z e$e=j                   e0            eDdd.e"de"de"d!eGd%eGf
d                                    Z e$e=j        j`                  eDd$e"d>e"de"de"dee"         deGdeGdeGdeGdee@         deee"         ee"         ee"         f         fdɄ                        Z e$e=j        ju                  d$e"d>e"de"de"dee"         deGdeGdeGdeGdee@         dej"        dej"        dej"        deee"         ee"         ee"         f         fd˄            Zdee"         dee"         fd̄Z e$e=j        j`                  de"d>e"deeG         de"de"dBee"         dee"         dee@         deee"         ee"         ee"         f         fdЄ            Z e$e=j        ju                  de"d>e"deeG         de"de"dBee"         dee"         dee@         dej"        dej"        dej"        deee"         ee"         ee"         f         fdф            Zd>e"dBee"         dee"         dee"         dee"         dJe@deLdeLde@dee"e"e"ee"         ee"         f         fdքZ e$e=j                   e0dddئ          d>e"dBee"         dee"         dee"         dee"         dJe@deLdeLdee"e"e"f         fdل                        Ze=j        j`        a                    e7j                  e=j        j`        a                    e7j                  d>e"dBee"         dee"         dee"         dee"         dJe@deLdeLdee"e"e"f         fdڄ                        Ze=j        j`        a                    e7j                  ddee"         fdۄ            Z e$e=j        j`                  d>e"dBee"         dee"         de"de"deLdeLdee"e"e"f         fd܄            Z e$e=j        j`                  d>e"dBee"         dee"         de"de"dJe@deLdeLdee"e"e"f         fd݄            Z e$e=j        j                  d>e"dBee"         dee"         dJe@deLdeLdee"e"e"f         fdބ            Z e$e=j        j`                  d>e"dBee"         dee"         de"de"dJe@deLdeLdee"e"e"e"e"f         fd߄            Zd>e"dBee"         dee"         de"de"deLdJe@de"fdZ e$e=j        j`                  d>e"dBee"         dee"         de"de"deLdeLdee"e"e"e"f         fd            Z e$e=j        j`                  d>e"dBee"         dee"         de"de"deLdeLdee"e"e"e"e"e"f         fd            Z e$e=j        j`                  d>e"dBee"         dee"         de"de"deLdeLdee"e"e"e"f         fd            Z e$e=j                   e0dd          eDdd                                    Z e$e=j                   e0            ddddddddee"e(f         dTeejh                 deej                 de@de@deej                 fd                        Z e$e=j        e=j        e=j        g           e0            d                         Ze=j        j`        a                    e7j                   e$e=j                   e0dddd          d>e"dBe"dee"         dee"         dee"         dJe@deLdeLfd                                    Zd Z e$e=j        j`                  de"d>e"dBee"         dee"         dee"         dee"         dee"         de@deLdee@         de"dee"ee"         ee"         f         fd            Z e$e=j        j`                  de"d>e"dBee"         dee"         dee"         dee"         dee"         de@deLdee@         dee"ee"         ee"         f         fd            Z e$e=j        ju                  de"d>e"dBee"         dee"         dee"         dee"         dee"         de@deLdee@         dej"        dej"        dej"        dee"ee"         ee"         f         fd            Z e$e=j                   e0dddʦ          d>e"d$e"dBe"dee"         dee"         dee"         dee"         deLfd                        Z e$e=j                   e0dddʦ          d>e"d$e"dBe"dee"         dee"         dee"         dee"         deLde"fd                        Z e$e=j                   e0            eDd>e"deeGeGf         fd                                    Z e$e=j                  dwdde*deGd|e*de*d%e(f
d            Z e$e=j                   e0            dwdde*deGd|e*de*d%e(f
d                        Zdwdde*deGd|e*de*de@d%e(fdZ e$e=j        j`                  e=j        j`        a                    e7j                  dd                        Z e$e=j                  de*deGd|e*de*fd            Z e$e=j                   e0            de*deGd|e*de*fd                        Zde*deGd|e*de*de@f
dZ e$e=j                   e0ddO          eDd.e"dee"e"f         fd                                    Z e$e=j                   e0            	 	 	 dde"dee@eGeLf         dee@eGeLf         dKeejc                 fd                        Z e$e=j                  dd	            Zؐd
 Zِd Z e$e=j        j                   e$e=j        j                   e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d>e"deeeG                  deeeL                  de"fd                                                                                                            Z e$e=j        j                   e$e=j        j                   e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d>e"deeeG                  deeeL                  de"fd                                                                                                            ZddZ e$e=j        j`        e=j        ju        g          e=j        j`        a                    e7j                  e=j        j`        a                    e7j                   e0dd          	 dd>e"deeG         deeL         de"fd                                                Z e$e=j        j`        e=j        ju        g          e=j        j`        a                    e7j                  e=j        j`        a                    e7j                   e0dd          	 dd>e"deeG         deeL         de"fd                                                Z e$e=j        j`        e=j        ju        g          e=j        j`        a                    e7j                  e=j        j`        a                    e7j                   e0dd          	 	 dd>e"deeG         deeL         deeL         de"f
d                                                Z e$e=j        j`        e=j        ju        g          e=j        j`        a                    e7j                  e=j        j`        a                    e7j                   e0dd          	 	 dd>e"deeG         deeL         deeL         de"f
d                                                Z e$e=j        j`        e=j        ju        g          e=j        j`        a                    e7j                  e=j        j`        a                    e7j                   e0dd          	 	 	 dd>e"deeG         deeL         deeL         deeL         de"fd                                                Z e$e=j        j`        e=j        ju        g          e=j        j`        a                    e7j                  e=j        j`        a                    e7j                   e0dd          	 	 	 dd>e"deeG         deeL         deeL         deeL         de"fd                                                ZeD	 dd>e"deeG         deeeL                  de@de"f
d            Zd Zd Zd Zd  Z	 dd!Zd" Zd# Zdd$Zdd%Zd& Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d'                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d(                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d)                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d*                                     Zd+ Zdd,Zdd-Zd. Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d/                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d0                                     Zd1 Z d2 Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d3                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d4                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d5                                     Z e$e=j        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  d6                                     Z e$e=j	        j                   e$e=j
        j                  e=j        j        a                    e7j                  e=j        j        a                    e7j                  e=j	        j        a                    e7j                  e=j	        j        a                    e7j                  e=j
        j        a                    e7j                  e=j
        j        a                    e7j                  d7                                                                                                 Z e$e=j        j`        e=j        ju        g           e0            	 dd>e"deeG         d8e@deeL         de"f
d9                        Z e$e=j	        j`        e=j	        ju        g          e=j	        j`        a                    e7j                   e0            	 	 dd>e"deeG         d8e@deeL         deeL         de"fd:                                    Z	 e$e=j
        j`        e=j
        ju        g           e0            	 	 	 dd>e"deeG         d8e@deeL         deeL         deeL         de"fd;                        Z
dd<Zd= Zdzee"         d>ee"         d?e"de"fd@Zd>e+de"fdAZeDd>e"deeG         d8e@deeeL                  de"f
dB            Z e$e=j        j`                  dCe"dDe"de@fdE            Z e$e=j        e=j        g           e0            dF                         Z e$e=j        g          dG             Z e$e=j        g          ddH            Z e$e=j        g          dI             Z e$e=j        g          dJ             Zd.e"dVe"dBee"         dReGdaeGdee"e"f         fdKZ e$e=j                   e0ddb          d.e"dVe"dBee"         dReGdaeGdee"e"f         fdL                        Z e$e=j                   e0ddb          d.e"dVe"dBee"         dReGdaeGdee"e"f         fdM                        Zde"dNeLde"fdOZde"dNeLde"fdPZdQe"de+fdRZdSe+dTe"de"fdUZdTee"         de"fdVZ dWeGd8e@dTejh        dej        fdXZ!dYe"dZeGd[eGd8e@fd\Z"dYe"d]eGdZeGd[eGd8e@f
d^Z#dYe"deeG         d8e@fd_Z$dYe"deeG         d8e@fd`Z% e$e=j&                   e0            eDdYe"deeG         d8e@fda                                    Z&	 	 	 	 ddCe"dbe"dceGddeGd8e@dee@de"fdfZ' e$e=j(                   e0            eD	 	 	 ddCe"dbe"dceGddeGd8e@de"fdg                                    Z( e$e=j)                   e0            eDdh                                     Z) e$e=j*                   e0            dde>jk        jl        fdi                        Z*djej"        dkej"        dle@de@fdmZ+e=j,        j`        a                    e7j                  e=j,        ju        a                    e7j                   e0dn          ddodp                                    Z, e$e=j-        j`        e=j-        ju        g          e=j-        j`        a                    e7j                   e0            eD	 	 dd>e"deeGeGf         d8e@dqeeL         dreeL         de"fds                                                Z. e$e=j-        j                  e=j-        j        a                    e7j                  e=j-        j        a                    e7j                   e0            eD	 ddCe"deeeGeGf                  d8e@deeeLeLf                  de"f
dt                                                            Z/ e$e=j0                   e$e=j1                   e$e=j2                  eD e0            dCe"deeGdf         de"fdu                                                            Z3 e$e=j4                   e$e=j5                   e$e=j6                  eD e0            dCe"deeGdf         de"fdv                                                            Z7dCe"deeGdf         dweeGeGeGge"f         de"fdxZ8 e$e=j9                   e$e=j:                   e$e=j;                   e0d          dy                                                 Z< e$e=j=                   e0dzd{          ddd|d}                        Z= e$e=j>                   e0            ddd~d                        Z> e$e=j?        j`        e=j?        ju        g           e0            dej@        ddddte(dTeejh                 dejA        deej                 de@f
d                        ZB e$e=j?        jC        g          dej@        ddddse(dte(dTeejh                 dejA        deej                 de@fd            ZD e$e&          d             ZE e$e=jF                  e=jF        j`        a                    e7j                   e0            dwdwde>jk        jl        fd>e"dVe"dme(de(dBee"         dReGde"fd                                    ZF e$e=jG                  e=jG        j`        a                    e7j                   e0dd          d>e"dVe"dReGdee"e"f         fd                                    ZG e$e=jH        j`                  	 	 ddddde"de"d,e"deLde@dee"         d&eeL         dee"e"f         fd            ZId ZJ e$e=jK        g           e0            eDdd                                    ZK e$e=jL                   e0            d                         ZL e$e=jM                  d             ZM e$e=jN        j`        e=jN        ju        g          dddd.e"dTeejh                 dee"         de"fd            ZO e$e=jP        j`        e=jP        jQ        g          dd.e"deeG         fd            ZR e$ej<        j=        jS                  dd            ZS e$e=jT                   e0            dddd                        ZTdddZUddddZV e$e=jW                   e0            d                         ZW e$e=jX                  dd            ZX eJe=jY        e=jZ                    eJe=j[        e=j                    eJe=j\        e=j                    eJe=j]        e=jK                    eJe=j^        e=jN                    eJe=j_        e=j`                    eJe=ja        e=jU                    eJe=jb        e=jc                    eJe=jd        e=jR                    eJe=je        e=jf                    eJe=jg        e=jh                    eJe=ji        e=jj                    eJe=jk        e=jl                    eJe=jm        e=jn                    eJe=jo        e=jp                    eJe=jq        e=jr                    eJe=js        e=jt                    eJe=ju        e=jv                    eJe=jw        e=jx                    eJe=jy        e=jz                    eJe=j{        e=j|                    eJe=j}        e=j~                    eJe=j        e=j                    eJe=j        e=j                    eJe=j        e=j[                   dS (      N)Enum)partialreduce)chainproduct)AnyCallablecastIterableListOptionalTupleUnion)	sym_floatsym_intTensorregister_decomposition)	out_dtype)IntLike
NumberTypesuggest_memory_format
TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_outout_wrapper)_pytree)tree_map__all__c                       e Zd ZdZdZdZdS )	Reductionr         N)__name__
__module____qualname__NONEMEANSUM     X/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/_decomp/decompositions.pyr#   r#   /   s        DD
CCCr-   r#   Fftype_promotioncompute_dtype_onlyc                 N     t          j                    fd            }|S )Nc                      d t          j        | i |D             }t          j        |d
i\  fd}fd} 	t	          ||           i t	          ||          }r|S t	          ||          S )Nc                 <    g | ]}t          |t                    |S r,   )
isinstancer   .0xs     r.   
<listcomp>z-type_casts.<locals>.inner.<locals>.<listcomp>?   s8     
 
 
*QPVBWBW

 
 
r-   type_promotion_kindc                 \    t          | t                    r|                               S | S Nr5   r   to)r8   computation_dtypes    r.   increase_precz0type_casts.<locals>.inner.<locals>.increase_precG   s-    !V$$ tt-...r-   c                 \    t          | t                    r|                               S | S r<   r=   )r8   result_dtypes    r.   decrease_precz0type_casts.<locals>.inner.<locals>.decrease_precM   s,    !V$$ ttL)))r-   )pytreearg_tree_leavesutilselementwise_dtypesr    )argskwargs	flat_argsr@   rC   rr?   rB   r1   r/   r0   s         @@r.   innerztype_casts.<locals>.inner=   s    
 
-t>v>>
 
 
	 +0*B+
,:+
 +
'<
	 	 	 	 		 	 	 	 	 Axt,,P0O0OPP 	.HM1---r-   )	functoolswraps)r/   r0   r1   rL   s   ``` r.   
type_castsrO   8   sF    
 _Q. . . . . . .6 Lr-   T)r0   r1   )r0   r8   dimreturnc                 ~    t          ||                                 z
            D ]}|                     d          } | S N)rangerP   	unsqueeze)r8   rP   _s      r.   _unsqueeze_to_dimrX   j   s;    3=!!  KKOOHr-   
grad_inputout_gradyc                 <    | d||z  z
                                   z  S Nr$   conj_physicalrZ   r[   s     r.   tanh_backwardra   p   s#     q1q5y//1111r-   c                 <    | |d|z
  z                                   z  S r]   r^   r`   s     r.   sigmoid_backwardrc   w   s#     qAE{113333r-   beta	thresholdc                 |    ||z                                   }t          j        ||z  |k    | | |z  |dz   z            S N      ?)exptorchwhere)rZ   r8   rd   re   zs        r.   softplus_backwardrm   ~   sA     
TA;DI-xAS9QRRRr-   grad_outputalphascaleinput_scale	is_resultself_or_resultc                     ||z  }|}|}|r&t          j        |dk    | |z  ||z   z  | |z            S t          j        |dk    | |z  |z  t          j        ||z            z  | |z            S Nr   )rj   rk   ri   )	rn   ro   rp   rq   rr   rs   negcoefposcoef
negiptcoefs	            r.   elu_backwardry      s     emGGJ 
{a*$(@A'!
 
 	
 {a*$w.>J;V1W1WW'!
 
 	
r-   c                 ,    t          j        | |          S r<   )rj   	full_likeselfvalues     r.   fill_scalarr      s    ?4'''r-   r~   c                     t          j                                        dk    fd           t                              |           S )Nr   c                  4    d                                   dS )Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrP   )r~   s   r.   <lambda>zfill_tensor.<locals>.<lambda>   s     kSXS\S\S^S^kkk r-   )rj   _checkrP   atencopyr|   s    `r.   fill_tensorr      sI    	L		qkkkk   99T5!!!r-   r}   c                 b    t          j        t          j        | dz   d          d          dz  S N   r   min   maxrj   clampr}   s    r.   hardsigmoidr      s1     ;u{4!8333;;;a??r-   c                 J    t          j        |dk    |dk     z  | dz  d          S )Ng      g      @gUUUUUU?        rj   rk   rn   r}   s     r.   hardsigmoid_backwardr      s3     ;	$y!  r-   min_valmax_valc                 D    t          j        ||k    ||k    z  d|           S )Nr   r   )rn   r}   r   r   s       r.   hardtanh_backwardr      s%    
 ;DGO<c;OOOr-   c                 h    | t          j        t          j        | dz   d          d          z  dz  S r   r   r   s    r.   	hardswishr      s6     %+ek$(:::BBBBQFFr-   c           
      x    t          j        |dk     dt          j        |dk    | |dz  dz   z  |                     S )Nr   r         ?r   r   s     r.   hardswish_backwardr      sE     ;r	DAI{tax3.>?MM  r-   c                 6    t          j        ||k    d|           S ru   r   )rn   r}   re   s      r.   threshold_backwardr      s     ;ty(![999r-   negative_slopeself_is_resultc                 <    t          j        |dk    | | |z            S ru   r   )rn   r}   r   r   s       r.   leaky_relu_backwardr      s!     ;taxkN.JKKKr-   nonegradapproximatec                    d}d}d}|dk    rh||z  dz  }d}||z  }||z  }	||||	z  z   z  }
t          j        |
          }d|z  }d|z   }d|z  }d||z  z
  }|dd|z  |z  z   z  }||z  |z  }| ||z   z  S |}||z  dz  }ddt          j        ||z            z   z  }|t          j        ||z  d	z            z  }| |||z  z   z  S )
Ng;f?g;f?gmBP?tanhr   gHm?r$   r   g      )rj   r   erfri   )r   r}   r   M_SQRT2	M_SQRT1_2
M_2_SQRTPIkBetakKappax_sqx_cuberL   
tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs                        r.   gelu_backwardr      s-    %G&I'Jf*$s*d{/0Z&&
TzJ+j:55 AF
T(9$9:/14DD)99::Y&,Q4&=1112eitd 2333sTCZ'((r-   inputc                     t          j        t          j        |                    }t          j        |          }||z  d||z  z
  z  }| ||z   z  S r]   )rj   r   Fsoftplussigmoid)rn   r   input_tanh_softplusinput_sigmoidouts        r.   mish_backwardr     sY      *QZ%6%677M%((M
-
1':=P'P#P
QC-344r-   c                 0    | t          j        |           z  S r<   )rj   r   r   s    r.   silur     s     %-%%%%r-   c                 Z    ddt          j        |           z   z  }| |z  d|d|z
  z  z   z  S r]   )rj   ri   )rn   r}   r   s      r.   silu_backwardr     s=     1uy$'''(G AG(<$<==r-   weightc                 <    t          j        | dk    | || z            S ru   r   )r}   r   s     r.   _prelu_kernelr   $  s    ;taxv}555r-   c                 ~    t          j        |dk    | || z            }t          j        |dk    d|| z            }||fS )Nr   r   r   )rn   r}   r   
input_gradweight_grads        r.   _prelu_kernel_backwardr   )  sG     TAX{F[4HIIJ+dQhTK-?@@K$$r-         ?UUUUUU?noiseloweruppertraining	generatorc                     |J |rf| dk    }t                               | ||          }t          j        || |z  |           }|                    t          j        ||d                     |S ||z   dz  }	t                               | |	          S )Nr   r$   r%   )r   uniformrj   rk   copy_
leaky_relu)
r}   r   r   r   r   r   not_positiverK   outputr   s
             r.   rrelu_with_noiser   4  s      5qyLLue,,\4!8T::EKa33444%-1,t^444r-   c           
      P    |                      t          | |||||                    S r<   )r   r   )r}   r   r   r   r   r   s         r.   rrelu_with_noise_r   L  s)     ::&tUE5(IVVWWWr-   c                     |r||z
  dk    r|                      |          S ||z   dz  }t                              | |||          S )Ngư>r%   )mulr   r   )rn   r}   r   r   r   r   r   r   s           r.   rrelu_with_noise_backwardr   Z  s[      
EEMD((u%%%%-1,''~~
 
 	
r-   bufferc                     |dk     }t          j        |dd          }t          j        |dd          }t          j        t          j        |                     }| |||d|z   z  z  z
  z  S )Nr   r$   rT   )rj   rk   ri   abs)rn   r}   r   in_negative	max_derivsignrl   s          r.   log_sigmoid_backwardr   o  sj     (KKA..I;{Ar**D	59T??"##A)da1q5k&::;;r-   loss	reductionc                     |t           j        j        k    rt          j        |           S |t           j        j        k    rt          j        |           S | S r<   )r#   r*   r~   rj   meanr+   sum)r   r   s     r.   apply_loss_reductionr   |  sE    IN(((z$	im)	)	)yr-   dtypec                     | t           j        k    rt           j        S | t           j        k    rt           j        S | t           j        k    rt           j        S d S r<   )rj   	complex32float16	complex64float32
complex128float64r   s    r.   to_real_dtyper     sJ    }	%/	!	!}	%"	"	"} 
#	"r-   targetc                 2    | |z
  dz  }t          ||          S )Nr%   )r   )r}   r   r   r   s       r.   mse_lossr    s"     6MaDi000r-   c                 t    |t           j        j        k    rd|                                z  nd}|||z
  z  | z  S )N       @)r#   r*   r~   numel)rn   r   r   r   norms        r.   mse_loss_backwardr    s>     #,y~/C"C"C3D56>"[00r-   c                     t          j        | ||          }|                     t          d                    }t          j        ||d          }t          j        |          }t          j        |||          S )N)rP   r   z-infTrP   keepdim)rj   softmaxeqfloatall
zeros_likerk   )r}   rP   r   r   maskedmasked_rowszeross          r.   safe_softmaxr    si    
-#U
3
3
3CWWU6]]##F)FT:::KS!!E;{E3///r-   rh   c                     | |z
                                   }t          j        ||k     d|dz  z  |z  |d|z  z
            }t          ||          S )Nr   r%   )r   rj   rk   r   )r}   r   r   rd   r   s        r.   smooth_l1_lossr    sV     6M  D;td{C$'MD$8$t:KLLDi000r-   c                    |t           j        j        k    rd|                                z  nd}||z
  }t	          j        |          }|| z  }t	          j        ||k     ||z  |z  |t	          j        |          z            S rg   )r#   r*   r~   r  rj   r   rk   r   )	rn   r}   r   r   rd   r  r8   abs_x	norm_grads	            r.   smooth_l1_loss_backwardr    s    
 "+in.B!B!B3DvAIaLLE{"I;AEJqMM!  r-   c                 v    t          | ||||          }t          ||j                   t          ||d          S NT	copy_fromcopy_toexact_dtype)r  r   shaper   )rn   r}   r   r   rd   rY   results          r.   smooth_l1_loss_backward_outr!    s@     %[$	4PPFj&,///FJDQQQQr-   deltac           
          |t           j        j        k    rd|                                z  nd}||z
  }t	          j        || k     | | z  |z  t	          j        ||k    || z  |z  ||z  | z                      S rg   )r#   r*   r~   r  rj   rk   )rn   r}   r   r   r"  r  r8   s          r.   huber_loss_backwardr$    s    
 "+in.B!B!B3DvA;	UF
	e#AItk1E94!8k;QRR  r-   c                 v    t          | ||||          }t          ||j                   t          ||d          S r  )r$  r   r  r   )rn   r}   r   r   r"  rY   r   s          r.   huber_loss_backward_outr&    s@     !dFIuMMFj&,///FJDQQQQr-   ignore_indextotal_weightc                    |                                 dk     rdnd}|t          j        j        k    r| |z  } |                    |          }t          j        ||k    |d          }t          j        |          }	t          j        |	||d          }	|	                                 |                                  cxk    rdk    rn n|                     |          } |Ud t          |                                           D             }
|j
        d         |
|<   |                    |
          }| |z  } t          j        ||k    | d          } |	| z  S )Nr%   r   r$   g      c                     g | ]}d S r$   r,   r7   rW   s     r.   r9   z&_nll_loss_backward.<locals>.<listcomp>  s    2221Q222r-   )rP   r#   r*   r~   rV   rj   rk   r  scatterrU   r  reshape)rn   r}   r   r   r   r'  r(  channel_dimsafe_targetrY   	new_shapes              r.   _nll_loss_backwardr2    sG    xxzzA~~!!1KIN(((!L0k**F+f4fa@@K!$''Jz;TJJJ~~+//++////a/////!++K8822dhhjj 1 1222	!'a	+	**!F*+f4k1EEK##r-   c                    |                                 dk    s
J d            t          j        |                                 |          }|                    |          }|dz  dk    sJ d| d|             |dz  }|                    |d|          }|                    |||          }t          j        |          }d|z
  |z  |z  | z  }	|| z  }t          j        ||	g|          S )Nr   z*glu does not support 0-dimensional tensorsr%   z.Halving dimension must be even, but dimension z	 is size rh   r   )rP   rF   canonicalize_dimsizenarrowrj   r   cat)
rn   r}   rP   wrap_dimnIn	inputSize	firstHalf
secondHalfgradInputFirstHalfgradInputSecondHalfs
             r.   glu_backwardr?    s     88::>>>G>>>%dhhjj#66H
))H

Ca1PPP3PP 	qIHa33IXy)<<Jz22	!	!%77)CkQ  ,k99(*=>HMMMMr-   c           	      6   d|                                 cxk    rdk    sn J d            |                                 dk    s
J d            |                                 dk    o|                                 dk    }|s7|j        d         |j        d         k    sJ d|j         d|j         d            |                                dk    s)J d	|j         d
|                                 df            |-|                                |j        d         k    s
J d            |t          j        j        k    r|                                 dk    rp|                                  dk    r| j        d         |j        d         k    s;J d|j        d          d|                                   d| j        d                      nB|                                  dk    r|                                 dk    sJ d| j                     t          | ||||||          S )Nr   r%   input tensor should be 1D or 2Dr$   ;0D or 1D target tensor expected, multi-target not supportedsize mismatch (got input: 
, target: ):expected total_weight to be a single element tensor, got: z (z
 elements)rT   z<weight tensor should be defined either for all or no classesz7Expected a tensor of dimension 1 and tensor.size[0] == z but got: dimension z and tensor.size[0] == z7Expected a single element grad_output tensor, but got: )rP   r  r  r#   r)   r~   r2  )rn   r}   r   r   r   r'  r(  no_batch_dims           r.   nll_loss_backwardrH  1  sc    

a!B

D 	 88::?8vzz||q'8L J
1a(((IDJII&,III 	)((1$$$DAA!3!3!5!5AAA'$$$ 	&,,..DJrN:::E ;:: IN(((TXXZZ1__  A%%+*;A*>$*Q-*O*O*O_djQRm _ _)oo//_ _HSHYZ[H\_ _ +P*O*O*O OO""{'8'8':':a'?'?'?X[EVXX (@'?'? T669lL  r-   c           	      d   |                                 dk    sJ d|                                              |                                 dk    sJ d|                                              |j        d         |j        d         k    r8|j        d         |j        d         k    r|j        d         |j        d         k    sJ d|j         d	|j                     |                                dk    s(J d
|j         d|                                 d            t          | ||||||          S )N   zSonly batches of spatial inputs supported (4D tensors), but got input of dimension: r   zUonly batches of spatial targets supported (3D tensors) but got targets of dimension: r   r%   r$   rC  rD  rF  z ( z, elements))rP   r  r  r2  )rn   r}   r   r   r   r'  r(  s          r.   nll_loss2d_backwardrK  ]  sb    	

ai]a]e]e]g]gii 	 	

m_e_i_i_k_kmm 	 	
1a((JqMV\!_,,JqMV\!_,,,HDJHH&,HH -,, 1$$$	I"	I 	I'3'9'9';';	I 	I 	I %$$
 T669lL  r-   c           	      2   |dz
  t          j        t          j        |            |                     dd                    z  |t          j        t          j        |           |                     dd                    z  z
  }|||z  }t          ||          S )Nr$   r,   i)rj   maximumlog1pnew_fulllogr   )r}   r   r   r   r   s        r.   binary_cross_entropyrQ    s     QJ%-TEDMM"d33  uyb$0G0GHHHID f}i000r-   c                     d}| ||z
  z  t          j        |d|z
  z  |          z  }|||z  }|t          j        j        k    r||                                z  }|S )Ng-q=r$   r   )rj   r   r#   r*   r~   r  )rn   r}   r   r   r   EPSILONr   s          r.   binary_cross_entropy_backwardrT    sk     GD6M*U[T9JPW-X-X-XXF&IN((($**,,&Mr-   c                 v    t          j        t          j        |  |z                      }t          ||          S r<   )rj   rN  ri   r   )r   r   r   r   s       r.   soft_margin_lossrV    s3     ;uy%&1122Di000r-   c                     || z  t          j        ||z            dz
  z  }|t          j        j        k    r||                                z  }|S r]   )rj   r   r#   r*   r~   r  )rn   r}   r   r   rY   s        r.   soft_margin_loss_backwardrX    sN     +%v})E)E)IJJIN((($**,,.
r-   r%   otherpc                 @    t                               | |z
  |          S )N)rZ  )r   r  )r   rY  rZ  s      r.   distr\    s     99UU]a9(((r-   x1x2c                 .   |                      d                              dd          }t          j        |t          j                  }|                     d                              dd          }t          j        |t          j                  }t          j        |                     d          ||gd          }t          j        |||gd          }|                    |j                  }|	                    d          
                                S )Nr%   rT   Tmemory_formatr   )powr   rj   	ones_likecontiguous_formatr7  r   matmulmT	clamp_minsqrt)	r]  r^  x1_normx1_padx2_normx2_padx1_x2_r   s	            r.   _euclidean_distrp    s     ffQiimmB%%G_WE4KLLLFffQiimmB%%G_WE4KLLLF
)RVVBZZ&12
6
6C
)R)2
.
.CZZFA##%%%r-   input_sizesstartendstepc                 ^    |                      |          }t          j        || ||||          S r<   )	new_zerosrj   slice_scatter)rn   rq  rP   rr  rs  rt  rY   s          r.   slice_backwardrx    s2     &&{33Jz;UCNNNr-   r$   c                    ddl m}m} |                                 }|dk    rt	          d          t          j        |                                 |          }t          |                                           }t          | 	                                          }	|dk    rt	          d          ||nd}
||nt          j        } ||
dk               r|
||         z  }
 ||dk               r|||         z  } ||
dk               rd}
n ||
||         k              r||         }
 |||
k               r|
}n6 ||t          j        k              s ||||         k              r||         }|                                 |
|	|         z  z   }||
z
  }||z   dz
  |z  ||<   |	|xx         |z  cc<   | j        rt          d          |                     ||	|          S )Nr   )guard_size_obliviousstatically_known_truez,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver$   z<Slice decomposition for quantized tensors aren't implemented)%torch.fx.experimental.symbolic_shapesrz  r{  rP   RuntimeErrorrF   r4  listr5  stridesysmaxsizestorage_offsetis_quantizedNotImplementedError
as_strided)r}   rP   rr  rs  rt  rz  r{  ndimsizesstrides	start_valend_valr  lens                 r.   slice_forwardr    sI          
 88::DqyyIJJJ

 S
1
1CE4;;==!!Gqyy8999*I_cc#+GIM**  U3Z	GaK(( 5:IM** 				i%*4	5	5 #J	Gi/00 		w#+5	6	6 :N:N%*; ;  *((**Y-EEN
I
C*q.T)E#JCLLLDLLL ?!J
 
 	
 ug~>>>r-   c                 x    | j         |         dt          ffd} ||dd          } |||          }||fS )zn
    Normalize start and end such that both are in the range
    [0, x.get_size()[dim]] and start <= end.
    rQ   c                 ^    | |S | dk     r| z   } t          t          | |          |          S ru   r   r   )valr   r   defaultdim_sizes       r.   
clamp_wrapz(_normalize_start_end.<locals>.clamp_wrap(  s7    ;N77.C3sE??E***r-   r   )r  int)r8   rP   rr  rs  r  r  s        @r.   _normalize_start_endr    sk     ws|H+# + + + + + + Jua1--E
*S%8
4
4C#:r-   srcc           	         t          j        | j        |          }| j        |         }t	          | |||          \  }}t          | j                  }||z
  |dz
  z   |z  ||<   |                    |          }|dk    r ||k    r|dk    r|                                S d g|                                 z  }t          j
        || j                  }	|	|z
  |z  ||<   t          j        || j        t          j                  }
|dk    rt          j        |
|	|k              }
||k    rt          j        |
|	|k               }
|dk    rt          j        |
|	|z
  |z  dk              }
dg|                                 z  }d||<   |
                    |          }
t                               |
t                               ||
|d          |           S )Nr$   r   devicer  r   rT   )rF   r4  r  r  r  r~  expandclonerP   rj   aranger  onesboollogical_andviewr   rk   _unsafe_masked_index)r   r  rP   rr  rs  rt  r  src_sizeindicesidxmask
mask_shapes               r.   rw  rw  6  s    
 S
1
1C{3H%eS%==JE3EK  H5[D1H-$6HSM
**X

CzzcXoo$!))yy{{fuyy{{"G
,x
5
5
5C%KD(GCL:hu|5:FFFDzz se|44
h sSy11qyy et';q'@AAuyy{{"JJsO99Z  D::dD55c4!LLeTTTr-   indexc                 Z    |                      |          }t          j        || ||          S r<   )rv  rj   select_scatter)rn   rq  rP   r  rY   s        r.   select_backwardr  _  s.     &&{33J
KeDDDr-   offsetdim1dim2c                 \    |                      |          }t          j        || |||          S r<   )rv  rj   diagonal_scatter)rn   rq  r  r  r  rY   s         r.   diagonal_backwardr  f  s0    
 &&{33J!*k64NNNr-   input_dtypec                 F    | j         |k    r|                    |          }|S r<   )r   r>   )rn   rY   r  s      r.   _cast_grad_to_input_dtyper  o  s)     K'']];//
r-   r   c                     | |z  }||t          j        ||d          z  z
  }t          | ||                                          S NTr  )rj   r   r  
contiguous)rn   r   rP   r  new_grad_outputrY   s         r.   _softmax_backward_datar  w  sY     "F*O 6EIS$- - - $ J %[*kJJUUWWWr-   c                     | t          j        |          t          j        | |d          z  z
  }t          | ||          S r  )rj   ri   r   r  )rn   r   rP   r  rY   s        r.   _log_softmax_backward_datar    sM     uy0059d4 4 4   J %[*kJJJr-   c                     | |dz  z   ||dz
  z  z
  }t          t          j        t          j        |          } |d||                              d          } |d||z  |                              d          }	||	z   S )z/Utility function to implement im2col and col2imr%   r$   r   r  r   rT   )r   rj   r  int64rV   )
input_dkernel_d
dilation_d	padding_dstride_dr  blocks_d	arange_kwblocks_d_indiceskernel_grids
             r.    _im2col_col2im_indices_along_dimr    s     Q&x!|)DDHEKGGGI !yHh77AA!DD )Ax*4jAAKKBOOK k))r-   kernel_sizedilationpaddingr  c           
         t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            dd} |d            |d	            |d
d            |d           | j        t                    }t          j        |dv o t	          d dd          D                       fd           t          d t          dd                    D                       t          j        t	          d D                       fd           |dk    }|s|                     d          } | j        \  }}	}
}\  }}\  }}\  }}\  }}t          |
||||| j	                  }t          |||||| j	                  }t          j        | ||||f          }|                    d                              d          }|d d d d ||f         }|                    dddddd          }|                    d          }|                    d          }|                    ||	|z  |z  ||z            }|s|                    d          }|S )Nr%   c                      dS )Nz"im2col(): only 2D kernel supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  s    0T r-   c                      dS )Nz$im2col(): only 2D dilation supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  s    -S r-   c                      dS )Nz#im2col(): only 2D padding supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  s    ,Q r-   c                      dS )Nz"im2col(): only 2D stride supportedr,   r,   r-   r.   r   zim2col.<locals>.<lambda>  s    +O r-   Tc                     |rt          d | D                       nt          d | D                       }t          j        |d            d S )Nc              3   "   K   | ]
}|d k    V  dS r   Nr,   r7   rZ  s     r.   	<genexpr>z1im2col.<locals>.check_positive.<locals>.<genexpr>  &      ((Q1q5((((((r-   c              3   "   K   | ]
}|d k    V  dS r  r,   r  s     r.   r  z1im2col.<locals>.check_positive.<locals>.<genexpr>  &      ;R;RqAF;R;R;R;R;R;Rr-   c                      dS )Nz<{param_name} should be greater {'than' zero, but got {param}r,   r,   r-   r.   r   z0im2col.<locals>.check_positive.<locals>.<lambda>  s    X r-   r  rj   r   param
param_namestrictconds       r.   check_positivezim2col.<locals>.check_positive  sf    ,2Rs((%((((((;R;RE;R;R;R8R8RXX	
 	
 	
 	
 	
r-   r  r  r  Fr  r  r   rJ  c              3   "   K   | ]
}|d k    V  dS r  r,   r7   ds     r.   r  zim2col.<locals>.<genexpr>  &      ::!qAv::::::r-   r   c                  (    dt                      S )NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tupler  s   r.   r   zim2col.<locals>.<lambda>       <-25\\< < r-   c              3   V   K   | ]$\  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z   V  %dS )r$   r%   Nr,   r7   r   paddilkersts         r.   r  zim2col.<locals>.<genexpr>  sd        "Cc3 	
S1s7]SC!G_,q0R77     r-   rb  c              3   "   K   | ]
}|d k    V  dS r  r,   )r7   cs     r.   r  zim2col.<locals>.<genexpr>  s&      ''aAE''''''r-   c                  X    dt          dd                     d d  d d d dS )	Nz!Given an input with spacial size rb  , kernel_size=, dilation=
, padding=	, stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r  )r  r  output_sizer  r  r  s   r.   r   zim2col.<locals>.<lambda>  s|     FE%*4E4E F F"F F/7F FF F%+F F 	F F F r-   rJ  r   rT   r$   r      T)rj   r   r  r  r  r  ziprV   r  r  r   r  permuter5  r.  squeeze)r   r  r  r  r  r  r  batched_input	batch_dimr/  input_hinput_wstride_hstride_w	padding_h	padding_w
dilation_h
dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indicespadded_inputr   num_blocks_rownum_blocks_colr  r  s    ````                     @@r.   im2colr    s    
L[!!Q&(T(TUUU	LX!#%S%STTT	LW"$Q$QRRR	LV!#O#OPPP
 
 
 
 N;...N8Z(((N8Yu5555N68$$$KEu::D	L:3::uRSSz:::::	< 	< 	< 	<  
   &)"##J;'
 '
    K 
L'';'''''	F 	F 	F 	F 	F 	F 	F 	F 	F   AIM #""/4{,I{GWHh"Iy%J
$Hh9:y(EL  ::y(EL  5Iy) LMML+55b99CCBGG!!!QQQ 24FFGF^^Aq!Q1--F',,Q//N',,Q//N^^;)H4n~6U F  #""Mr-   r  c           
         t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            d d} |d	            |d
            |dd            |d            |d           | j        t                    }t          j        |dv o t	          d dd          D                       fd           d         d         z  }t          j        d         |z  dk    fd           d t                    D             }	|	d         |	d         z  t          j        d         k    fd           t          j        dk    fd           |dk    }
|
s|                     d          } | j        \  }}\  }}\  }}\  }}\  }}|                     d         d         |z  gz   |	z             } |                     dddddd          } t          |||||| j
                  }t          |d          }t          |||||| j
                  }d t                    D             }|                     d         d         t                    z  g|z             }d d ||f}t                              ||| d          }t!          j        || | | | f          }|
s|                    d          }|S )!Nr%   c                      dS )Nzonly 2D output_size supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  s    0O r-   c                      dS )Nzonly 2D kernel supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  s    0J r-   c                      dS )Nzonly 2D dilation supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  s    -I r-   c                      dS )Nzonly 2D padding supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>  s    ,G r-   c                      dS )Nzonly 2D stride supportedr,   r,   r-   r.   r   zcol2im.<locals>.<lambda>	  s    +E r-   Tc                     |rt          d | D                       nt          d | D                       }t          j        |d            d S )Nc              3   "   K   | ]
}|d k    V  dS r  r,   r  s     r.   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r-   c              3   "   K   | ]
}|d k    V  dS r  r,   r  s     r.   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r-   c                      dS )Nz9{param_name} should be greater than zero, but got {param}r,   r,   r-   r.   r   z0col2im.<locals>.check_positive.<locals>.<lambda>  s    U r-   r  r  s       r.   r  zcol2im.<locals>.check_positive  sf    ,2Rs((%((((((;R;RE;R;R;R8R8RUU	
 	
 	
 	
 	
r-   r  r  r  Fr  r  r  )r%   r   c              3   "   K   | ]
}|d k    V  dS r  r,   r  s     r.   r  zcol2im.<locals>.<genexpr>  r  r-   rb  c                  (    dt                      S )NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: r  r  s   r.   r   zcol2im.<locals>.<lambda>  r  r-   r   r$   c                       dd          d  S )Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rb  z and kernel_size=r,   )r  r  s   r.   r   zcol2im.<locals>.<lambda>!  s(     %=B2Y% %"% % r-   c                 N    g | ]"\  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z   #S r$   r%   r,   r  s         r.   r9   zcol2im.<locals>.<listcomp>%  sV       "Cc3 	
S1s7]SC!G_,q0R77  r-   rT   c                  @    d d d d d d  dd          d	S 
NzGiven output_size=r  r  r  r  z , expected input.size(-1) to be 	 but got rT   .r,   Lr  r  r  r  r  r  s   r.   r   zcol2im.<locals>.<lambda>.  |     B[ B B B BB B(/B B:@B B)*B B5:2YB B B r-   c                  @    d d d d d d  dd          d	S r!  r,   r$  s   r.   r   zcol2im.<locals>.<lambda>4  r&  r-   r   rJ  r  c                 $    g | ]\  }}|d |z  z   S r%   r,   )r7   orZ  s      r.   r9   zcol2im.<locals>.<listcomp>P  s$    JJJ1!a!e)JJJr-   
accumulater  )rj   r   r  r  r  r  rV   r.  r  r  r  rX   rv  prodr   _unsafe_index_putr   r  r  )r   r  r  r  r  r  r  r  prod_kernel_sizecolr  out_hout_wr  r  r  r  r  r  r	  r
  indices_rowindices_coloutput_padded_sizer   r  r%  r  s    `````                    @@r.   col2imr6    s    
L[!!Q&(O(OPPP	L[!!Q&(J(JKKK	LX!#%I%IJJJ	LW"$G$GHHH	LV!#E#EFFF
 
 
 
 N;...N8Z(((N7Ie4444N68$$$N;...KEu::D	L:3::uRSSz:::::	< 	< 	< 	<  
 #1~A6	Lb	$$)	% 	% 	% 	% 	%   &)(K'
 '
  C 	AQA	Lb	Q	B 	B 	B 	B 	B 	B 	B 	B 	B 	B   
L	A	B 	B 	B 	B 	B 	B 	B 	B 	B 	B   AIM #""KELE5Hh"Iy%J
$Hh MM58U1X1A%AB[PSVVWWEMM!Q1a++E2xY%, K $K33K2xY%, K KJK0I0IJJJ__	q58tK00014FF F {K
0C##FC4#HHFU6YJ
YJ
KLLF #""Mr-   r  c                     | |                     |           |z  z                      t          j        |                     }|S Nr`  )type_asr  rF   r   )rn   r  rp   rK   s       r.   native_dropout_backwardr:  ]  sJ     
[11E9	:AA1+>> 	B 	 	A Hr-   
input_size	dimensionr5  c                 <   t          |          dk    rt          j        | d          S t          j        t          |          |          }t          j        ||         | j        t          j                  }|                    d||          	                                }| 
                    d|dz             	                    ||dz             } |                     |          }d|z  |fz   }t                              ||| d                                          S )Nr   r  rT   r$   r<   Tr+  )r  rj   squeeze_copyrF   r4  r  r  int32unfoldflattenmovedimrv  r   r.  r  )	r   r;  r<  r5  rt  rP   r  rY   r  s	            r.   unfold_backwardrC  l  s    
 :!!$***

 Z)
<
<C
,z#t{%+
N
N
NC
**Qd
#
#
+
+
-
-C<<C!G$$,,S#'::D 
++JcMSF"E!!*eTd!KKVVXXXr-   epsc           
      >   |A|}d|z
  }t          j        t          j        ||k    ||k              | |d|z
  z  z  d          S t          j        t          j        |dk    |dk              | |d|z
  z  z  |                    dt	          d                              S )Nrh   r   r,   nan)rj   rk   r  rO  r  )rn   r}   rD  lohis        r.   logit_backwardrI    s    
 2X{dbj$"*5543:./
 
 	
 {dck43;7743:./MM"eEll++
 
 	
r-   trainc                 ~    |r(|dk    r"t                               | ||          d         S |                                 S ru   )r   native_dropoutr  )r   rZ  rJ  s      r.   dropoutrM    s?      a""5!U33A66{{}}r-   out0out1c                 n   |r|dk    r|dk    r4t          j        |           t          j        | t           j                  fS | j        j        st          d          t          j        |           |k    }|| z  t          dd|z
  z            z  }||fS | t          j        | t           j                  fS )Nr   r$   r   z?result type Float can't be cast to the desired output type Longrh   )	rj   r  r  r   is_floating_pointr}  	rand_liker  rd  )r   rZ  rJ  	bool_maskress        r.   rL  rL    s      Aa66$U++U-=e5:-V-V-VWW{, 	Q   OE**Q.	%%sQw"8"88YuuEJ???@@r-   half_to_floatc                    |                                  } |r| j        t          j        k    sJ t	          j        | t          j        j                  \  }}|                     |          } | 	                                dk    rt          j
        |           }n.t          j        | |d          }t          j
        | |z
            }|t          j        ||d          z  }|s|                    |          }|S Nr:   r   T)r	  )r  r   rj   halfrF   rG   ELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTr>   r  ri   amaxr   )r8   rP   rU  r?   rB   unnormalizedx_maxr   s           r.   _softmaxr_    s    
 	
A %w%*$$$$&+&>	uDL' ' '#| 	
AwwyyA~~y||
1c4000yU++EIlCFFFFF )<((Mr-   c                    |                                  } |r| j        t          j        k    sJ t	          j        | t          j        j                  \  }}|                     |          } | 	                                dk    r| }nt          j
        | |d          }| |z
  }t          j        t          j        t          j        |          |d                    }||z
  }|s|                    |          }|S rW  )r  r   rj   rY  rF   rG   rZ  r[  r>   r  r\  rP  r   ri   )	r8   rP   rU  r?   rB   shiftedr^  shifted_logsumexpr   s	            r.   _log_softmaxrc    s    
 	
A %w%*$$$$&+&>	uDL' ' '#| 	
AwwyyA~~
1c4000e)	%)EIg,>,>T"R"R"RSS((F )<((Mr-   rT   r  padding_idxscale_grad_by_freqsparsec                     |                                  dk    s
J d            |j        dk    r8|                     d|          }|j        dk    r|                    d          }|S | |         S )Nr%   z'weight' must be 2-Dr$   r   )rP   r  index_selectr  )r   r  rd  re  rf  r   s         r.   	embeddingri    ss     ::<<14|q!!!W--<1++a..C
gr-   num_weightsc                    t          j        | t           j        j                  \  }}|                     |          } t          |t          j                  }|ri|                    |f          }t          j	        |          }t                              ||g|d          }||         }	| |	                    d          z  } t          ||k    | j                  }
|                     |
d          }|                     |f| j        |j        d          z             }t                              ||g|d                              |          S )NrX  Tr+  rT   r   )rF   rG   rZ  r[  r>   r   rj   longrv  rd  r   r.  rV   rX   r  masked_fillr  )rn   r  rj  rd  re  r?   rB   countsr  grad_weights_scaler  r   grad_weights                r.   embedding_dense_backwardrq    sN    ',&>)N)V' ' '#| ..!233K%guz::G E""K>22w''''	4D'QQ#G_!$6$@$@$D$DDW3[5EFFD""4++D''	*7<>>:: K !!+y$4!PPSS  r-   c                     d}| D ]}||z  }|S r]   r,   )r8   rK   is      r.   r-  r-    s&    	A  	QHr-   tensors
num_chunksc                    g }| D ]}|                                 }||         |z   dz
  |z  |z  }|||         k    r>dgdz  |j        |z
  dz
  z  d|||         z
  gz   }t                              ||d          }|d |         t	          j        |dg          z   }|                    |                    |                     |S )Nr$   r   r%   rT   )r5  r  r   constant_pad_ndrj   Sizeappendr  )	rt  rP   ru  padded_tensorstensortensor_sizepad_along_dimr  	view_sizes	            r.   
_pad_chunkr    s    
 N 6 6kkmm$S)J6:zIJVK,,,#'V[3.23C 007 C ))&#q99F%
J3C(D(DD	fkk)445555r-   c                 F    | d         j         }| D ]}|j         |k    r dS dS )Nr   FTr  )rt  r  r{  s      r.   have_same_ndimsr  .  s:    1:?D  ;$55 4r-   c                     | d                                          d |         }| D ]6}t          j        |                                 d |         |k    d            7d S )Nr   c                      dS )NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr,   r,   r-   r.   r   z+leading_dimension_matches.<locals>.<lambda>;  s    ] r-   )r5  rj   r   )rt  rP   leading_dim_sizesr{  s       r.   leading_dimension_matchesr  6  sq    
))$3$/ 
 
KKMM$3$#44]]	
 	
 	
 	

 
r-   c                    t          j        |dk    d            t          j        t          |           dk    d            | d         j        }| d         j        }| D ]l}t          j        |                                dk    d            t          j        |j        |k    d            t          j        |j        |k    d            mt          |           r.t          j        | d         	                                |          }n>t          j        |dk    d            | D ]!}t          j        ||j
        k     d	            "t          | |           |S )
Nr$   c                      dS )Nz&_chunk_cat expects positive num_chunksr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>D  s    *R r-   r   c                      dS )Nz0_chunk_cat expects a non-empty input tensor listr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>F  s    "T r-   c                      dS )Nz#_chunk_cat expects non-empty tensorr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>K  s    1V r-   c                      dS )Nz8_chunk_cat expects all input tensors with the same dtyper,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>N      N r-   c                      dS )Nz8_chunk_cat expects all inputs tensors on the same devicer,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>R  r  r-   c                      dS )NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>Y  s    a r-   c                      dS )Nz3_chunk_cat expects dim < ndim for all input tensorsr,   r,   r-   r.   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>^      M r-   )rj   r   r  r   r  r  r  rF   r4  rP   r  r  )rt  rP   ru  expected_dtypeexpected_devicer{  s         r.   _preprocess_chunk_cat_inputsr  ?  s   
 
Lq"R"RSSS	LGqTT   QZ%Naj'O 	
 	
V\\^^a')V)VWWWLN*NN	
 	
 	
 	M_,NN	
 	
 	
 	
 w $WQZ^^%5%5s;;1Haa	
 	
 	
  	 	FLfk!MM    gs+++Jr-   r   c                     t          | ||          }t          | ||          }|t          j        ||dz             S t          j        ||dz   |           |S )Nr$   )r   )r  r  rj   r7  )rt  rP   ru  r   rz  s        r.   
_chunk_catr  d  sa     'wZ
@
@Cj99N
{yq111	.#'s3333
r-   split_sizesc                     t          t                              D ]}t          j        |         d            t          j        t
          t                     j                 k     fd           t                    }g }d}t          |          D ]9}|         }|                     	                    ||                     ||z  }:|S )Nc                      dS )NzCsplit_with_sizes expects split_sizes have only non-negative entriesr,   r,   r-   r.   r   z"split_with_sizes.<locals>.<lambda>}  s    Y r-   c                  D    dt                     dj                   S )NzSplit sizes add up to z but got the tensor's size of )r   r  )rP   r}   r  s   r.   r   z"split_with_sizes.<locals>.<lambda>  s*    j[)9)9jjY]YcdgYhjj r-   r   )
rU   r  rj   _check_is_size_check_with
ValueErrorr   r  ry  r6  )r}   r  rP   rs  
num_splitssplits	start_idxlengths   ```     r.   split_with_sizesr  t  s     3{##$$ 
 
NYY	
 	
 	
 	
 
KDJsO+jjjjjj  
 [!!JFI:  Qdkk#y&99:::V		Mr-   c                     t          | ||          }|d |D             S t          ||          D ],\  }}t          ||j                   t	          ||d           -d S )Nr   c                 N    g | ]"}|                     t          j                   #S )r`  )r  rj   re  )r7   ss     r.   r9   z)split_with_sizes_copy.<locals>.<listcomp>  s)    OOO1e&=>>OOOr-   Tr  )r  r  r   r  r   )r}   r  rP   r   r  r   splits          r.   split_with_sizes_copyr    s     dKS999F
{OOOOOO f-- 	N 	NMFEfek222UFMMMMMtr-   
split_size.c                 D    t           j                            | ||          S r<   )r   r  r   )r   r  rP   s      r.   unsafe_splitr    s    :UJ444r-   c                 D    t           j                            | ||          S r<   )r   r  r  )r   r  rP   s      r.   unsafe_split_with_sizesr    s      ((SAAAr-   c                     | j         }||         }dk    r|dk    sJ | fS |z   dz
  z  }ddlm}  ||          }fdt          |          D             }|z  |z
  z
  |d<   t	          j        | ||          S )Nr   r$   )	guard_intc                     g | ]}S r,   r,   )r7   rs  r  s     r.   r9   zsplit.<locals>.<listcomp>  s    555!:555r-   rT   )r  r|  r  rU   rj   r  )r}   r  rP   rq  r  chunksr  r  s    `      r.   r  r    s    *K3HQ1}}}}w#a'J6F @?????YvF5555uV}}555K J$7($BCKO;t[#...r-   tensor_indices_or_sectionsc                    |j         j        dk    sJ |j        t          j        k    sJ |                                t          j        dk    pdk    fd           dk    rA|                                }t          |t                    sJ | 
                    ||          S d |D             }| 
                    ||          S )Ncpur$   r   c                      d  dS )Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr,   )	split_dims   r.   r   zAtensor_split_tensor_indices_or_sections_py_impl.<locals>.<lambda>  s      M<EM M M r-   c                 6    g | ]}|                                 S r,   )item)r7   rs  s     r.   r9   zCtensor_split_tensor_indices_or_sections_py_impl.<locals>.<listcomp>  s     @@@16688@@@r-   )r  typer   rj   r  rP   r   r  r5   r   tensor_split)r}   r  rP   sectionsr  r  s        @r.   /tensor_split_tensor_indices_or_sections_py_implr    s     &,1U::::%+u{::::*..00I	LQ()q.	M 	M 	M 	M  
 A~~-2244(G,,,,,  3///@@%?@@@   #...r-   mat1mat2c                     |                                  s2|                                 st          |          }t          |          }|t          j        ||          z  }|dk    r|S ||| z  z   S ru   )rQ  
is_complexr  rj   mm)r}   r  r  rd   ro   r   s         r.   addmmr    st     !!## DOO,=,= 4yyE


%(4&&
&Cqyy
 r-   use_geluc                     t          | ||||          }|r=| j        rt                              |d          S t                              |          S t                              |          S )Nr   )r   )r  is_cudar   gelurelu)r}   r  r  rd   ro   r  r   s          r.   _addmm_activationr    sa     dD$
.
.C "< 	"99Sf955599S>>!99S>>r-   vecc                     |                                  s2|                                 st          |          }t          |          }|t          j        ||          z  }|dk    r|S ||| z  z   S ru   )rQ  r  r  rj   mv)r}   r  r  rd   ro   r   s         r.   addmvr    sr     !!## DOO,=,= 4yyE


%(4%%
%Cqyy
r-   r   rstdgammaNCHxWgroupoutput_maskc
           	      2	   t          j        | ||d           t          j        || d           t          j        |d           t          j        |                                z  z  k    fd           t          j        j        fk    fd           t          j        d u p                                k    fd           t                    \  }
}t          j        |dk    fd           t          j        | |          	                              
                    dg	          }| 	                              
                    dg	          }d }d }d }|	d         rdd
|
z  z  }t          j        |                    d                                        |
          
                    d          }t          j        |                    d                                        |
          
                    d          }t          j        |                    d                              d|
                    }n|                    |
          
                    d          }|                    |
          
                    d          }t          j        |                    d          t          j        d|
f|j                            }|z  |z
  |z  |z  |z  |z  }| z  ||z  |z  z
  }|                    d          }t          |d          }t          |d          }t          j        |                     |
          |          t          j        |                    |
          |          z   |z   }|                    |j                                      |j                  }|	d         r|	                    |
          |	                    |
                              d          z  z
  |                    d          z  
                    dg	                                        }|	d         r|
                    dg	          }|||fS )NF)allow_cpu_scalar_tensorsc                      d z  z   dS )NzExpect input to have z	 elementsr,   )r  r  r  s   r.   r   z,native_group_norm_backward.<locals>.<lambda>.  s    >A>>> r-   c                  $    d  d dj          S )NzExpect mean to have shape (, z
, but got r  )r  r  r   s   r.   r   z,native_group_norm_backward.<locals>.<lambda>2  s"    PaPP5PPDJPP r-   c                  @    d  d                                 nd S )NzExpect gamma to have z elements but got rT   )r  )r  r  s   r.   r   z,native_group_norm_backward.<locals>.<lambda>6  s*    iiieN_U[[]]]egii r-   r   c                      d  d S )NzExpect number of channels z, to be evenly-divisible by number of groups r,   )r  r  s   r.   r   z,native_group_norm_backward.<locals>.<lambda><  s    cQcc\acc r-   r%   r   rh   rT   r$   r  rJ  )rF   check_same_devicecheck_same_shaperj   r   r  r  divmodr   r  r   rV   r.  r  r  rX   r>   r   )rn   r   r   r  r  r  r  r  r  r  cpg_remdsdbd_inputd_gammad_biasr  ds_valdb_valc1c2c3s     ` `````              r.   native_group_norm_backwardr    s    
UD$    
5+NNNN	4FFFF	LQ$>>>>>>   
L
q%j PPPPPP   
L+!+iiiii  
 q%  IC	L	ccccc   
;	&	&	+	+Aq#	6	6	:	:s	:	C	CB			!Q	$	$	(	(aS	(	1	1B $G $G#F1~ ?39Yr5??1#5#566>>q%MMQQRSTTFYr5??1#5#566>>q%MMQQRSTTFr""a,, BB
 ZZ5#..22155FZZ5#..22155Fr""
Auc?4;??? B tmf$,t3d:Q>S4Z&4-!++\\"r1%%r1%%Ik))!UC==rBBiaS992>>? 	
 //%+..11%+>>1~ 
 E3''"''!UC*@*@4>>RTCUCU*UU..$$% SaSS\\WQZZ 	 1~ !QCWf%%r-   out2c
                    t          | |||||||||	
  
        }|
||f}t          |          D ]:\  }}|3t          ||         |j                   t	          |||         d           ;|S r  )r  	enumerater   r  r   )rn   r   r   r  r  r  r  r  r  r  rN  rO  r  r   rY   rs  rK   s                    r.   native_group_norm_backward_outr  r  s    " (UD$q!S% F d#J&!! Q Q1=jmQW555Q
14PPPPr-   c                 4    | |                      |          S | S r<   r>   )r8   r   s     r.   _maybe_castr    s    }ttE{{Hr-   grad_outnormalized_shapebiasc                   ! |j         }|                                }	t          j        |j                  !!fd| |||fD             \  }
}}}|
J |	t          |          z
  }||d          }|d |         }g }g }t          |	          D ]3}||k    r|                    |           |                    |           4t          |          }t          |          }|dk    s|dk    rl|d         r|	                    |          nd |d         r|	                    ||d                    nd |d         r|	                    ||d                    nd fS t          ||                                          }t          ||                                          }||z
  |z  }||
|z  }n|
}||z  }t          j        ||d          }t          j        ||          }t          j        ||d          }t          j        ||          }||z
  |z
  }d }d }d } |d         r||z  |z  }|d         r4|2t          |          dk    rt          j        |
|z  |d          }n|
|z  }|d         r@|>t          |          dk    rt          j        |
|d          } n|
                                } t          ||j                  t          ||j                  t          | |j                  fS )Nc              3   n   K   | ]/}|'|                                                               n|V  0d S r<   )r>   r  r7   r8   r?   s     r.   r  z-native_layer_norm_backward.<locals>.<genexpr>  sY       9 9 12**,,,19 9 9 9 9 9r-   r   r$   r%   TF)r  rP   rF   get_computation_dtyper   r  rU   ry  r-  rv  rX   rj   r   r   r  r  )"r  r   r  r   r  r   r  r  input_shape
input_ndimgrad_out_cast
input_castweight_cast	bias_castaxis
inner_dims
outer_dimsinner_dim_indicesouter_dim_indicesrs  r  Mx_hat
grad_x_hatabr  r  r  rL   r  d_weightr  r?   s"                                    @r.   native_layer_norm_backwardr    sM    +KJ3EK@@9 9 9 9E6409 9 95M:{I $$$,---DTUU#JUdU#J#%#%: ( (99$$Q''''$$Q''''ZAZAAvva,7NDEOOK(((3>q>KEOOK.///t3>q>KEOOK.///t
 	

 T:>>#3#344DT:>>#3#344D$$&E"[0

"
QA	*/66A	:u	%	%B	2($	/	/B	5"		BEBJE $G!%H#F1~ %!8u$1~ -+1 !!A%%y!68I5QQHH$u,H1~ +)/ !!A%%Y}.?GGFF"((**F 	GU[))Hek**FEK(( r-   c          
          t          | |||||||          }||	|
f}t          |          D ]:\  }}|3t          ||         |j                   t	          |||         d           ;|S r  )r  r  r   r  r   )r  r   r  r   r  r   r  r  rN  rO  r  r   rY   rs  rK   s                  r.   native_layer_norm_backward_outr
    s     (%)4vt[ F d#J&!! Q Q1=jmQW555Q
14PPPPr-   running_meanrunning_varmomentum
functionalc	                    dgt          t          d|                                                     z   }	t          j        | j                  }
|}|}|r$t          j        | j                  }
|                     |
          }t          j        ||	dd          \  }}t          j	        ||z             }| |z
  |z  }t          j
        ||	          }t          j
        ||	          }|%||z  d|z
  |z  z   }|s|                    |           |g|                                 | j        d         z  }t          j
        ||	          }|||dz
  z  z  }||z  d|z
  |z  z   }|s|                    |           n||J |                    |
d          }|}|                    |
d          }|}|}dt          j        ||z             z  }| j        j        dk    r|}|}n*|                     d	          }|                     d	          }t%          ||                                 dz
            }t%          ||                                 dz
            }| |z
  |z  }|>|                                }t%          ||                                 dz
            }||z  }|>|                                }t%          ||                                 dz
            }||z   }| j        j        dk    r6|                    | j                  }|                    | j                  }|                    | j                  ||||fS )
Nr   r%   r   T)rP   
correctionr	  r$   )r   r   r  r   )r~  rU   rP   rF   r  r   r>   rj   var_meanrsqrtr  r   r  r  ri  r  r  rv  rX   rA  )r   r   r  r  r  r   r  rD  r  reduction_dimsr?   new_running_meannew_running_var	input_acc
biased_varr   r  r   	save_mean	save_rstdnsqueezed_varunbiased_varinvstds                           r.   native_batch_norm_helperr     sP    S4a 5 5666N3EK@@#!O +)!7DDHH#4H55	 >>a
 
 

D {:+,,$,$&M$77	M$77	#')3q8||6SS 5""#3444"A.A !=^DDL'1A;7L&5X8TTO 3!!/222'K,C,C,C#->TJJ'!nn+<4nHH%ejs!2334<%%$III--I--I uyy{{Q77"6599;;?;;$,&(!!"6599;;?;;&||~~ uyy{{Q77$|E!!LLu{L33	LLu{L33				$$ r-   r  save_invstdc                 F    t          | |||||||d	  	        \  }}	}
}}||	|
fS NFr  r   r   r  r  r  r   r  rD  r   r  r  rW   s               r.   native_batch_normr%  R  sB     *Bvt\;(CQV* *&FIy!Q 9i''r-   c           
         |!|t                               | |||||          S |t          d          |t          d          |r!t                               | |||||||          S t                               | ||||||          S )Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r   _native_batch_norm_legitr}  $_native_batch_norm_legit_no_training)r   r   r  r  r  r   r  rD  s           r.   native_batch_norm_decompositionr)  n  s      3,,648S
 
 	
 <
 
 	
 <
 
 	
  
,,64{HhPS
 
 	
 8864{Hc
 
 	
r-   c                 F   |                      |          }||z   dz
  |z  dk    rP|dk    rJfd|D             }|z  |z
  z
  ||dz
  <   t          j        j        j                            | ||          S t          j        j        j                            | |          S )Nr$   r   c                     g | ]}S r,   r,   )r7   rW   r  s     r.   r9   z(unsafe_chunk_py_impl.<locals>.<listcomp>  s    222az222r-   )r5  rj   opsr   r  r  r  r   )r{  r  rP   r  r  r  s        @r.   unsafe_chunk_py_implr-    s    {{3HV#a'F2JQ8q==22226222",
V0Ch0N"OFQJy~5==fkSVWWW9>&--fj#FFFr-   c           
      N    t           j                            | ||||d||          S r"  )r   r'  r  )r   r   r  r  r  r  rD  s          r.   r(  r(    s7     (00	 	 	r-   c                 F    t          | |||||||d	  	        \  }}	}
}}||	|
fS r"  r#  r$  s               r.   r'  r'    sB     *Bvt\;(CQV* *&FIy!Q 9i''r-   c                 F    t          | ||d d |||d	  	        \  }}}}	}	|||fS r"  r#  )
r   r   r  r   r  rD  r   r  r  rW   s
             r.   !_native_batch_norm_legit_no_statsr1    sA     *BvtT48S%* *&FIy!Q 9i''r-   c                 z    t          | |||||||d	  	        \  }}	}
}}|
J d            |
J d            ||	|
||fS )NT#new_running_mean should not be None"new_running_var should not be Noner#  )r   r   r  r  r  r   r  rD  r   r  r  r  r  s                r.   #_native_batch_norm_legit_functionalr5    sw    " 	!vt\;(CQU	 	 '')N'''&&(L&&&9i)9?JJr-   c           	         t           j                            | ||||d|          }d}|t           j        j        j        k    r t           j                            | |          }t          j        |t           j        | j        | j	                  S )a  
    Return a reserve tensor for batch norm, used only by cudnn to pass forward state to the
    backward pass. This is needed for `_batch_norm_with_update` and `_batch_norm_no_update`,
    which support a variety of backends including cudnn. We create this tensor here to get
    the correct shape in the traced graph if we detect that will call the cudnn kernel,
    and rely on DCE to avoid materializing this tensor.
    Tr   )r   layoutr  )
rj   _C_select_batch_norm_backend_BatchNormBackendCudnn(_get_cudnn_batch_norm_reserve_space_sizeemptyuint8r7  r  )	r   r   r  r  r  rD  r   backendreserve_sizes	            r.   _get_batch_norm_reserve_tensorrA    s      h11vt\;c G L%(,222xHHPXYY;EKU\   r-   c                 t    t          | ||||d||d	  	        \  }}}	}
}
t          | |||||d          }|||	|fS )NTFr   r  rA  r   r   r  r  r  r  rD  r   r  r  rW   reserves               r.   _batch_norm_with_updaterG  	  sp     *B
* 
*&FIy!Q -vt\;d  G 9i00r-   c                     t          | ||||d||d	  	        \  }}}	}
}t          | |||||d          }|

J d            |
J d            |||	||
|fS )NTrC  r3  r4  rD  )r   r   r  r  r  r  rD  r   r  r  new_rmnew_rvrF  s                r.   "_batch_norm_with_update_functionalrK  $  s      	!vt\;hT	 	 -vt\;d  G DCIy'66BBr-   c                 t    t          | ||||d||d	  	        \  }}}	}
}
t          | |||||d          }|||	|fS )NFrC  rD  rE  s               r.   _batch_norm_no_updaterM  ?  sp     *B
* 
*&FIy!Q -vt\;e  G 9i00r-   c                     |J t          j        |           |k                         t           j                  }|                    |           | z  d|z  z  }||fS )Nr   rh   )rj   rR  r>   r>  r9  )r   rZ  r   r  rT  s        r.   _fused_dropout_decompositionrO  Z  s`     OE""Q&***==D
,,u


%q
1C;r-   )r   r7  r  
pin_memorynon_blockingra  r  rP  rQ  ra  c                   |r|t           j        k    s
J d            |r
J d            t          | t           j        t          t
          t          t          f          sJ |4|2|0t          | t           j                  r|                                 S | S d}t          | t           j                  r| }nt          j	        |           }|Z||j
        k    rO|-|j        dk    r"t           j                            ||          }d}t           j                            ||          }|$|s"t           j                            ||          }d}|t          j        ||          S |S )NTODOFr  Tr`  )rj   stridedr5   r   r  r  r  complexr  scalar_tensorr  r  _primsconvert_element_type
device_put)	r8   r   r7  r  rP  rQ  ra  dtype_convertedx_tensors	            r.   _to_copyr\  d  s^    85=000&000!!6!!!a%,UD'BCCCCC~%-M,Aa&& 	7799HO!U\"" *&q))f77!5!5|885IIH"O<**8V<<<44XuEE {8=AAAAOr-   c                 6    t                               |           S r<   )r   alias)r8   s    r.   nop_decompositionr_    s     ::a==r-   out3exponential_average_factorepsilonc           
      0   t                               | |||||||          \  }}	}
|r%||	|
|                     dt          j                  fS ||                    d          |                    d          |                     dt          j                  fS )Nr  r   )r   r%  rv  rj   r>  )r   r   r  r  r  r   ra  rb  r  r  r  s              r.   cudnn_batch_normrd    s     $$"	 	GAq!  C1aU[AABB	EK00	 r-   c                     t          |          D ]<\  }}|dk    r1|| j        k     r| j        |         |k    s|                     |          } =| S r]   )r  r  r  rV   )r8   broadcast_maskr  r  s       r.   _broadcast_batch_norm_backwardrg    sW    // " "
d199dQVmm0E0ED!!AHr-   rF  c                 2    t          | |||||||||	
  
        S r<   )native_batch_norm_backward)r  r   r   r  r  r  r   rJ  rD  r  rF  s              r.   batch_norm_backwardrj    s4     &  r-   c
                 >  & |j         }
||j         }n|
}t          j        |j                   &&fd| ||||||fD             \  }}}}}}}|j        }|                                }|dk    s
J d            d}t          t          |                    ||         z  }|}|}|r||J n||J |}t          j        ||z             }dg|z  }||         ||<   g }t          |          D ]}||k    r|
                    |           t          ||          }d|z  }t          j        ||          }t          j        |||z
  z  |          }t          ||z  |          }t          t          j        ||z  ||z            |          } |t          ||          dz  }!nt          ||z  |          }!|r||z
  | z  }"||"z
  |z
  |!z  }#n||!z  }#|	d         r||z  }$nd }$|	d         r|}%nd }%|#                    |
          t          |$|          t          |%|          fS )Nc              3   J   K   | ]}||                               n|V  d S r<   r  r  s     r.   r  z-native_batch_norm_backward.<locals>.<genexpr>  sM       	 	 $%=a	 	 	 	 	 	r-   r%   z$rank of the input must be at least 2r$   rh   )r   rF   r  r  rP   r-  r~  rj   r  rU   ry  rg  r   r   r>   r  )'r  r   r   r  r  r  r   rJ  rD  r  r  weight_dtyper  r  r  running_mean_castrunning_var_castsave_mean_castsave_invstd_castr  
input_rankr  num_featuresr   r  rf  reduction_axesrs  r  grad_output_sumdot_p	grad_mean
proj_scale
grad_scaleprojrY   rp  	grad_biasr?   s'                                         @r.   ri  ri    s    +K|"3EK@@	 	 	 	 
	 	 	 +KJ???B???D[))**[->>LDF 5).>.J.J.J.J ,1A1M1M1M -344!"j 0N&t,N4 "N: % %99!!!$$$)$??DDi~>>OImzD'89>JJE./E~VVI/	%$,QW0X0XZhiiJ3FNKKcQ

3[ .
 

  0T!Z/$t+y8JF

"Z/
1~ fn1~ #			 	k""K..I|,, r-   c
                    t          | |||||||||	
  
        }|
||f}t          |          D ]:\  }}|3t          ||         |j                   t	          |||         d           ;|S r  )ri  r  r   r  r   )r  r   r   r  r  r  r   rJ  rD  r  rN  rO  r  r   rY   rs  rK   s                    r.   native_batch_norm_backward_outr}  D	  s    " ( F d#J&!! Q Q1=jmQW555Q
14PPPPr-   save_varc                 L    t                               || |||||d|g d
  
        S NT)TTTr   ri  )r   rn   r   r  r  r  r~  rb  s           r.   miopen_batch_norm_backwardr  j	  s@     **  r-   reserveSpacec	                 L    t                               || |||||d|g d
  
        S r  r  )	r   rn   r   r  r  r  r~  rb  r  s	            r.   cudnn_batch_norm_backwardr  	  s@     **  r-   c                 ^   | j         | j        t                    t          j        dv fd           | j        dd          D ]}t          j        |dk    fd           d         |d         z  dk    rd         |d         z  dk    rt          d t          dd          |          D                       }t          d t          dd          ||          D                       }t          j        j        	                    | ||          S d	 d
 fd} |d         |d                   \  }}}}	 |d         |d                   \  }
}}}| dt          |d          |
f         }|	s|st          j        |d          S d } |||||	d          \  }} |||||d          \  }}d }t          t          |j        d                   t          |j        d                             D ]'\  }}||d|d d |f         }||d|d d |f         z   }(|||z  z  S )Nr  c                      d  S )Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r,   r  s   r.   r   z%adaptive_avg_pool2d.<locals>.<lambda>	  s    RDRR r-   rb  r   c                  *    dt                      dS )Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape r#  r  r  s   r.   r   z%adaptive_avg_pool2d.<locals>.<lambda>	  s&     I9>uI I I r-   rT   c              3   &   K   | ]\  }}||z  V  d S r<   r,   )r7   rs  r*  s      r.   r  z&adaptive_avg_pool2d.<locals>.<genexpr>	  s*      GG$!QqAvGGGGGGr-   c              3   4   K   | ]\  }}}||d z
  |z  z
  V  dS )r$   Nr,   )r7   rs  r*  r  s       r.   r  z&adaptive_avg_pool2d.<locals>.<genexpr>	  sE       
 
 '1aAQ!O
 
 
 
 
 
r-   c                 6    t          j        | |z  |d          S )Ntruncrounding_moderj   divr  r  r  s      r.   start_indexz(adaptive_avg_pool2d.<locals>.start_index	  s    yQ9999r-   c                 H    t          j        | dz   |z  |z   dz
  |d          S )Nr$   r  r  r  r  s      r.   	end_indexz&adaptive_avg_pool2d.<locals>.end_index	  s*    y!a%1q1,awGGGGr-   c                    t          j        |t           j                  } |||           }| |z  dz   }| |z  }|dk    p||z  dk     }|r|dz  }n|dk    r|dz  }t          j        |t           j                  }|                    d          |z   }|rLt          j        | dz
  |j        |j                  }	t          j        ||	          } |||           }
|
|z
  }n|}||||fS )Nr  r$   r   rT   r  )rj   r  r  rV   rV  r   r  minimum)in_sizeout_sizeorangei0	maxlengthin_size_modadaptive	range_maxr  maxvali1r  r  r  r  s               r.   compute_idxz(adaptive_avg_pool2d.<locals>.compute_idx	  s1   hvU[III[733 x'!+	(#q(GH{,Ba,GH 	NIIANIL6MMM	ll2* 	 (!39SZ  F -V,,C 68W55B"WFFFFIx//r-   .rJ  )r   rT   r   c                     t          |t                    r| |fS |dk     sJ ||                    d          k    }|dk    rt          |d          }t	          j        | |d          } t          ||           }| |fS )Nr   rT   rb  rJ  r   )r5   r   rV   rX   rj   rm  )valsr  r  r  rP   r  s         r.   
maybe_maskz'adaptive_avg_pool2d.<locals>.maybe_mask	  s    fg&& 	 < 7777 0 0 4 44Dbyy(q11$T455D&vt44F<r-   )r  rP   r   )r  r  r  rj   r   r  r  nnr  
avg_pool2drX   r   r   rU   )r   r  r  r  kernelr  idxhlength_hrange_max_h
adaptive_hidxwlength_wrange_max_w
adaptive_wr  r  retrs  jr  r  r  r  r  s                      @@@@@r.   adaptive_avg_pool2dr  	  s   
 \FKEu::D	LRRRR   [ 
 
FI I I I	
 	
 	
 	
 Ry;r?"a''E"IB,G1,L,LGG#eBCCj+*F*FGGGGG 
 
+.uRSSz;+O+O
 
 
 
 
 x"--eVVDDD: : :H H H0 0 0 0 0 0 0@ /:k%)[QS_.U.U+D(K.9k%)[QS_.U.U+D(K'a00$67D .j .z$H----       Zhjb  ND(  Zhjb  ND(
 Cdjn--uTZ^/D/DEE + +1;sAqqq!|$CCS!QQQ\**CC(X%&&r-   )ro   r{  c                ,    t          | |||d|          S )NTinplacero   
_index_addr8   rP   r  r{  ro   s        r.   
index_add_r  
  s     aeVTGGGGr-   c                ,    t          | |||d|          S )NFr  r  r  s        r.   	index_addr  
  s     aeVU%HHHHr-   r  c                "   t          j        | j                  t          j        j        dk    fd           j        dk    r                    d          nd|j        dk    r|                              ndt          j        k    fd           dk    rbt          j        | j                  t          j        t          k    p!t          j	        t                              fd           |z  }| j        dk    }|r|                     d          n| }dz  fz   }|rt          j        nt          j        }	 |	|||d          }
|r| S |r|
                    d          n|
                                S )	Nr$   c                      d j          dS Nz(Index should have dimension 1 or 0 (got rE  r  r  s   r.   r   z_index_add.<locals>.<lambda>-
      H5:HHH r-   r   c                      d d d S )NzNumber of indices (z') should be equal to tensor.size(dim) (z), for dim=r,   )rP   
index_sizer|  s   r.   r   z_index_add.<locals>.<lambda>3
  s"    sjssYdssmpss r-   c                  0    dt                      d dS )Nzalpha argument of type z cannot be safely cast to type !)r  )ro   python_types   r.   r   z_index_add.<locals>.<lambda>:
  s"    hd5kkhhZehhh r-   r<   Tr+  )rF   canonicalize_dimsr  rj   r   r5  dtype_to_typer   r  is_weakly_lesser_typer  rV   r   
index_put_	index_putr  r  )r8   rP   r  r{  r  ro   zero_dimr]  r  r  r   r  r  r|  s    ``  `     @@@r.   r  r  !
  s    
!!&#
.
.C	L
aHHHH   #(*//AqJ&,kAoo&++c"""1K	Lz!ssssss   zz)!'224 E*4;;DDhhhhh	
 	
 	

 %v{H#	*QB
C-5(
"C#*>I
)BV
5
5
5C @!)?s{{1~~~s~~/?/??r-   r   c           
         t          j        t          |           dk    d            t          |           }| d                                         }|dd          }t	          d | D                       }|r||f}n||f}||z   }| d                             ||          }dt          |          z  }	t          |          D ]}
| |
         }t                              ||	d||                    d          z
  fz   |          }|rt          	                    ||d|
          }bt          	                    ||d|
          }|S )Nr   c                      dS )Nz#received an empty list of sequencesr,   r,   r-   r.   r   zpad_sequence.<locals>.<lambda>L
  s    -R r-   r$   c              3   @   K   | ]}|                     d           V  dS r  r5  r6   s     r.   r  zpad_sequence.<locals>.<genexpr>P
  s,      //!&&))//////r-   )r   r   rP   r  )
rj   r   r  r5  r   rO  rU   r   rw  r  )	sequencesbatch_firstpadding_valuesequences_sizemax_sizetrailing_dimsmax_lenout_dimsr   dim_paddingsrs  currseqrows                r.   pad_sequencer  I
  sd    
LY!#%R%RSSS^^N|  ""HQRRLM//Y/////G -"G,^,-'H
A,

-
8
8CC...L>"" @ @A,""\Q',,q//(A$BBM
 
  	@%%c3AQ%??CC%%c3AQ%??CCJr-   c                 *    t          | |||d          S )NTr  _index_copyr8   rP   r  r{  s       r.   index_copy_r  d
  s    q#ufd;;;;r-   c                 *    t          | |||d          S )NFr  r  r  s       r.   
index_copyr  i
  s     q#ufe<<<<r-   c                   t          j        | j        |          }t          j        j        dk    fd           | j        dk    }|r|                     d          n| }j        dk    r                    d          nd|z  fz   }|rt          j        nt          j        } ||||          }	|r| S |r|		                    d          n|	
                                S )Nr$   c                      d j          dS r  r  r  s   r.   r   z_index_copy.<locals>.<lambda>u
  r  r-   r   r<   )rF   r  r  rj   r   rV   r   r  r  r  r  )
r8   rP   r  r{  r  r  r]  r  r  r   s
     `       r.   r  r  o
  s     
!!&#
.
.C	L
aHHHH  
 v{H#	*QB"'*//EOOAuE
C-5(
"C#*>I
)BV
$
$C @!)?s{{1~~~s~~/?/??r-   c                    t          j        |                     d          |           }t          j        t          j        |                      }| j        r|                     d          }n|}|t          j        |          z
  |fS )Nr,   r  )rj   r  rv  ri   r   r  rN  )r}   r   rl   r   s       r.   log_sigmoid_forwardr  
  su     -r**D
1
1C	59T??"##A| %%Q''r-   lowhighc                     t          j        | j        t          |          t          |          | j        | j        |          S )N)r  r  r   r  r   )prims_uniform_helperr  r   r   r  )r8   r  r  r   s       r.   r   r   
  sB      	cNNt__gx   r-   c                 L    |                      t          | |||                    S r<   )r   r   )r}   r  r  r   s       r.   uniform_r  
  s"    ::gdCy99:::r-   c                 p   t          |           dz
  }|At          j        |d u d            t          j        t          |          |k    d            |S |t          j        |d u d            t          j        t          |          |k    d            g }t          |          D ]u\  }}t	          |          |k    r/|                    | |dz            t	          |          z             G|                    t          | |dz            |z                       v|S t          j        dd            d S )Nr%   c                      dS Nz9Must specify exactly one of output_size and scale_factorsr,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
      O r-   c                      dS N r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  s    R r-   c                      dS r  r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  r  r-   c                      dS r  r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  s    r r-   Fc                      dS r  r,   r,   r-   r.   r   z.upsample_compute_output_size.<locals>.<lambda>
  s    R r-   )r  rj   r   r  r  ry  r   )r;  r  scale_factorsspatial_dimensionsrs  r  s         r.   upsample_compute_output_sizer  
  sg   Z1,T!OO	
 	
 	
 	S%%);;ZZHHH 4OO	
 	
 	
 	S''+==zzJJJm,, 	C 	CDAq1vv{{"":a!e#4s1vv#=>>>>""7:a!e+<q+@#A#ABBBB	LRR    r-   c                     | d S | |         S r<   r,   )scalesr  s     r.   get_scale_valuer  
  s    ~t#;r-   r  c                     t          |                                 ||          }|r|nd gt          |          z  }t          | ||          S r<   r  r5  r  _upsample_nearestr   r  r  osizer  s        r.   _upsample_nearest_vecr  
  sO     ){MRRE&?TFSZZ,?  UE6222r-   c                     t          |                                 ||          }|r|nd gt          |          z  }t          | ||d          S NTexactr  r  s        r.   _upsample_nearest_exact_vecr
  
  sT     ){MRRE&?TFSZZ,?  UE6>>>>r-   c                    g }t          |          }|rdnd}t          |          D ]}||         }| j        | |z            }	||         |	|	||         z  z  n|	|z  }
t          j        |t          j        | j                  }||z   |
z                      t          j                  }t          |dz
  |z
            D ]}|	                    d          }|
                    |           |S )Nr   r   r  r$   rT   )r  rU   r  rj   r  r   r  r>   r  rV   ry  )r   r  r  r	  r  num_spatial_dimsr  r  r  isizerp   output_indicesinput_indicesrW   s                 r.   !_compute_upsample_nearest_indicesr  
  s    G;''"SSsF#$$ & & A--12/5ay/D*++%RW-e5=VVV(61U:>>u{KK'!+a/00 	8 	8A)33B77MM}%%%%Nr-   )preserve_memory_formatr  r  c                 &    t          | ||g          S r<   r  r   r  r  s      r.   upsample_nearest1dr    s     UK&:::r-   c                 *    t          | ||gd          S r  r  r  s      r.   upsample_nearest_exact1dr  "  s     UK&FFFFr-   scales_hscales_wc                 (    t          | |||g          S r<   r  r   r  r  r  s       r.   upsample_nearest2dr  0  s     UK(H1EFFFr-   c                 ,    t          | |||gd          S r  r  r  s       r.   _upsample_nearest_exact2dr  =  s      UK(H1ETRRRRr-   scales_dc                 *    t          | ||||g          S r<   r  r   r  r  r  r  s        r.   upsample_nearest3dr"  L  s     UK(Hh1OPPPr-   c                 .    t          | ||||gd          S r  r  r!  s        r.   _upsample_nearest_exact3dr$  Z  s+     {Xx:$   r-   r	  c                 8   t          | |||          }d d g|z   }t                              | |          }|j        dk    rYt	          j        |           }| j        d         }| j        j        dk    r|dk     rt          j
        }|                    |          }|S )Nr  rJ  r$   cudar`  )r  r   _unsafe_indexr  rF   r   r  r  r  rj   re  r  )	r   r  r  r	  spatial_indicesr  r   ra  
n_channelss	            r.   r  r  l  s     8{F%  O Tl_,Gw//F{a3E:: [^
<&&:>>!3M"""??Mr-   c                      |r|rdn|rdn|rdndt                     z  dk    sJ t                                  fdt          dt                               D             S )Nr  rJ  r   r%   r   c                 D    g | ]}t          ||z                      S r,   r  )r7   rs  
group_sizeparamss     r.   r9   z!gather_params.<locals>.<listcomp>  s<       ./fQZ'())  r-   )r  rU   )r-  
has_biaseshas_projectionsr,  s   `  @r.   gather_paramsr0    s     o 

	 

	 


v;;#q(((#f++(((    38CKK3T3T   r-   c                     |r3| d|z           |d|z           }}| d|z  dz            |d|z  dz            }}n| |         ||         }}d\  }}||||fS )Nr%   r$   NNr,   )r-  hiddensrs  bidirectional
cur_params
cur_hiddenbidir_paramsbidir_hiddens           r.   params_hiddensr9    sr     0!'AAJ
%+AEAI%6A	8Jl!'GAJJ
%/"lz<==r-   c                     ||k    sJ |                     |                     d|||z
                       |                     dd|          S ru   )ry  r6  )r6  last_batch_size
batch_sizer3  s       r.   update_hidden_for_packedr=    sT    Z''''NN:$$Q
Oj4PQQRRRQ:...r-   c           	      ~    ||k    r| S ||k     sJ t          j        | |                    d|||z
            f          S ru   )rj   concatr6  )r6  r;  r<  
inp_hiddens       r.    update_hidden_for_packed_reverserA    s\     *$$Z''''<a*2NOO	
  r-   c           	         |d         }|d         }|r|d         nd }	|r|d         nd }
g }g }|r|d         n|d         }|                     dd|          }t          j        | t          |                    }|r|d d d         }|D ]d} | j        d         }||k    rn'|rt          ||||          }nt          ||||          } || |||	||
          }|}|                    |           e|r|                                 n)|                    |           |                                 t          j	        |d          }|st          j	        |d          n|}||fS )Nr   r$   r%   r   rT   )
r6  rj   r  r~  r  rA  r=  ry  reverser7  )inphiddenr-  r.  	hidden_fnbatch_sizesrC  	ih_weight	hh_weightih_biashh_biasstep_outputr3  r;  r6  	split_inprs  r   
hidden_outs                      r.   one_layer_rnn_datarO    s    q	Iq	I%/fQii4G%/fQii4GK"$G)0Dk"ook!nOq!_55JCk!2!233I $dddO	 ' 'IaLa 	9OQ JJ 2OQ J YsJ	7IwWW
:&&&& z"""
)K
#
#C.5E7A&&&:J
?r-   c                       fd}|S )Nc                 H     t          j        |||          | z             S r<   r   linearrs  r6  rH  rJ  rI  rK  nonlinearitys         r.   rL   zrnn_cell.<locals>.inner  s&    |AHZGDDqHIIIr-   r,   rU  rL   s   ` r.   rnn_cellrW    s(    J J J J J Lr-   c                       fd}|S )Nc                 t    t          j        | ||          }  t          j        |||          | z             S r<   rR  rT  s         r.   rL   zrnn_cell_data.<locals>.inner  s9    HQ	7++|AHZGDDqHIIIr-   r,   rV  s   ` r.   rnn_cell_datarZ    s(    J J J J J Lr-   c           	         |d         }|d         }|r|d         nd }|r|d         nd }	t          j        | ||          }
|r|
                    d          n|
}
|                    d          }g }|
D ]'} |||||||	          }|                    |           (|r|                                 t          j        |d          }||                    d          fS )Nr   r$   r%   r   )	r   rS  fliprV   ry  rC  rj   r7  r  )rD  rE  r-  r.  rF  rC  rH  rI  rJ  rK  precomputed_inputr6  rL  rs  r   s                  r.   one_layer_rnnr^    s   q	Iq	I%/fQii4G%/fQii4Gi995<S)..q111BS!!!$$JK ' 'Yq*i)WUU
:&&&& 
)K
#
#C
""1%%%%r-   c                    |d         }|d         }|r|d         }|d         }nLt          j        |                                          }t          j        |                                          }|d                             d          }	|d                             d          }
g }d}|	                    d          }d}d}d}d}|                                 } |	                                }	|
                                }
t           j        j        j                            | |||||	|
|||||||||          }|d         |d         |d         }}}||	                    d          |	                    d          ffS )Nr   r$   r%   r   F)
rj   r  r5  rV   r  r,  r   mkldnn_rnn_layerr  r  )rD  rE  r-  r.  rC  w0w1w2w3hxcxrG  modehidden_size
num_layersr4  r  rJ  outputsr[   hycys                         r.   mkldnn_one_layer_lstmrm  	  sq   	B	B $AYAY[##[##			Q		B			Q		BKD''!**KJ MKE ..

C	B	Bin-55





! G$ 
GAJ
2rArzz!}}bjjmm,,,r-   c
                 8   |r|                      dd          n| } g }
t          |          D ]}t          ||||          \  }}}}|r||dz
  k     r|nd} |	| |||          \  }}|
                    |           |r( |	| |||d          \  }}|
                    |           |r-t	          j        ||g|                                dz
            } n|} |dk    r"|r ||dz
  k     rt	          j        | |d          } |r|                      dd          n| } | |
fS )Nr   r$   r   T)rC  )rJ  )	transposerU   r9  ry  rj   r7  rP   rM  )r   rE  r-  r.  ri  rM  rJ  r4  r  layer_fnfinal_hiddensrs  r5  r6  r7  r8  fwd_inp
fwd_hiddenbwd_inp
bwd_hiddens                       r.   _rnn_helperrv  ;  sz    &1;EOOAq!!!eEM: > >=KFA}>
 >
:
Jl $D
QU(:(:''&huj*jQQZ((( 	-"*(|\:t# # #GZ   ,,, 	Iw0'++--!2CDDEEEa<<E<a*q.&8&8M%===E%0;EOOAq!!!eE-r-   c	                    |                     d          }	t          ||d          }t          | |	|||||||t          t          t          t          j                            
  
        \  }
}|
t          j        |d          fS Nr   FrF  )	unbindr0  rv  r   r^  rW  rj   r   stackr   re  r-  r.  ri  rM  rJ  r4  r  rE  r   rq  s               r.   rnn_tanh_inputr}  d       YYq\\F6:u55F$%*)=)=>>> C M1----r-   c	                    |                     d          }	t          ||d          }t          | |	|||||||t          t          t          t          j                            
  
        \  }
}|
t          j        |d          fS rx  )	rz  r0  rv  r   r^  rW  rj   r  r{  r|  s               r.   rnn_relu_inputr    r~  r-   c	                 
   |                     d          }	t          ||d          }t          | |	||||||dt          t          |t          t          j                            
  
        \  }
}|
t          j        |d          fS Nr   FrG  rF  )	rz  r0  rv  r   rO  rZ  rj   r  r{  datarG  re  r-  r.  ri  rM  rJ  r4  rE  r   rq  s               r.   rnn_relu_datar         YYq\\F6:u55F$##EJ//	
 	
 	
 C  M1----r-   c	                 
   |                     d          }	t          ||d          }t          | |	||||||dt          t          |t          t          j                            
  
        \  }
}|
t          j        |d          fS r  )	rz  r0  rv  r   rO  rZ  rj   r   r{  r  s               r.   rnn_tanh_datar    r  r-   c                    t          j        |||          | z   }|                    d|          }|d                                         }	|d                                         }
|d                                         }|d                                         }|
|z  |	|z  z   }||                                z  }||nt          j        ||d           }||fS )NrJ  r   r$   r%   r   r   rS  chunkr   r   )rD  re  rf  rI  rK  	hr_weight	chunk_dimgateschunked_gatesin_gateforget_gate	cell_gateout_gaterl  rk  s                  r.   	lstm_cellr    s    HRG,,s2EKK9--MA&&((G"**,,Ka %%''IQ''))H	r	Wy0	1B	BGGII	B ahr9d&C&CBr6Mr-   c           
         |d         }|d         }|r|d         nd }|r|d         nd }t          |          dk    r|d         nt          |          dk    r|d         nd }	|d                             d          }
|d                             d          }t          j        | ||          }|r|                    d          n|}g }|D ]0} t          | |
||||	d          \  }
}|                    |
           1|r|                                 t          j	        |d          }||

                    d          |
                    d          ffS )Nr   r$   r%   r   r  rJ  r  )r  rV   r   rS  r\  r  ry  rC  rj   r7  r  )rD  rE  r-  r.  rC  rH  rI  rJ  rK  r  re  rf  r]  rL  r   s                  r.   one_layer_lstmr    so   q	Iq	I%/fQii4G%/fQii4G[[A%%q		Fq8H8H6!99d  
		Q		B			Q		Bi995<S)..q111BSK   3B	7IQRSSSB2 
)K
#
#CA

1...r-   c           
      d   |d         }|d         }|r|d         nd }|r|d         nd }	t          |          dk    r|d         nt          |          dk    r|d         nd }
g }g }|r|d         n|d         }t          j        | t          |                    }|r|d d d         }|d         }|d         }|                    dd|          |                    dd|          }}|D ]5} | j        d         }t          j        | ||          } ||k     ru|                    |                    d|||z
            |                    d|||z
            f           |                    dd|          |                    dd|          }}||k    r^t          j	        ||                    d|||z
            fd          }t          j	        ||                    d|||z
            fd          }t          | ||||	|
d          \  }}|}|                    |           7|r|                                 ||f}na|                    ||f           |                                 t          | \  }}t          j        |d          t          j        |d          f}t          j        |d          }||fS )	Nr   r$   r%   r   r  rJ  rT   r  )r  rj   r  r~  r6  r  r   rS  ry  r?  r  rC  r  r7  )rD  rE  r-  r.  rG  rC  rH  rI  rJ  rK  r  rL  r3  r;  rM  orig_hxorig_cxre  rf  rs  rN  hidden0hidden1r   s                           r.   one_layer_lstm_datar    s   q	Iq	I%/fQii4G%/fQii4G[[A%%q		Fq8H8H6!99d  KG)0Dk"ook!nOCk!2!233I $dddO	QiGQiG^^Aq/22GNN	1o5 5B   IaLhsIw// NNIIaOa$788IIaOa$788   YYq!Q''1a););B W^^AO8KLLMq B W^^AO8KLLMq B 3B	7IQRSSSB2 B"X

Bx   =Yw**EIgq,A,AA

)K
#
#C
?r-   c                 >    d } || ||          rt           S t          S )a*  Check whether we could use decompose lstm with mkldnn_rnn_layer.
    All the below conditions need to be met:
        * ``torch._C._get_mkldnn_enabled()`` returns ``True``.
        * All the input args are on CPU.
        * The dtypes of args are either torch.float or torch.bfloat16.
        * Inference.
        * ``has_projections`` returns ``False``.

    Args:
        * input: the input sequence to LSTM
        * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM
        * params: the weight and bias tensors of LSTM
    c                 4   t           j                                        sdS | gt          |          z   t          t	          j        |                    z   }d |D             }t          |          dk    rdS |                                }|t          j        d          k    rdS d |D             }|D ]}|t           j	        t           j
        fvr dS  | j        rdS |d                             d          |d                             d          k    }|rdS dS )	NFc                     h | ]	}|j         
S r,   r  r7   ts     r.   	<setcomp>zEselect_one_layer_lstm_function.<locals>.use_mkldnn.<locals>.<setcomp>d  s    ---18---r-   r$   r  c                     h | ]	}|j         
S r,   r   r  s     r.   r  zEselect_one_layer_lstm_function.<locals>.use_mkldnn.<locals>.<setcomp>l  s    +++a!'+++r-   r   r%   T)rj   r8  _get_mkldnn_enabledr~  r   from_iterabler  popr  r  bfloat16requires_gradr5  )	r   re  r-  rt  devicesr  dtypesr   r/  s	            r.   
use_mkldnnz2select_one_layer_lstm_function.<locals>.use_mkldnn_  s#   x++-- 	5'DHH$tE,?,G,G'H'HH--W---w<<15U\%((((5++7+++ 	 	EU[%.999uu :  	5Q%**Q--2a5::a==8 	5tr-   )rm  r  )r   re  r-  r  s       r.   select_one_layer_lstm_functionr  P  s5      : z%V$$ $$r-   c	                    t          |          dk    s
J d            t          |||d                             d          |d                             d          k              }t          t	          |d         |d                             }	t          | ||          }
t          | |	||||||||

  
        \  }}t          t	          |           }|t          j        |d         d          t          j        |d         d          fS )Nr%   lstm expects two hidden statesr   r$   )	r  r0  r5  r~  r  r  rv  rj   r{  )r   re  r-  r.  ri  rM  rJ  r4  r  rE  rp  r   rq  s                r.   	lstm_implr    s     r77a<<<9<<<6:r!uzz!}}1

1/MNNF#beRU##$$F-eR@@H$ C m,--MM!,a00%+mA>NPQ2R2RRRr-   c	                    t          |          dk    s
J d            t          |||d                             d          |d                             d          k              }t          t	          |d         |d                             }	t          | |	||||||dt          t          |          
  
        \  }
}t          t	          |           }|
t          j	        |d         d          t          j	        |d         d          fS )Nr%   r  r   r$   F)rG  )
r  r0  r5  r~  r  rv  r   r  rj   r{  r  s               r.   lstm_data_implr    s     r77a<<<9<<<6:r!uzz!}}1

1/MNNF#beRU##$$F$#=== C m,--MM!,a00%+mA>NPQ2R2RRRr-   c                 p   |                      dd          }t          j        |||                               dd          }|d         |d         z                                   }|d         |d         z                                   }	|d         |d         |z  z                                   }
||
z
  |	z  |
z   S )Nr   r$   r%   r   )r  r   rS  r   r   rD  r6  rH  rJ  rI  rK  chunked_igateschunked_hgates
reset_gate
input_gatenew_gates              r.   gru_cellr    s    YYq!__NXj)W==CCAqIIN #nQ&77@@BBJ #nQ&77@@BBJq!^A%6%CDJJLLH!Z/(::r-   c                    t          j        | ||                              dd          }t          j        |||                              dd          }|d         |d         z                                   }|d         |d         z                                   }	|d         |d         |z  z                                   }
||
z
  |	z  |
z   S )Nr   r$   r   r%   r  r  s              r.   gru_cell_datar    s    Xc9g66<<QBBNXj)W==CCAqIIN #nQ&77@@BBJ #nQ&77@@BBJq!^A%6%CDJJLLH!Z/(::r-   c	                     t          ||d          }t          | |                    d          ||||||dt          t          |t
                    
  
        \  }	}
|	t          j        |
d          fS )NFr   r  )r0  rv  rz  r   rO  r  rj   r{  )r  rG  re  r-  r.  ri  rM  rJ  r4  r   rq  s              r.   gru_impl_datar    sz     6:u55F$
		!"}UUU C M1----r-   c	                     t          ||d          }t          | |                    d          |||||||t          t          t
                    
  
        \  }	}
|	t          j        |
d          fS )NFr   ry  )r0  rv  rz  r   r^  r  rj   r{  )r   re  r-  r.  ri  rM  rJ  r4  r  r   rq  s              r.   gru_implr    sw     6:u55F$
		!222 C M1----r-   c                     t          |                                 ||          }t          |d          }t          |d          }t          j        j                            | ||||          S Nr   r$   )r  r5  r  rj   r,  r   _upsample_bilinear2d_aar   r  align_cornersr  r  scale_hscale_ws          r.   upsample_bilinear2d_aa_vecr    sb     ){MRREmQ//GmQ//G9>11umWg  r-   c                     t          |                                 ||          }t          |d          }t          |d          }t          j        j                            | ||||          S r  )r  r5  r  rj   r,  r   _upsample_bicubic2d_aar  s          r.   upsample_bicubic2d_aa_vecr    sb     ){MRREmQ//GmQ//G9>00umWg  r-   c                     t          |                                 ||          }|r|nd gt          |          z  }t          | |||          S r<   )r  r5  r  _upsample_linear)r   r  r  r  r  r  s         r.   _upsample_linear_vecr  +  sL     ){MRRE+D]]$#e**1DFE5-@@@r-   r  c                 (    t          | |||g          S r<   r  )r   r  r  r  s       r.   upsample_linear1dr  9  s     E;zJJJr-   c                 *    t          | ||||g          S r<   r  )r   r  r  r  r  s        r.   upsample_bilinear2dr  D  s     E;(?STTTr-   c                 ,    t          | |||||g          S r<   r  )r   r  r  r  r  r  s         r.   upsample_trilinear3dr  S  s&     {MHh+I  r-   c                 P    |r|dk    r| dz
  |dz
  z  ndS ||dk    rd|z  n| |z  S )Nr$   rh   r   r,   )r  r  r  rp   s       r.   _compute_scaler  d  sK     V5=\\#(S.11qH#/EAIIsU{{7XCUUr-   c                 &    |r| |z  S | |dz   z  dz
  S Nr   r,   )rp   	dst_indexr  s      r.   _compute_source_indexr  k  s)     /y  	C(3..r-   weightsweights_precisionc                     t          d t          | |          D                       d|dz
  z  z   }||z	  }t          j        |dd                              t          j                  S )Nc              3      K   | ]F\  }}|                     t          j                  |                     t          j                  z  V  Gd S r<   )r>   rj   r?  )r7   r  r  s      r.   r  z%_sum_tensors_uint8.<locals>.<genexpr>u  sX        26!QU[ADD---     r-   r$   r      )_sum_tensorsr  rj   r   r>   r>  )r  r  r  r   s       r.   _sum_tensors_uint8r  r  s}       :=c7:K:K    	
 1$	%'F ((F;vq#&&))%+666r-   c                     t          j        |                                           }d}t          j        ||j                  }d|d|dz   z  z  z   }|dk    }||                                z
  S )N   r  r   r$   i   )rj   r{  r   r  r  r   )r  
max_weightmax_weight_precision
precisionsvaluesr  s         r.   _compute_weight_precisionr  |  sp    W%%))++J2:;LMMMJ:zA~!677FgD$((**,,r-   c           	           j         d d         \  }} j         dd          }t          |          t          j         t          j        j                  \  } fdfdt          t          |||                    D             }t          t          |           \  }	g }
t          ddggz   D ]dd d gfdt                    D             z   }t                               |          }t          |          }|
                    |           et          t                              D ]h}|	|         |         z
                      dd	                                        fd
t          |
d d d         |
dd d                   D             }
it          |
          dk    sJ |
d         }t          j                   } j        j        dk    r|dk     rt*          j        }t/          |t*          j                  sJ |                    |          }                                 s|                                }|S )Nr%   rX  c                    t          | |	|          }t          j        |j                                      
          }t          ||	                              d          } |j        |j        d         gdg|z  R  }|                    t          j	                  }|dz                       | dz
            }|||fS )Nr  r   r   r   r   r$   r   )
r  rj   r  r  r>   r  r   r.  r  r  )inp_sizer  r  nsqueezescale_factorrs  x_f32r8   xp1r  r   r   s            r.   
get_valuesz$_upsample_linear.<locals>.get_values  s    %h-PP L%,777:::GG%lA}EEKKPSKTTek!n@sh/?@@@HHU[!!1umm1m--a}r-   c           	      F    g | ]\  }\  }}} |||d z
  |z
            S r+  r,   )r7   rs  r  r  r  r  n_dimss        r.   r9   z$_upsample_linear.<locals>.<listcomp>  sL       +A+(F 	
8XvvzA~>>  r-   r   r$   c                 H    g | ]}|         d k    r|         n|         S r  r,   )r7   kr  xp1sxss     r.   r9   z$_upsample_linear.<locals>.<listcomp>  s1    UUU!qtqyybeed1gUUUr-   r   rh   c                 L    g | ] \  }}|t          j        ||z
            z   !S r,   )rj   r   )r7   v1v2xscales      r.   r9   z$_upsample_linear.<locals>.<listcomp>  sD     
 
 
 B 27F+++
 
 
r-   r&     r`  )r  r  rF   rG   rZ  INT_TO_FLOATr  r  r~  r   rU   r   r'  r   ry  reversedr   r>   r   r  r  rj   re  r5   r   r  rQ  round)r   r  r  r  n_batchr)  	inp_sizesrW   r  xs_f32vsr  vrs  r   ra  r  r   r  r  r  r  r  s   ` `             @@@@@@@r.   r  r    s     +bqb/GZABBI^^F'!AN  HAu
          /8	;//0
 0
  F CL))FB	B1vh'(  TlUUUUUUuV}}UUUUuc**#Au--
		!eFmm$$ 
 
)be#**34477>>
 
 
 
 b1gr!$Q$x00
 
 
 r77a<<<<UF /66M |F""zB/fel+++++];;F""$$  Mr-   r  r  c                 "    | j         |j         k    S r<   r  )r  r  s     r.   is_same_sizer    s    7agr-   c                 8    t                               | |          S r<   )r   r  )r8   r  rH   s      r.   _reshape_aliasr	    s     99Qr-   c                 8    t                               | |          S r<   )r   r  )r8   r  s     r.   r'  r'    s    ::a!!!r-   c                 <    t                               | |||          S r<   )r   r  )r8   r  r~   r,  s       r.   r.  r.    s    >>!WeZ888r-   c                    |D ]7}|3t          j        |j        t           j        t           j        fv d            8t          j        |j        t           j        k    d            |                                 dk    r;t           j                            | |          }| 	                    |j
        |          S t          t          |                    D ]<}||         }|0|                    d|                     |          dz
            ||<   =t                              | |                              | |          S )Nc                      dS Nz3tensors used as indices must be long or int tensorsr,   r,   r-   r.   r   z&_unsafe_masked_index.<locals>.<lambda>  r  r-   c                      dS Nz*tensors used as masks must be bool tensorsr,   r,   r-   r.   r   z&_unsafe_masked_index.<locals>.<lambda>      < r-   r   r$   r  )rj   r   r   rl  r  r  r  _meta_registrationsmeta_index_TensorrO  r  rU   r  r   r5  r   r'  rm  )r8   r  r  fillr  meta_resultrs  s          r.   r  r    s+     L
EI66MM  
 
L
ej <<  
 	wwyyA~~/AA!WMMzz++T2223w<<   ? ?
q		A>>GAJa))55teTBBBr-   c                 d   |D ]7}|3t          j        |j        t           j        t           j        fv d            8t          j        |j        t           j        k    d            |                                 dk    r|                                 S t          t          |                    D ]P}||         }|D|
                    |                     |           |                     |          dz
            ||<   Q|                    | d          }t                              | ||d          S )Nc                      dS r  r,   r,   r-   r.   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>  r  r-   c                      dS r  r,   r,   r-   r.   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>	  r  r-   r   r$   r  Tr+  )rj   r   r   rl  r  r  r  r  rU   r  r   r5  rm  r   r.  )r8   r  r  r  r  rs  masked_values          r.   #_unsafe_masked_index_put_accumulater    s$     L
EI66MM  
 
L
ej <<  
 	wwyyA~~wwyy3w<<   H H
!&&))QGGGAJ%%teQ//L!!!Wlt!LLLr-   c                    |                                  }d}|dk     rd}|9|dk    r,dg|z  }|j        d         ||<   |                    |          }n|}| |z  } t          j        ||k    |d          }	|	                    |          }
t          j        | ||
                              |           }t          j        ||k    |d          }|t          j	        j
        k    r |dk    r|                     dd          }||fS |r|                    | j                  }t          j        |||
                              |          }t          j        ||k    |d          }|                                }n+||k                                                        |           }|t          j        j
        k    r|                                }n,|t          j        j
        k    r|                                |z  }||fS )Nr$   r%   r   r,   r   )rP   r  r  rj   rk   rV   gatherr  r#   r)   r~   rO  r  r   r>   r+   r*   )r}   r   r   r   r'  r  r/  r  wr0  safe_target_r   r(  wsums                 r.   _nll_loss_forwardr     s    XXZZFKzzA::E "(aE+E""AAAax+f4fa@@K((55L l4l;;CCKPPPF[</;;FIN(((VaZZ}}R--|##HHTZ  |A{L99AA+NN{6\14;;xxzz,.335588>>IM'''	in*	*	*,<r-   c                 Z   |                                  dk    r|                                  dk    s
J d            |                                 dk    s
J d            |                                  dk    o|                                 dk    }|s7| j        d         |j        d         k    sJ d| j         d|j         d            | j        d	         }|E|                                 dk    r|                                |k    sJ d
| d|j                     t          | ||||          S )Nr   r%   rA  r$   rB  rC  rD  rE  rT   z/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rP   r  r  r   )r}   r   r   r   r'  rG  	n_classess          r.   nll_loss_forwardr#  M  sL    88::>>dhhjjAooo/Pooo

D 	 88::?8vzz||q'8L J
1a(((IDJII&,III 	)(( 
2I>

fllnn	999 J  J  J  |B  |H  J  J :99 T669lKKKr-   c                 (    t          | ||||          S r<   )r   )r}   r   r   r   r'  s        r.   nll_loss2d_forwardr%  i  s     T669lKKKr-   Ac                 0    |dz   | z  |dz   z
  | z  | z  dz   S )Nr%   r   r$   r,   r8   r&  s     r.   _upsample_cubic_convolution1r)  w  s(    UaK1q5!Q&*Q..r-   c                 <    || z  d|z  z
  | z  d|z  z   | z  d|z  z
  S )Nr     rJ  r,   r(  s     r.   _upsample_cubic_convolution2r,  {  s0    UQU]a!a%'1,q1u44r-   r  c                    d}| j         t          j         d          k    rt          j        | d| z
  gd          }t          j        | dz   d| z
  gd          }t          ||          }t	          ||          }t          j        |d          \  }}t          j        |d          \  }}	|||	|fS t          | dz   |          t	          | |          t	          d| z
  |          t          d| z
  |          fS )Ng      r  rh   r   r   r  )r  rj   r{  r,  r)  rz  )
r  r&  tt1tt2w03w12ra  rd  rb  rc  s
             r.    _upsample_get_cubic_coefficientsr2    s   Ax5<&&&&k1cAg,A...k1s7C!G,!444*322*322cq)))Bcq)))B2r2~ )S!44(A..(q!44(q!44	
 	
r-   coeffstsc                 n    t          |          }t          d t          | |          D                       S )Nc              3   &   K   | ]\  }}||z  V  d S r<   r,   r7   r  r  s      r.   r  z+_upsample_cubic_interp1d.<locals>.<genexpr>  s*      EEHRREEEEEEr-   )r2  r  r  )r3  r4  coeffs2s      r.   _upsample_cubic_interp1dr9    s6    .r22GEEFG0D0DEEEEEEr-   c                 6    t          t          j        |           S r<   )r   rj   add)r4  s    r.   r  r    s    %)R   r-   	num_stepsc                     | dk    rt          j        d||          S |s| dz
  | z  nd}t          j        | || ||          S )Nr$   r   r  )stepsr  r   )rj   r{  linspace)r<  r  r   r  r  s        r.   _linspace_from_neg_oner@    sY     A~~|AfE::::-:A)a-9	$	$A>1"ayuMMMMr-   thetahr  c                    | j         }| j        }t          ||||                              d|d          }t          ||||                              |dd          }t	          j        d||          }t          j        j                            |ddd          }t          j        j                            |ddd          }t          j        j                            |d	dd          }||z   |z   S )
Nr$   )r$   r$   r$   r  )r   r%   constantr   r  rg  r~   r$   r$   )r%   r   	r   r  r@  r  rj   r  r  r  r  )	rA  rB  r  r  r   r  grid_xgrid_ygrid_ones	            r.   _make_base_grid_4drK    s    KE\F $A}eVDDII!QPQRRF#A}eVDDII!QPQRRFz)5@@@H X $$VjPQ$RRFX $$VjPQ$RRFx"&&xV*TU&VVHF?X%%r-   r  c                    | j         }| j        }t          ||||                              dd|d          }t          ||||                              d|dd          }t          ||||                              |ddd          }	t	          j        d||          }
t          j        j                            |ddd          }t          j        j                            |ddd          }t          j        j                            |	d	dd          }	t          j        j                            |
d
dd          }
||z   |	z   |
z   S )Nr$   )r$   r$   r$   r$   r  )r   r   rD  r   rE  r  r%   r$   )r   r   rG  )rA  r  rB  r  r  r   r  rH  rI  grid_zrJ  s              r.   _make_base_grid_5drO    s?   KE\F#A}eVDDII!QPQSTUUF#A}eVDDII!QPQSTUUF#A}eVDDII!QPQSTUUFz,eFCCCH X $$VjPQ$RRFX $$VjPQ$RRFX $$VjPQ$RRFx"&&xV*TU&VVHF?V#h..r-   c                     |\  }}}}t          | |||          }|                    ddd          | j                            d          z                      d          }|                    |||d          S )Nr  rT   r   r$   rb  r%   )rK  r  rg  rV   r   )	rA  r5  r  r  rW   rB  r  	base_gridgrids	            r.   _affine_grid_generator_4drT    sw    JAq!Q"5!QmLLLI NN2q!$$ux'9'9!'<'<<AA"EED99Q1a   r-   c                     |\  }}}}}t          | ||||          }|                    ddd          | j                            d          z                      d          }	|	                    ||||d          S )NrQ  rT   rJ  r$   rb  r   )rO  r  rg  rV   r   )
rA  r5  r  r  rW   r  rB  r  rR  rS  s
             r.   _affine_grid_generator_5drV    s}    MAq!Q"5!QOOOI NN2q!$$ux'9'9!'<'<<AA"EED99Q1a###r-   c                     t          j        t          |          dv d            t          |          dk    rt          | ||          S t	          | ||          S )N)rJ  r  c                      dS )NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r,   r,   r-   r.   r   z'affine_grid_generator.<locals>.<lambda>  s    U r-   rJ  rQ  )rj   r   r  rT  rV  )rA  r5  r  s      r.   affine_grid_generatorrY    sg     
LD		VUU   4yyA~~(MRRRR(MRRRRr-   rS  interpolation_modepadding_mode_expand_gridc           	          !"#$%&'()*+,- t          j        dv fd           t          j        dv fd           dt          dt          dt          ffd-dt          dt          d	t          dt          fd
+dt          dt          dt          f+fd dt          dt          dt          f -fd} j        \  $%|j        \  })*}|dk    sJ r0|                    d)*|                              )*d          }dt          dt          dt          f$%fd&t          j         j                                      ddd          t          j         j                                      ddd          dt          dt          dt          dt          f&)*fddt          dt          dt          f fd"|d         }	|d         }
dk    r ||	%          } ||
$          }|
                                |
                                c'('dz   (}}'(dz   }}||}}||z
  ||z
  z  }||z
  ||z
  z  }||z
  ||z
  z  }|'z
  |(z
  z  }t          "fd'(|f|||f|||f|||ffD                       S dk    rM ||	%          } ||
$          }|                                }|                                } "||d          S  -|	%          } -|
$          }|
                                '|
                                (|'z
  ,|(z
  }s*,                    d          ,|                    d          }dt          dt          dt          f "$%fd#dt          dt          f#'(,fd!t          !fdt          d           D                       }t!          ||          S )!N)r   r$   r%   c                      d  S )NzInvalid interpolation mode r,   )rZ  s   r.   r   z"_grid_sampler_2d.<locals>.<lambda>  s    B.@BB r-   c                      d  S )NzInvalid padding mode r,   )r[  s   r.   r   z"_grid_sampler_2d.<locals>.<lambda>  s    +Q<+Q+Q r-   coordsr5  rQ   c                 B    r|dz  dz
  n|dz  }|dz  dz
  }| |z  |z   S r  r,   )r`  r5  r   ofsr  s       r.   unnormalizez%_grid_sampler_2d.<locals>.unnormalize  s>     %2CtczCs
Sj3|c!!r-   	twice_low
twice_highc                 ^   ||k    rt          j        |           S |dz  }||z
  dz  }| |z
                                  }t          j        ||          }||z                                                      t           j                  }t          j        |dz  dk    ||z   ||z   |z
            S )Nr%   r   r$   r   )rj   r  r   fmodfloorr>   int8rk   )r`  rd  re  
coords_mincoords_spancoords2extraflipss           r.   reflect_coordinatesz-_grid_sampler_2d.<locals>.reflect_coordinates  s    
""#F+++]
!I-2J&++--
7K00;&--//222DD{AINEJ.j0H50P
 
 	
r-   c                     dk    r| S dk    rt          j        | d|dz
            S r | dd|dz
  z            }n | dd|z  dz
            }t          j        |d|dz
            S )Nr   r$   r%   rT   r   )r`  r5  coords_reflectedr  r[  ro  s      r.   compute_coordinatesz-_grid_sampler_2d.<locals>.compute_coordinates  s    1MQ;vq$(333 Q#6#6vq!tax.#Q#Q  #6#6vr1t8a<#P#P ;/D1H===r-   c                 4     | |          } ||          S r<   r,   )r`  r5  	coords_unrr  rc  s      r.   compute_source_indexz._grid_sampler_2d.<locals>.compute_source_index%  s'    K--	""9d333r-   r%   r$   r  ysc                     t          j        d| k    t          j        | k     t          j        d|k    |k                                   S ru   rj   r  )r  rv  iHiWs     r.   in_bounds_condz(_grid_sampler_2d.<locals>.in_bounds_cond5  sH     GU&rBw0A!r'2PR70S0STT
 
 	
r-   r  wsc                      | |          rndt          	
fd|                     t          j                  |                    t          j                  |fD                       S )Nr$   c              3   p   K   | ]0}t          j        |d                                         V  1dS r  )rj   rk   r  )r7   r  r  r  r  oHoWs     r.   r  z1_grid_sampler_2d.<locals>.clip.<locals>.<genexpr>D  sY       
 
 Ka##((Ar266
 
 
 
 
 
r-   r   )r  r>   rj   r  )r  rv  r|  r  r  r  r  r\  r{  r  r  s      @@r.   clipz_grid_sampler_2d.<locals>.clip=  s    ~b"%%
 $AA1 
 
 
 
 
 
 
 
ee%+e..EK0H0H"M
 
 
 
 
 	
r-   ixiyc                 D     	| ||          \  }}}||f         |z  S r<   r,   )
r  r  r  idx_xidx_yw_C_idxN_idxr  r  s
         r.   get_summandz%_grid_sampler_2d.<locals>.get_summandI  s4    4B??ubue+,r11r-   ).r   ).r$   r   c              3   :   K   | ]\  }}} |||          V  d S r<   r,   )r7   r  r  r  r  s       r.   r  z#_grid_sampler_2d.<locals>.<genexpr>_  sK       
 
R KB""
 
 
 
 
 
r-   c                 N     |           } |          } ||d          S r]   r,   )r  r  r8   r[   rr  r  ry  rz  s       r.   get_value_boundedz+_grid_sampler_2d.<locals>.get_value_bounded~  s;    ##B++A##B++A;q!Q'''r-   rb  c                     | dz
  z   } dz
  |           |           dz   |           dz   |          f}t          |          S )Nr$   r%   )r9  )rb  iy_ofscsr  ix_nwiy_nwtxs      r.   	get_coeffz#_grid_sampler_2d.<locals>.get_coeff  sv    cAg&F!!%!)V44!!%00!!%!)V44!!%!)V44	B ,B333r-   c              3   .   K   | ]} |          V  d S r<   r,   )r7   rb  r  s     r.   r  z#_grid_sampler_2d.<locals>.<genexpr>  s+      ::#yy~~::::::r-   rJ  )rj   r   r   r  r  r  r  r  r  r   rh  r  r   rV   r  rU   r9  ).r  rS  rZ  r[  r  r\  ru  rW   twor8   r[   r  r  ix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se
ix_nearest
iy_nearesttyr3  r  r  r  r  r  rr  r  r  r  ry  rz  r{  r  r  r  r  ro  r  rc  s.   ` ````                     @@@@@@@@@@@@@@@@@@@r.   _grid_sampler_2dr    s    
Li'BBBB   
L	!#Q#Q#Q#Q  "F "# "& " " " " " "

F 

s 

 

PV 

 

 

 


>F 
># 
>& 
> 
> 
> 
> 
> 
> 
> 
>4V 43 46 4 4 4 4 4 4 4 7LAq"bZNAr2s!8888 D yyAr2s++221aRCC
6 
v 
& 
 
 
 
 
 
 

 L18,,,11!Q1==EL18,,,11!Q1==E

 

V 

 

4F 

 

 

 

 

 

 

 

 

 

 

2 2F 2& 2 2 2 2 2 2 2 2 2
 	VAVAQ!!!R((!!!R((xxzz288::uqy%ueaiueu
urz*U
urz*
rEz*U
rEz* 
 
 
 
 t$t$t$t$	 
 
 
 
 
 	
 
q	 	 !!!R((!!!R((XXZZ
XXZZ
{:z1555[B[B



%Z%Z 	!aBaB	(& 	(f 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(
	43 	46 	4 	4 	4 	4 	4 	4 	4 	4 	4 ::::q:::::'333r-   c                 *    t          | ||||          S )N)rS  rZ  r[  r  )r  )r  rS  rZ  r[  r  s        r.   grid_sampler_2dr    s*     	-!#   r-   c                 L    t          j                                         dk    o                                dk     fd           t          j                             d                              d          k     fd            z                      d          S )Nr%   r$   c                  \    d                                   d                                  S )Nzmatrix @ vector expected, got r  r   r}   r  s   r.   r   zmv.<locals>.<lambda>  s(    JJJswwyyJJ r-   r   c                      d                      d           d                      d           d                     d           dS )Nzsize mismatch, got input (r   r8   r$   z), vec (rE  r  r  s   r.   r   zmv.<locals>.<lambda>  sG    `TYYq\\``DIIaLL``RURZRZ[\R]R]``` r-   r   )rj   r   rP   r5  r   r  s   ``r.   r  r    s     
L

a*CGGIINJJJJJ   
L		!#`````   3J"""r-   c                     |,|dz
  |z  dz   }d|z
  | z  |t          j        |           z  z
  }nd|z
  | z  t          j        |           z
  }|||z  }t          ||          S r]   )r   
logsigmoidr   )r}   r   r   
pos_weightr   
log_weightr   s          r.    binary_cross_entropy_with_logitsr    s}    
  1n.2
F
d"j1<3E3E&EFF
d"Q\$%7%77f}i000r-   tensor1tensor2is_outc           
         | j         |j         k    r| |fn|| f\  }}ddlm} |j         dk    r|j         dk    sdS |j        r|sdS | j         dk    rdS  ||                                dk              rdS |j        }|                                }t          d t          |d d         |d	d
         |d	d
                   D                       S )Nr   )rz  r   r%   FTc              3   0   K   | ]\  }}}|||z  k    V  d S r<   r,   )r7   st1st2s2s       r.   r  zshould_fold.<locals>.<genexpr>  sD        S#r 	sRx     r-   rb  r$   rT   )	r  r|  rz  r  r  r  r  r  r  )r  r  r  t1t2rz  t1_shape	t1_strides           r.   should_foldr    s	    $+<7<#?#?gwgwEWFBJJJJJJGqLLRW\\u	  t|quBHHJJ!O,, txH		I  !)CRC.)AbD/8AbD>RR     r-   )pass_is_out)r  c                2
   |                                  }|                                 }|dk    r|dk    sJ |dk    r|dk    rt          j        | |          S |dk    r|dk    rt          j        | |          S |dk    rA|dk    r;t          j        t          j        t          j        | d          |          d          S |dk    r|dk    rt          j        | |          S t          | ||          r.||k    }|r|j        n| }|s|n|dk    r| 	                                n| }|j
        }t          |d d                   }	t          t          j        |	          }
|                                 dk    }|r |	                    |j
        d                    |                    |
|d                   }|rE|                    |                              |	          }|r|j                                        n|S |                    |                              |	          S |dk    r|dk    r|dk    r|                     d          nd}|                     d          }| j
        d d         }|dk    r|                    d          n|                    d          }|dk    r|                    d          nd}g }t)          |dz
            D ]*}|                    |                    |                     +|dk    r|dk    r~|d         |d         k    rl|d         dk    r*| j        r#t-          |                     d          |          S |d         dk    r*|j        r#t-          | |                    d                    S t          t          j        ||                    }|||gz   }t1          |          }|                     |                              |||          }|dk    }|rC||gz   }|                    |                              ||                              d          }n1|||gz   }|                    |                              |||          }|}	|dk    r|	                    |           |dk    r|	                    |           |r;|                    |                              d                              |	          S |                    |                              |	          S t          j        dd            d S )	Nr   r$   r%   rT   rb  r   Fc                      dS )Nz/both arguments to matmul need to be at least 1Dr,   r,   r-   r.   r   zmatmul.<locals>.<lambda>S  s    $U r-   )rP   rj   dotr  r  r  rV   r  rg  r  r  r~  r   operatorr   ry  r.  r  r  r5  rU   r  rf  broadcast_shapesr-  r  bmmr   )r  r  r  dim_tensor1dim_tensor2ro  r  r  sizes_1output_shapefolded_dim1t2_is_matrix	t1_foldedr   r  m1batch_tensor1m2rZ  batch_tensor2rs  expand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded
vector_rhstensor2_expand_sizetensor2_expandeds                               r.   rf  rf    sV    ++--K++--K!q 0 0 0 0aK1,,y'***			kQ..x)))			kQ..}UXeogq&A&A7KKQOOO			kQ..x)))	Wgv	.	. hW  +-	$1WZZ'$XGG+:J:J799;;;PW 	 (GCRCL))X\<88 vvxx1} 	-,,, JJ{GBK88	 	7 \\"%%**<88F-6B69'')))FB<<##((666			kQ.. !,aGLLQ\\"crc*!,qW\\"gll26F6F +aGLLQ#%{Q'' 	2 	2A  a1111
 1q  a M!$444Q1$$)>$gooa00':::Q1$$)>$gwq'9'9:::  $"=-@@ 
  
 3aW<#$899 #>>*=>>FF !R
 
 !A%
 	"6""=233-r221  #7"a"@&~~.ABBJJ$b!    ,??"""??""" 	M#''(899AA"EEJJ<XXX#''(899>>|LLLUUUVVVVVr-   r  r  c                      j         \  }}t          |d         ||          }t          |d         ||          }t          j         t          j        j                  \  }}t          j        |d          j                  	                    |          }	t          j        |d          j                  	                    |          }
t          ||
|          }t          ||	|          }|                    d          }|                                }|                                }||z
                      dd          }||z
                      dd          }|	                    t          j                  }|	                    t          j                  }|dz
  ||dz   |d	z   f}|dz
  ||dz   |d	z   ft          |          t          |          }d
\   j        t          j        k    r:t%                    t%          |          fdD             fd|D             } fd fdt'          fd|D                       } j        t          j        k    rJ t)          ||          }n't+          d t-          ||          D                       }t          j                   }|                    |          }|S )Nr   r$   rX  r  r   rT   r   rh   r%   r2  c                     g | ]?}|d z  z  t          j        |          dz  z                       t           j                  @S r$   r   rj   r   r>   int16)r7   r  weights_precision_xs     r.   r9   z.upsample_bicubic2d_default.<locals>.<listcomp>  V     
 
 
 !**+ejmmc.AAEEekRR
 
 
r-   c                     g | ]?}|d z  z  t          j        |          dz  z                       t           j                  @S r  r  )r7   r  weights_precision_ys     r.   r9   z.upsample_bicubic2d_default.<locals>.<listcomp>  r  r-   c                     t          j        | ddz
            }t          j        |ddz
            }t                              d d ||g          }|S r  )rj   r   r   r'  )rv  r  y_idxx_idxr  in_hin_wr   s        r.   load_boundedz0upsample_bicubic2d_default.<locals>.load_bounded  sT    B4!8,,B4!8,,utT5%&@AAr-   c                      t           fdD                       }j        t          j        k    rJ t	          |          S t          d t          |          D                       S )Nc              3   0   K   | ]} |          V  d S r<   r,   )r7   x_ofsr  r[   s     r.   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s/      BBll1e,,BBBBBBr-   c              3   &   K   | ]\  }}||z  V  d S r<   r,   r7  s      r.   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s*      JJRBGJJJJJJr-   )r  r   rj   r>  r  r  r  )r[   src_xr   ixs_ofsr  r  	weights_xs   ` r.   get_x_interpz0upsample_bicubic2d_default.<locals>.get_x_interp  s~    BBBBB'BBBBB;%+%%&222%eY8KLLLJJCy4I4IJJJJJJr-   c              3   .   K   | ]} |          V  d S r<   r,   )r7   y_ofsr  s     r.   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s-      ;;%,,u%%;;;;;;r-   c              3   &   K   | ]\  }}||z  V  d S r<   r,   r7  s      r.   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s*      LL(2rb2gLLLLLLr-   r`  )r  r  rF   rG   rZ  r  rj   r  r  r>   r  rV   rh  r   r  r2  r   r>  r  r  r  r  r  r   r  )r   r  r  r  r  rW   h_scale_factorw_scale_factorr   rs  r  x_floaty_floatr8   r[   yscaler  iys_ofs	weights_ysrc_yr   ra  r  r  r  r  r  r  r  r  s   `                     @@@@@@@@r.   upsample_bicubic2d_defaultr  V  sH    {Aq$ $D+a.-QQN#D+a.-QQN'5#H#U  HAu 	[^EL999<<5<IIA[^EL999<<5<IIA#NA}EEG#NA}EEG##GAA k  c**Fk  c**F	U[A	U[A1uaQA&G1uaQA&G088I088I/9,,{ek!!7	BB7	BB
 
 
 

 
 
	
 
 
 

 
 
	
      K K K K K K K K K ;;;;7;;;;;E{ek!!"...#E96IJJLLc%6K6KLLLLL /66M];;FMr-   c                 f   t          j        t          |          t          |          z   dk    d            |]|J t          t          t
          t
          f         t          d t          | j        dd          |          D                                 }|r|nd\  }}t          | ||||          S )Nr$   c                      dS )Nz:Must specify exactly one of output_size and scale_factors.r,   r,   r-   r.   r   z(upsample_bicubic2d_vec.<locals>.<lambda>  s    L r-   c              3   Z   K   | ]&\  }}t          t          |          |z            V  'd S r<   )r   r   )r7   r  rp   s      r.   r  z)upsample_bicubic2d_vec.<locals>.<genexpr>  sM        Au 	!u,--     r-   r%   r2  )
rj   r   r  r
   r   r  r  r  r  r  )r  r  r  r  r  r  s         r.   upsample_bicubic2d_vecr    s     
L[D///14LL   (((#s(O   #AGABBK ? ?    
 
 )6G}}<GW%amWgVVVr-   c                 0      fd}t           ||          S )Nc                     t          j        |  ||z   j                  }|dz
  |dz
  |                                z
                                  z
  S )Nr  r$   )rj   r  r  r   r   middler   dim_idxr  s       r.   r  z_reflection_pad.<locals>.idx  sM    ,ufunQXFFFzVaZ'++--7<<>>>>r-   _reflection_or_replication_padr  r  r  s   `  r.   _reflection_padr    s9    ? ? ? ? ? *	  r-   c                 0      fd}t           ||          S )Nc                 v    t          j        |  ||z   j                  }t          j        |d|dz
            S )Nr  r   r$   )rj   r  r  r   r  s       r.   r  z_replication_pad.<locals>.idx  s9    ,ufunQXFFF{7Avz222r-   r  r  s   `  r.   _replication_padr    s9    3 3 3 3 3 *	  r-   idx_fnc                    t                    dz  t          j        |                                 dz   dz   fv fd           | j         d          }|                                 z
  }fdt                    D             }fdt                    D             }| }t                    D ]Z}d g|                                z  }	 |||         ||         ||                   |	||z   <   t                              ||	          }[t          j	        |          }
|
                    |
          }|S )Nr%   r$   c                  (    d  d dz    d dz    dS )Nreflection_padzd requires r$   zD or r%   zD inputr,   r   s   r.   r   z0_reflection_or_replication_pad.<locals>.<lambda>  s+    OOOqOOsQwOOO r-   c                 2    g | ]}d dz
  |z
  z           S rM  r,   r7   rs  rP   r  s     r.   r9   z2_reflection_or_replication_pad.<locals>.<listcomp>  +    CCC1GAq1-.CCCr-   c                 8    g | ]}d dz
  |z
  z  dz            S rM  r,   r  s     r.   r9   z2_reflection_or_replication_pad.<locals>.<listcomp>  0    HHHWQ#'A+.23HHHr-   r`  )r  rj   r   rP   r  rU   r   r'  rF   r   r  )r  r  r	  	inp_shapenc_dimpadding_leftpadding_rightr   rs  r  ra  rP   s    `         @r.   r  r    sU   
 g,,!
C	L	C!GS1W%%OOOO   IUUWWs]FCCCCCc

CCCLHHHHHU3ZZHHHMF3ZZ 1 1&**,,. &a)A,a@PQQAJ##FC00 /77M];;FMr-   c                     t                    dz  d |j         d          D             fdt                    D             fdt                    D             g }t          |j                  D ]^}dg|j        z  }d||<   |                    t          j        |j        |         |j                                      |                     _|d           | d          d fd	t                    D             fd
t                    D             }fdt                    D             }fdt                    D             t          j
        t          j        fdt                    D                       }t                               |z   d          }	 fd}
t          j        d t                    D              D ]}|t!          dgz            k    rg }g }t                    D ]}||         dk    r|         }|         }n]||         dk    r||         }|         d|         f}n7||         dk    r+||         }|         |         |         z
  |         dz
  f}|                    |           |                    |            |
|	||          }	|	S )Nr%   c                     g | ]}|d z
  S r+  r,   )r7   rB  s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>  s    
)
)
)Q1q5
)
)
)r-   c                 2    g | ]}d dz
  |z
  z           S rM  r,   r  s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>  r  r-   c                 8    g | ]}d dz
  |z
  z  dz            S rM  r,   r  s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>  r  r-   r$   rT   r  c                 H    | \  }}}t          j        ||k    ||k              S r<   rx  )index_rangers  lbubs       r.   index_range_conditionz7_reflection_pad_backward.<locals>.index_range_condition  s(    	2r b!r'222r-   c                 2    g | ]}|         |         z   S r,   r,   r7   rs  r  xyzs     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>*  s&    ;;;1c!f|A&;;;r-   c                 2    g | ]}|         |         z
  S r,   r,   r   s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>+  s&    AAALOc!f,AAAr-   c                 J    g | ]}d |         z  |         z   |         z
   S r)  r,   )r7   rs  dhwr  r!  s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>,  s3    OOOqQQZ,q/1CF:OOOr-   c                 V    g | ]%}|         d |         |         z   |         z   f&S r  r,   )r7   rs  centerr$  r  r  s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>0  sH       HIAs1vQ/-2BBC  r-   c                 2    g | ]} |                   S r,   r,   )r7   rs  r  range_cs     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>4  s)    QQQ00<<QQQr-   r   c                 :   t                    D ];}||         d         ||         d         k     }t          |t                    r|r| c S <t          j        t
          j        
fd|D                       }t
                              	||z   d          }| |z   S )Nr%   r$   c                 &    g | ]} |          S r,   r,   )r7   r  r  s     r.   r9   z@_reflection_pad_backward.<locals>.accumulate.<locals>.<listcomp>B  s%    PPPK"";//PPPr-   r   )rU   r5   r  rM   r   r   r  r  )r   r   index_rangesrs  upper_less_than_lowerr  gr  rP   rn   r  s          r.   r,  z,_reflection_pad_backward.<locals>.accumulate8  s     s 	 	A$0OA$6a9K$K!/66 ;P PPPP<PPP
 
 %%k4S#FFaxr-   c                     g | ]}g d S ))rT   r   r$   r,   r,  s     r.   r9   z,_reflection_pad_backward.<locals>.<listcomp>G  s    #C#C#C1JJJ#C#C#Cr-   r   )r  r  rU   r  ry  rj   r  r  r  rM   r   r   r  r  	itertoolsr   r  )rn   r8   r  r  rs  
view_shapeleft_reflectright_reflectr  r   r,  areaoutsr+  r   r  r  r&  r$  rP   r  r  r  r(  r!  s   ` `             @@@@@@@@@r.   _reflection_pad_backwardr5    s}   
 g,,!
C
)
)!'3$%%.
)
)
)CCCCCCc

CCCLHHHHHU3ZZHHHMG16]] S SS16\

1u|AGAJqx@@@EEjQQRRRR#A
3$%%.C3 3 3 <;;;;c

;;;FAAAAAeCjjAAALOOOOOOE#JJOOOM      MRSVZZ  G QQQQQeCjjQQQ D $$[$F
CHHD        !#C#Cc

#C#C#CD 4 45!s####s 	- 	-AAw!||Qi%ajaB"1o"1vq,q/:aA#A&"1vs1va0@'@#a&1*MKK,,,,z$l33Kr-   r   r   r  c                f    t          j        | ||          }t          j        | ||          }||fS )Nr  )rj   aminr\  )r}   rP   r	  r7  r\  s        r.   aminmaxr8  b  s:     :dW555D:dW555D:r-   r   c                    t                               t          j        t          j        |           d|           |||          S )Nr   r   )r   r   rj   rk   isnan)r}   rP   r	  r   s       r.   nansumr;  j  s7     88EKD 1 11d;;S'QV8WWWr-   r   r7  r  rP  r7  c          	      N    t           j                            d| d||||          S )Nr   r$   r<  r   r  
start_step)rs  r   r7  r  rP  s        r.   arange_defaultr@  p  s2     ;!!	3vf "   r-   c          	      N    t           j                            | |d||||          S )Nr$   r<  r>  )rr  rs  r   r7  r  rP  s         r.   arange_startrB    s2     ;!!sAU6&Z "   r-   c                      ddl m}  || i |S )Nr   )out_dtype_dense)!torch._higher_order_ops.out_dtyperD  )rH   rI   rD  s      r.   out_dtype_decomprF    s)    AAAAAA?D+F+++r-   marginc                 N   	
 t          j                    t          j                   j        d         
 j        d         	t          j        |dk    p|dk    d            t          j         j        dk    o	dk     fd           t          j        j        dk    o                                
k    
fd           Nt          j                  t          j        j        dk    o                                	k    	fd                               d          t          j         d          }||z
   z   }|	                    d          }|dk    r|n||z  }|         z  }t          j
        	 j        	          }t          j        |k    |d          }|t          j        j        k    r|                                S |t          j        j        k    r"|                                |j        d         z  S |                    d
          S )Nr   r$   r%   c                      dS )Nz only p == 1 and p == 2 supportedr,   r,   r-   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    +M r-   c                      d j          S NzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: r  )r   s   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    m`e`kmm r-   c                      d  dj          S )Nz#inconsistent target size, expected r"  r  )nframer   s   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    UfUUv|UU r-   c                      d  dj          S )Nz#inconsistent weight size, expected r"  r  )rP   r   s   r.   r   z#multi_margin_loss.<locals>.<lambda>  s    V#VVVV r-   r  r  r   )rj   
atleast_2d
atleast_1dr  r   r  r  rV   r  rh  r  r  rk   r#   r*   r~   r   r+   r   )r   r   rZ  rG  r   r   url   r  rP   rM  s   ``  `    @@r.   multi_margin_lossrR    s"    U##Ef%%F[^F
+a.C	La!16#M#MNNN	L
a$C1Hmmmm   
Lq5V\\^^v5UUUUU   !&))K163!6VVVVV	
 	
 	
 a  FU000A
UA	AA!VVQAv
,s5<
0
0
0CC6M1a((AIN(((vvxx	im)	)	)uuww##vv!v}}r-   	is_targetc                    | j         |j         t          j        |           } t          j        |          }| j         d         }t          j        t	                    dk    o|dk    fd           t          j        t	                    dk    ok    fd           t          j        ||j                  }|dk    }t          j        t          j        |||          dd	          }||k     }t          j        ||d          }t          j	        | d|
          }	t          j        ||d          }
t          j
        ||
                    d          k    d          }d|	j                            d          z
  | z   }|                    d          }||z  }t          j        |d|          }|t          j        j        k    r)|                    d                                          }n@|t          j        j        k    r|                                }n|                    d          }|                    | j                                                }||fS )Nr$   r%   r   c                      d  S rK  r,   )orig_input_shapes   r.   r   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    r`prr r-   c                      d d  S )Nzinconsistent target size: z for input of size: r,   )rV  orig_target_shapes   r.   r   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    f->ffTdff r-   r  rT   Tr  r  r   rh   )r   rT   )r  rj   rO  r   r  r  r  r7  rk   r  anyrV   Trh  r#   r*   r~   r   r   r+   r>   r   r.  )r   r   r   rP   r  is_endend_idxtarget_masktidx0rQ  tidx1rS  rl   rV  rX  s                @@r.   multilabel_margin_loss_forwardr`    sG    {U##Ef%%F
+a.C	L"/saxrrrr   
L!#M(9=M(Mfffff  
 ,s6=
1
1
1Cr\FjVS#66BMMMG-KKVQ//EU%000AKVR00E	#R!8!88a@@@Iacmmm###e+A	AA	CAIq!$$AIN(((EEgE##%%	im)	)	)EEGGEEgEU[))112CDDIi<r-   )	attn_maskrp   querykey	dropout_p	is_causalra  c          
           j         }t          j        t          j                    fd           t          j                                         dk    o/                                dk    o                                dk     fd           t          j        dk    fd           t          j         j        d         j        d         k    oj        d         j        d         k    d            t          j                             ||d |          \  }}	|	                    d	d
          
                    t          j                  }|	                    d	d
          |	fS )Nc                      d j          S )Nz-query must be FP32, FP64, BF16, FP16 but got r   )rb  s   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s    MMM r-   rJ  c                      d                                  d                                   d                                  S )Nz,q, k, v must be a 4 dimensional tensor, got r  r   )rc  rb  r~   s   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s=    huyy{{hhcggiihh[`[d[d[f[fhh r-   r   c                      d  S )Nz&dropout probability must be zero, got r,   )rd  s   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s    "V9"V"V r-   r   c                      dS )Nz&q, k, v should have the same head sizer,   r,   r-   r.   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s    8 r-   )ra  rd  re  dropout_maskrp   r$   r%   r`  )r   rj   r   rQ  rP   r  r   "_scaled_dot_product_attention_mathr  ro  r  re  )
rb  rc  r~   rd  re  ra  rp   r   r   attns
   ````      r.   *scaled_dot_product_flash_attention_for_cpurn    s    KE	L&&MMMM   
L		q@SWWYY!^@		q0@hhhhhh   
LSVVVV   
LA%+a.(KSYq\U[^-K88  
 :BB C 	 	LFDV a##..U=T.UUFQ""D))r-   c                 <    t          |           fd            }|S )Nc                  J     | i |}| d                              |          S ru   )r   )rH   rI   r   outplace_ops      r.   
inplace_opz$register_inplace.<locals>.inplace_opO  s-    k4*6**Aw}}S!!!r-   r   )aten_oprq  rr  s    ` r.   register_inplacert  N  s7    G$$" " " " %$" r-   c                 f   |                                  s2|                                 st          |          }t          |          }t          j        ||          }t          |t          j                  r|dk    r||z  }|dk    r|S t          |t          j                  r|dk    r| |z  } | |z   S )Nr$   r   )rQ  r  r  rj   r  r5   numbersNumber)r}   batch1batch2rd   ro   r   s         r.   baddbmmrz  W  s     !!## DOO,=,= 4yyE

Yvv&&FeW^,,  

%qyydGN++ tqyyd{&=r-   c                 0    t          j        | |d          S )Nrh  r  r  )r}   rY  s     r.   floor_divider|  h  s     9T58888r-   c                 L    t          j        t          j        | j        d          S r]   )rM   r   r  r   r  )r  s    r.   	sym_numelr~  n  s    HL!'1555r-   r   r   c                    |"t           j                            | g |          S t           j                            | g ||          S )Nr   r  )r   r   dim_IntListIntList_out)r}   r   r   s      r.   sum_defaultr  s  sE     {x##D"E#:::x##D"Es#CCCr-   c           	         t          | t          j                  s| S |Lt          j                            | t          t          |                                                               S t          j                            | |g          S r<   )	r5   rj   r   r   r  dimsr~  rU   rP   )r}   rP   s     r.   squeeze_defaultr    sl     dEL)) 
{|  tE$((**,=,='>'>???|  u---r-   c                 4   t          fdt          t          | j                            D                       }|j        t
          j        k    rt
          j        nd }|                     d|d|          }| ||	                    |j                  z  z  |fS )Nc              3   (   K   | ]}|k    |V  d S r<   r,   )r7   rs  rP   s     r.   r  z)_weight_norm_interface.<locals>.<genexpr>  s'      @@1qCxxQxxxx@@r-   r%   T)r	  r   )
r  rU   r  r  r   rj   r  r  r  r>   )r  r-  rP   keep_dim
norm_dtyper  s     `   r.   _weight_norm_interfacer    s     @@@@c!'ll 3 3@@@@@H !5> 9 9tJ66!Xt:6>>DDGGAG$$$%t++r-   assume_uniqueinvertc                   t          | t          j                  st          j        | |j                  } t          |t          j                  st          j        || j                  }|                                dt          |                                 d          z  k     rt          | ||          S t          | |||          S )Nr  g      $@g(\?r  r  )	r5   rj   r   r{  r  r  rc  isin_defaultisin_sorting)elementstest_elementsr  r  s       r.   isinr    s     h-- G<1EFFFmU\22 L]8?KKKtc(..*:*:E&B&BBBBHmFCCCCm=
 
 
 	
r-   r  c                B   |                                  dk    r t          j        | t          j                  S  | j        g | j        d|j        z  R  }|s||k    }n||k    }t          t          d|j         dz
  d                    }|	                    |          S )Nr   r   r+  rT   r$   r   )
r  rj   
empty_liker  r  r  r  r  rU   rY  )r  r  r  r8   cmprP   s         r.   r  r    s    ~~1
;;;;Dx~D0B)BDDDA != = 
b=--1266
7
7C77s7r-   c                   |                                  }|                                 }|rt          j        ||g          }t          j        |d          \  }}|dd          |d d         k    }	t          j        |	ddgd          }	|r|	                                }	t          j        |	          }
|
                    d||	          }
|
d|                                          S t          j        |          \  }}t          j	        ||          }t          j
        ||                                k     |d          }||         |k    }|r|                                n|}|                    | j                  S )NT)stabler$   rT   r   F)rA  rj   r7  sortrw  logical_notr  r  r  searchsortedrk   r.  r  )r  r  r  r  elements_flattest_elements_flatall_elementssorted_elementssorted_orderduplicate_maskr  sorted_test_elementsrW   r  test_idxr  s                   r.   r  r    ss   $$&&M&..00 + y-1C!DEE(-
<(M(M(M%(,0DD.~1vuMM 	:+7799N//q,??A((())"'*-?"@"@a !5}EE;s%9%?%?%A%AA3JJ"8,=#)2coos{{8>***r-   c                 <    |                      d          }||         S rS   )r.  )r}   r  	flatteneds      r.   taker    s      R  IUr-   c                     |t           j        }|t           j        k    rt          |          }t                              | |j        |          S r8  )rj   re  preserve_formatr   r   resizer  )r}   rY  ra  s      r.   	resize_asr    sE    /----e44;;tU[;FFFr-   )F)r   )r   r   FNr<   r)  )r   NNr$   )rT   FFr  r  rF  )r$   r$   F)Fr   )r   rh   N)r   r$   Nr2  )NNN)r   r   FT)r   r   Fr"  )r   F(  rM   r/  rv  r  r  enumr   r   r   r   r   typingr   r	   r
   r   r   r   r   r   rj   torch._meta_registrationstorch._primsrW  r  torch._prims_common_prims_commonrF   torch.nn.functionalr  r  r   r   r   r   torch._decompr   rE  r   r   r   r   r   r   torch._prims_common.wrappersr   r   r   r   torch.utilsr   rD   torch.utils._pytreer    r8  DispatchKeyr!   str__annotations___opsr,  r   r#   rZ  r  rO   r[  compute_only_pw_cast_for_opmathpw_cast_for_opmathr  pw_cast_for_int_to_realr  rX   ra   rc   rm   r  ry   r  Scalarr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  py_implAutogradCUDA	Generatorr   r   r   r   r   r   r  r*   r~   r  _safe_softmaxr  r  r  rY   r!  r$  r   r&  r2  r?  rH  rK  rQ  rT  rV  rX  r\  rp  rx  slicer  r  rw  r  r  r  r  r  r  r  r6  r:  rC  rI  rM  CompositeImplicitAutogradAutogradrL  r_  rc  ri  rq  r-  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r%  r)  unsafe_chunkr-  r(  r'  no_statsr1  r5  rA  rG  rK  rM  _fused_dropoutrO  r\  r  ra  detachlift
lift_freshr_  rd  rg  rj  ri  r}  r  r  _adaptive_avg_pool2dr  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r"  r  _upsample_nearest_exact1dr  r$  r
  r  r  r  r0  r9  r=  rA  rO  rW  rZ  r^  rm  rv  rnn_tanhr   r}  rnn_relur  r  r  r  r  r  r  r  lstmr  r  r  r  grur  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  _unsafe_viewr'  r.  r  r  r   r#  r%  r)  r,  r2  r9  r  r@  rK  rO  rT  rV  rY  r  r  r  r  r  rf  upsample_bicubic2dr  r  reflection_pad1dreflection_pad2dreflection_pad3dr  replication_pad1dreplication_pad2dreplication_pad3dr  r  reflection_pad1d_backwardreflection_pad2d_backwardreflection_pad3d_backwardr5  r8  r;  r  rT  r7  r@  rr  rB  rF  rR  r`  +_scaled_dot_product_flash_attention_for_cpurn  rt  rz  r|  r~  r   r  r  rP   r  r  r  r  r  r  r  addbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r  
hardswish_	hardtanh_hardtanhhardsigmoid___iand____and____ilshift__
__lshift__r  r  index_reduce_index_reduce__ior____or____irshift__
__rshift____ixor____xor__leaky_relu_r   logit_logitrelu_r  renorm_renormround_r   scatter_r-  scatter_add_scatter_addscatter_reduce_scatter_reducesilu_r,   r-   r.   <module>r     sq_              



       % % % % % % % % $ $ $ $ $ $ $ $ N N N N N N N N N N N N N N N N N N N N                # # # # # #          , , , , , , , , , , 0 0 0 0 0 0 7 7 7 7 7 7                         * ) ) ) ) ) ( ( ( ( ( ( h" c   z~         %! !!9! ! ! ! !H #*'8@# # # 
 WuDL    "'uDQ    c f     *++\2F 2v 2 2 2   ,+2 -..\4v 4& 4 4 4   /.4 .//\S S6 S S5 S S S   0/S
 )**\


 
 	

 
 
 
 
   +*
2 )*++( ( ,+( )*++"V " " " ,+" ())@f @ @ @ @   *)@ 122\f F      32 .//\PP%P05P@EP P P  0/P ''GF Gv G G G   ('G /00F & V      10 /00\:F :& :U : : :  10: 011\LL%L7<LNRL L L   21L *++\) ) )f )3 ) ) )   ,+)< *++5v 5f 5 5 5  ,+5 	""&v && & & &   #"& *++\>v >V > > > >   ,+>
 *++6 6 66 6 6 6 ,+6 344%%
% % 66>	% % % 54% -..&&{'?@@ %+/5 5
55 5 	5
 5 (5 5 5 5   A@ /.5( .//''(@AA %+/X X
XX X 	X
 X (X X X X  BA 0/X 677



 
 	

 
 
 
 
 
 
   87
$ 122\<f <F <F <v < < <   32<v #         &&3<>3G1 1
1 1-011 1 1   '&1 .//\11 &1061CF1 1 1   0/1 *++0 0 0 ,+0 +,, ^)	1 1
11 1 	1 1 1   -,1 4<==%/5BEMR    >= 4?@@
R
R

R 
R 	
R
 
R 
R 
R 
R  A@
R 0899		%	/5	BE	NS	 	 	  :9	 0455
R
R

R 
R 	
R
 
R 
R 
R 
R  65
R$$
$ $ V	$
 $ $ $ $ $ $ $@ )**\Nf NF N N N N N   +*N$ .//\''
' ' V	'
 ' ' ' ' ' '  0/'T 011\
  V	
        21B 122  $^)	1 1
11 V1 	1
 1 1 1   321& :;;\
  $^) 
  V	
       <;  -.. ^)1 111 1 	1 1 1   /.1 677\
 ^)		 		
	 	 		
 	 	 	   87	 	"") ) )v )% ) ) )  #") ,--& &F &v & & &  .-& +,,	O	Oc	O 
	O 		O
 
	O 	O 	O 	O  -,	O 
)** 6? 6?
6? 
6? C=	6?
 
#6? 6? 6? 6? +*6?r (4<SM
38_   . *++ $U $U$U	$U 
$U C=	$U
 
#$U $U $U $U  ,+$UN ,--E Ed3i Ec ERU E E E  .-E
 .//OO&*3iO9<ODGOORO O O  0/O%+:?+    344\ XX!'X.1X@EX X X !   54X 788 KK!'K.1K@EK K K !   98K* * *& $$MMcM 3iM #Y	M
 IM M M M  %$M` $$]]c] c] 3i	]
 #Y] I] ] ] ]   %$]@ 455
 
v 
e 
 
 
  65
 ,--Y
Y"3iY47Y?BYJMYY Y Y  .-Y" +344>B
 

%
,4UO

 
 
  54
& %%kCDDk2336 e HTN    43 ED &% +,,VVA& AU A8D> A A A  -,A && S      '&* )**F  T     +*, '' $   	
      ('& 566  	
     76:DI    &\	  
&\	   *T&\    
tF| 
# 
 
 
 
"&\"	" " " " "J 0$/2EFGG
 !	 &\	  
&		
    HG -..56 
#Cy/2	&\   /.6 	')C)GH  "&	 
c 
 
$v,		
 d6l     )0115 5 5C 5c 5%PSBT 5 5 5 215 4<==67B BB $S	B03B
63;B B B >=B 
)**/ / /C /c /%:L / / / +*/" -55)  / /
/ &/ 
/ 63;	/ / / /@ 
##  f F # #      $#" .//
  

  	
       0/" 
##  f 6        $# 7?@@S&S&S& S& 	S&
 FS& S& S& 
S& S& dS& 8FXf-x/??@S& S& S&  A@S&n 7;<<  	
 F   
  d , , , 8FXf-x/??@   =<88F# x/?     7?@@JJJ 3iJ 	J
 J VJ 6
J dJ 8FXf-x/??@J J J A@J\ 7;<< 3i 	
  V 6
 d , , , 8FXf-x/??@   =<4OOVO 6
O 6"	O
 &!O O O 
O O 6668F#3Xf5EEFO O O Od .//UK//((V( 6
( 6"	(
 &!( ( ( 
( 666!"( ( ( 0/ 0/(4 ''(<==''(MNN 
 
V 
 6
 
 6"	 

 &! 
  
  
 
 
 666!" 
  
  
 ON >= 
F "";#HIIG G4< G G G JIG AIJJV 6
 	
   
 666!"   KJ* 5=>>((V( 6
( 	(
 ( ( ( 
( 666!"( ( ( ?>(  5>??((V( 6
( 	(
 ( 
( 666!"( ( ( @?( @HIIKKVK 6
K 	K
 K K K 
K 6666612K K K JIK0V 6
 	
  
     6 4<==11V1 6
1 	1
 1 1 
1 6666)*1 1 1 >=14 ?GHHCCVC 6
C 	C
 C C 
C 6666669:C C C IHC4 2:;;11V1 6
1 	1
 1 1 
1 6666)*1 1 1 <;14 +,,VV     -, && $(%)37& & &VZ & EK &
 U\"& & & E/0& & &  '&&X diABB   CB &&{';<<-..VVVV,, 6
 6"	
 &!  !&    -, /. =<>   0899 V 6"	
 &!  &!  
 d  68F#Xf%556   :96 7?@@``` V` 6"	`
 &!` ` &!` ` 
` d` 68F#Xf%556` ` ` A@`H 7;<<""" V" 6"	"
 &!" " &!" " 
" d" ," ," ," 68F#Xf%556" " " =<"J 788VVV$$  6"	
 &!  v    %$ 980 677VVV$$  6"	
 &!  v     %$ 872 122c'v c'E#s(O c' c' c'   32c'L (( H H HH	H H 	H H H H )(H '' I I II	I I 	I I I I  ('I$ %@ %@ %@%@	%@ %@ 	%@ %@ %@ %@ %@ %@P )122"";#HII   JI 322 ())<: <C <
 <J < < < *)< ((=* =3 =z =: = = =  )(=@@@$.@8B@PT@ @ @ @, 011Xx  (f (vv~)> ( ( (  !  21( %% $'$'+/	 	tS%	  c5 
! (	    &%  &&; ; ; '&;
  8   /344/344/344$$[%JKK$$[%9::$$[%JKK$$[%9::$$[%JKK$$[%9::	3	3$s)$	3 DK(	3 		3 	3 	3 ;: LK ;: LK ;: LK 54 54 54	3 6:;;6:;;6:;;#++K,QRR#++K,@AA#++K,QRR#++K,@AA#++K,QRR#++K,@AA	?	?$s)$	? DK(	? 		? 	? 	? BA SR BA SR BA SR <; <; <;	?   @ 08$:Q:UVWW (()NOO (()=>>Dd;;; #; ;;c; UO; 	; ; ; <; ?> PO XW; 	#+T-K-OP  '//0UVV'//0DEEDd;;; #G GGcG UOG 	G G G <; FE WV G 08$:Q:UVWW (()NOO (()=>>Dd;;; !% $	G GGcG uoG uo	G
 G G G <; ?> PO XWG 	#+T-K-OP  '//0UVV'//0DEEDd;;; !% $	S SScS uoS uo	S
 S S S <; FE WV S 08$:Q:UVWW (()NOO (()=>>Dd;;; !% $ $Q QQcQ uoQ uo	Q
 uoQ Q Q Q <; ?> PO XWQ 	#+T-K-OP  '//0UVV'//0DEEDd;;; !% $ $	 		c	 uo	 uo		
 uo	 	 	 	 <; FE WV 	 
 	 c %! 	
    4   > > >/ / /   FK+ + + +\    & & & &,/- /- /- /-d&  &  & R +,,[BCC[122. . 32 DC -,.8 +,,[BCC[122. . 32 DC -,.8 *++KABBK011. . 21 CB ,+.@ *++KABBK011. . 21 CB ,+.@  / / / /6< < < <~/ / /d 	((>??-..S S /. @? )(S> 	''=>>,--S S .- ?> ('S<; ; ;; ; ; &&{<=={+,,. . -, >= '&.6 ''=>>,--. . .- ?> ('.6 4899!))+*OPP!))+*>??  @? QP :9 3788 (()NOO (()=>>  ?> PO 98 04551566##K$IJJ##K$899%%k&KLL%%k&:;;&&{'LMM&&{';<<A A =< NM <; ML :9 KJ 76 65A /79O9STUU
 !%	K KKcK K uo	K
 K K K  VUK 	%t'?'CD  !))+*>??
 !% $U UUcU U uo	U
 uoU U U U  @? 
U 	&(A(EF  
 !% $ $
 

c
 
 uo	

 uo
 uo
 
 
 
  
V V V V/ / /7	&	7$,V$47IO77 7 7 7-'9 -f - - - - EEcE E %!	E
 E E E ER )122F v $    32 ,d.?@AA   BA +,--" " .-" /0119 9 9 219 2344C C 54C2 ABCCM M DCM22 
2 2  V2  	2 
 2  66>2  2  2  2 j -..X~&&L
LL VL 	L
 L 66>L L L '& /.L4 /00X~&&L
LL VL 	L
 L 66>L L L '& 10L/F /u / / / / /5F 5u 5 5 5 5 5
 
3E 
 
 
 
(F%7 FV F F F F F!Xf% !& ! ! ! !NN#'N05NEJ\N N N N&f & & &T & & & &"/f / / / /T / / / /"!V !49 !T ! ! ! !$V $49 $T $ $ $ $ 233S StCy S S S S   43S  c4 c4c4
c4 c4 	c4
 c4 c4 c4 c4 c4 c4L ,--   
  	
       .-    	# 	#   ! 	# =>>$).:N1 1 1  ?>1  d t    0 [BCC>??', tW tW tW tW  @? DCtWn 08$:Q:UVWW (()=>>
  $#P PPsCxP P e_	P
 e_P P P P   ?> XWPf /344$$[%JKK$$[%9::
 48	W WW%S/*W W E%,/0	W
 W W W   ;: LK 54
W. -..-..-..	v 	c3h 	F 	 	 	   /. /. /.
	 .//.//.//	 	sCx 	V 	 	 	   0/ 0/ 0/
	38_ c3_f,- 	   8 677677677\T T  87 87 87Tn %%UEu      &% $$X4 X X X X  %$X ,dko>?? $( =%)
 
 
	
 EK 
 L	

 U\"
 
 
 
  @?
 *+,,
 $( =%)  	 EK 	
 L U\"    -, 	"", , #", .//''(<== #^)) ))) ) 	)
 V) ) ) ) )  >= 0/)X ;<<$,44[5IJJX{##--- - 66>	- - - $# KJ =<-x HPQQ
 G* #'!G* G* G*G*	G* G* 	G*
 G* G* E?G* 66>G* G* G* RQG*T   ''     (' )**9 9  +*9 ''6 6 ('6 )48<899 $( 		D 	D 	D
	D EK 	D 
&			D
 	D 	D 	D :9	D -t|/?@AA. .& .x} . . . BA. 	=>>, , , ?>, 	""38 
 
 
 
  #"
 5: 
 
 
 
 
 <A + + + + +: 	""   #"
 ''G G G ('G  t{ + + +  dj ) ) )  dj ) ) )   - - -  TY ' ' '  TY ' ' '  $. 1 1 1   / / /  "D$4 5 5 5   - - -  !4? 3 3 3  $. 1 1 1  #T%6 7 7 7  t{ + + +  !4? 3 3 3   - - -  !4? 3 3 3  dj ) ) )  TY ' ' '  t{ + + +  dj ) ) )   - - -  "D$4 5 5 5  %t': ; ; ;  TY ' ' ' ' 'r-   