
    çg)$                     n   d dl Z ddlmZmZmZmZmZ ddlmZ ddl	m
Z
mZ e j        e j        e j        e j        gZd Zd Zd	 Zd
 Z e eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd           eddddddd          g e            z   g de
edd           edd i          edej        dej        dej        dej        dej        d ej        d!ej        d"ej        dej        d#ej        fd$                                    Z G d% d&e j        j                  Zej        ZdS )'    N   )Configautotunecdiv
heuristicsjit)language   )early_config_pruneestimate_matmul_timec                 @    dt          |           v rt          j        S | S )Nfp8)strtorchfloat16)as    M/var/www/html/ai-engine/env/lib/python3.11/site-packages/triton/ops/matmul.pyupcast_if_fp8r   
   s    A}H    c                     t          |           } t          |          }| |u r| S | t          v sJ |t          v sJ t          D ]}| |u r|c S ||u r| c S d S N)r   _ordered_datatypes)r   bds      r   get_higher_dtyper      s    aAaAAvv""""""""""  66HHH66HHH  r   c                       fdS )Nc                 8    |                                           S r   )zero_)nargsnames    r   <lambda>zinit_to_zero.<locals>.<lambda>!   s    t**,, r    )r    s   `r   init_to_zeror#       s    ,,,,,r   c                     g } dD ]}dD ]}}dD ]x}dD ]s}|dk    rdnd}|                      t          |||dd	||
                     dD ]:}|                      t          ||||d	||t          d                               ;ty~| S )N)r               )       )r*   @   )r*   r+         r+   r   r&   r
   BLOCK_MBLOCK_NBLOCK_KSPLIT_K
num_stages	num_warps)r   r&      r)   C)r4   r5   pre_hook)appendr   r#   )configsr4   block_mblock_kblock_nr5   split_ks          r   get_configs_io_boundr?   $   s&   G% l l
 	l 	lG# 
l 
l1 	l 	lG%,]]INN7wSZghii*4	K K KL L L $1 l l"w7W^kr#s#s.8IXdehXiXik k kl l l ll	l
l	l Nr   r,   r-   r*   r.   r%   r6   r3   r+   r&   r'   )MNK
   )r   
perf_modeltop_k)r:   keyprune_configs_byEVEN_Kc                 >    | d         | d         | d         z  z  dk    S )NrB   r1   r2   r   r"   )argss    r   r!   r!   U   s!    49Y$y/(IJaO r   	acc_dtypeinput_precisionfp8_fast_accumr/   r0   r1   GROUP_Mr2   AB_DTYPEc                 :   t          j        d          }t          j        d          }t          j        ||          }t          j        ||          }||z  }||z  }t          |||z  z
  |          }||z  ||z  z   }||z  |z  }||z  t          j        d|          z   }||z  t          j        d|          z   } t          j        t          j        ||z  |          |          }!t          j        t          j        | |z  |          |          }"||z  t          j        d|          z   }#| |!d d d f         |z  |#d d d f         |z  z   z   } ||#d d d f         |z  |"d d d f         |	z  z   z   }t          j        ||f|          }$t          dt          j        |||z                      D ]}%|r)t          j	        |           }&t          j	        |          }'nu||%||z  z  z
  }(t          j        d|j
        j                  })t          j	        | |#d d d f         |(k     |)          }&t          j	        ||#d d d f         |(k     |)          }'|*|&                    |          }&|'                    |          }'|rt          j        |&|'|$||          }$n|$t          j        |&|'||          z  }$| ||z  |z  z  } |||z  |z  z  }|$                    |j
        j                  }$||z  t          j        d|          z   }||z  t          j        d|          z   } ||d d d f         |
z  | d d d f         |z  z   z   }||k     d d d f         | |k     d d d f         z  }*|dk    rt          j        ||$|*           d S t          j        ||$|*           d S )Nr   r
   )dtype)r
   r
   )maskother)	out_dtyperL   )rR   )tl
program_idr   minarangemax_contiguousmultiple_ofzerosrangeloadrQ   
element_tytodotstore
atomic_add)+ABr7   r@   rA   rB   	stride_am	stride_ak	stride_bk	stride_bn	stride_cm	stride_cnrK   rL   rM   r/   r0   r1   rN   r2   rH   rO   pidpid_zgrid_mgrid_nwidthgroup_id
group_sizepid_mpid_nrmrnramrbnrkacckr   r   k_remaining_0rR   s+                                              r   _kernelr}   6   s   Z -

CM!EWQ  FWQ  FfEe|HVh00'::Jw#
"23E5[j)E	29Q00	0B	29Q00	0B

BN267;;W
E
EC

BN267;;W
E
EC	29Q00	0B	SD\I%47i(??@A	R4[9$s47|i'??@A
(GW%Y
7
7
7C1bga7!23344 + + 	E

A

AAa7W#455K&(:;;;B47k 9DDDA111d7k 9DDDAXAXA 	V&AsiYYYCC26!Q)_UUUUC	Ww**	Ww**
&&#
$
$C	29Q00	0B	29Q00	0B	R4[9$r$'{Y'>>?AFAAAtGQaaa00D!||
Cd######
a4((((((r   c                   D    e Zd ZeZi Zed             Zedd            ZdS )_matmulc                    | j         }|                     d          dk    r-|                     d          dk    r|                                 } |                    d          dk    r-|                    d          dk    r|                                }| j        d         |j        d         k    s
J d            | j        \  }|j        \  }t	          | j        |j                  }	||	}t          j        f||          }
t          j        t          j	        t          j        ft          j
        t          j	        t          j
        ft          j	        t          j	        ft          j        t          j        fi}|||	         d         }nVt          |t          j                  s
J d            ||| j                 v s
J d            |||j                 v s
J d            d } ||          } ||	          }	 ||          }| j        t          j        t          j        fv r!|j        t          j        t          j        fv rd }	fd	}t#          |         | ||
||                     d          |                     d          |                    d          |                    d          |
                    d          |
                    d          |||d
|	           |
S )Nr   r
   zincompatible dimensions)devicerQ   zacc_dtype must be a torch.dtypez+acc_dtype not compatible with the type of az+acc_dtype not compatible with the type of bc                 x    t          t          t          |                               d          d                   S )N.)getattrrU   r   split)tys    r   
to_tl_typez!_matmul._call.<locals>.to_tl_type   s)    2s2ww}}S11"5666r   c                 n    t          | d                   t          | d                   z  | d         fS )Nr/   r0   r2   )r   )METAr@   rA   s    r   r!   z_matmul._call.<locals>.<lambda>   s4    T!T)_55QY8P8PPRVW`Rab r   r6   )rK   rL   rM   rN   rO   )r   stride
contiguousshaper   rQ   r   emptyr   float32bfloat16int8int32
isinstancerU   
float8e4nvfloat8e5r}   )r   r   rK   rL   rM   output_dtyper   rB   _ab_dtypecsupported_acc_dtypesr   gridr@   rA   s                 @@r   _callz_matmul._call   s   88A;;??qxx{{QA88A;;??qxx{{QAwqzQWQZ''')B'''w1w1 $AGQW55  #LKAv\BBB MEM5=95>EM[`[iKjMEM,ej5;/ 

 ,X6q9IIi55XX7XXXX 4QW ====?l=== 4QW ====?l===	7 	7 	7 Jy))	:h''!z,// 7r}bk222qw2=RTR]B^7^7^Hbbbbbq!Q1HHQKK!HHQKK!HHQKK!+)	* 	* 	* 	* r   NTc                 B    t                               ||||||          S )N)rK   rL   rM   r   )r   r   )ctxr   r   rK   rL   rM   r   s          r   forwardz_matmul.forward   s+    }}QYhv*6  8 8 	8r   )NNTN)	__name__
__module____qualname__r}   kernel_locksstaticmethodr   r   r"   r   r   r   r      sX        FF7 7 \7r 8 8 8 \8 8 8r   r   )r    r   r   r   r   r   r	   rU   matmul_perf_modelr   r   r   r   r   r   r   r   r   r#   r?   	constexprr}   autogradFunctionr   applymatmulr"   r   r   <module>r      s~    6 6 6 6 6 6 6 6 6 6 6 6 6 6       G G G G G G G Gj%-O      - - -  $ 
 	332!LLYZfghhh332!LLYZfghhh32"KKXYefggg2#"KKXYefggg332!LLYZfghhh32"KKXYefggg2#"KKXYefggg32"KKXYefggg2"JJWXdefff3331MMZ[ghiii3331MMZ[ghiii32#!LLYZfghhh2##!LLYZfghhh3331MMZ[ghiii32"KKXYefggg2#"KKXYefggg32"KKXYefggg2"JJWXdefff)* 	+, 	0* 1  < OO   ;) |	;)
  \;) L;) \;) -/L;) DF<;) \;) -/L;) CE,;) []Zf;) ;) ;)  = D;)|B8 B8 B8 B8 B8en% B8 B8 B8J 
r   