çgLddlmZddlmZmZmZmZmZddlm Z ddl m Z ddl m Z edZGd d eZddZddZddZddZddZ ddd$Zdd'Zdd(Zdd)Zdd*Zdd+Zdd-Zdd.Zdd3Zdd4Zdd7Z dd8Z!dd9Z"dd:Z#dd;Z$dd<Z%dd=Z&dd>Z'dd?Z(dd@Z)ddAZ*ddBZ+ddCZ,ddDZ-ddGZ.ddHZ/ddIZ0ddJZ1ddKZ2ddLZ3ddMZ4ddPZ5ddTZ6ddVZ7ddYZ8ddZZ9dd[Z:dd^Z;dd_ZdddZ?ddgZ@ddiZA dddlZBdmZCdnZDdoZEdpZFdqZGdrZHdsZIdtZJduZKddZLddZMddZNdZOdZPddZQddZRddZSddZTddZUddZVddZWddZXddZYddZZdZ[ddZ\ddZ]dZ^ddZ_ddZ`ddZaddZbddZcddZdddZeddZfddZgdZhddZiddZjddZkdjS)) annotations)ListOptionalSequenceTupleTypeVar)ir)core)mathTceZdZfdZxZS)IncompatibleTypeErrorImplc||_||_d|jzdz|jz|_t t ||jdS)Nzinvalid operands of type  and )type_atype_b__repr__messagesuperr__init__)selfrr __class__s T/var/www/html/ai-engine/env/lib/python3.11/site-packages/triton/language/semantic.pyrz"IncompatibleTypeErrorImpl.__init__sl  2T[5I5I5K5KKgUX\XcXlXlXnXnn  '..77 EEEEE)__name__ __module__ __qualname__r __classcell__)rs@rrr sAFFFFFFFFFrraxisintbuilder ir.builderreturn tl.tensorc|dvrtd|tj||tjS)Nrr r z+program_id axis must be 0, 1, or 2 but got ) ValueErrortltensorcreate_get_program_idint32r!r#s r program_idr/sF 9MtMMNNN 9W22488"( C CCrc|dvrtd|tj||tjS)Nr(z-num_programs axis must be 0, 1, or 2 but got )r)r*r+create_get_num_programsr-r.s r num_programsr2 sF 9OOOPPP 9W44T::BH E EEra_tytl.dtypeb_tyc|j}|j}|j}|j}||kr ||kr|n|S|tjjjkr ||kr|n|S|tjjjkr ||kr|n|St d|d|)Nzunexpected signedness r) int_bitwidthint_signednessr*dtype SIGNEDNESSUNSIGNED TypeError)r3r5a_rankb_ranka_snb_sns rinteger_promote_implrA+s  F  F  D  D t||ttD0 $- - -''ttT1 $- - -''ttT1 >T>>>> ? ??r div_or_modboolc~|s|r tjS|s|r tjS|s|r|r tjStjS|s|rN|r tjS|r |r tjStjS| r| std|d||rO|j |j kr?td| zdz| zdzt||S)Nunexpected type rzCannot use /, #, or % with x because they have different signedness;this is unlikely to result in a useful answer. Cast them to the same signedness.)is_fp64r*float64is_fp32float32is_fp16float16is_bf16bfloat16is_intr<r8rrA)r3r5rBs rcomputation_type_implrP;s ||~~z ||~~z ||~~  : :  ||~~  :  <<>> dllnn ; z ;;==> ><4<>> J >>>- 1 1 3 3-J(>(@(@,eu*+*+Uy..u|U\JJEJWWW  $ $ & &Uy,,U\5<HH%*UUU    ! !Uy++EL%,GGTTT 888 9 99rc Nt|||dd\}}|jj}|rFt j||jt||j|jS| r8t j| |j|j|jS| r8t j| |j|j|jStd|)NTFrE)rfr]r^rUr*r+rjrkminusrV create_fsubrO create_subr<rgrhr# scalar_tys rsubrxs/ugtUSSLE5 !Ihy..u|U5'=R=R=YZZ\a\fgggUy,,U\5<HH%*UUU     Uy++EL%,GGTTT 2y22 3 33rct|||\}}|jj}|r8t j||j|j|jS|r8t j| |j|j|jStd|NrE) rfr]r^rVr*r+ create_fmulrkrO create_mulr<rvs rmulr}s/ugFFLE5 !IUy,,U\5<HH%*UUU     Uy++EL%,GGTTT 2y22 3 33rc Vt|||dddd\}}|jj}|jj}|r'|rt |||}n|r&|rt |||}n|rK|r7t |t j|}t |t j|}nn|rH|r4|j|jkrt |||}n$t |||}ntd|t j | |j |j |jSNFTrE) rfr]r^rVrOr_r*rJfp_mantissa_widthr<r+ create_fdivrkrns rtruedivrs/ugueUY[_``LE5j'Oj'O""$$>)?)?)A)A>UOW55    ! !>o&A&A&C&C>UOW55    ! ! >o&<&<&>&> >UBJ00UBJ00  $ $ & &>?+F+F+H+H>  ,/P P P99EE99EE,>,@,@@6IOLdLdLfLffjoopp p  " " $ $ Z9W00u|LLejYY Y9W00u|LLejYY Y 2y22 3 33rxy propagate_nantl.PropagateNanc t|||\}}|j}|r|tjjkr8tj||j|j|j S|tjj kr8tj| |j|j|j Std|| r8tj||j|j|j S|r8tj||j|j|j St#d|NzUnexpected propagate_nan Unexpected dtype )rfr9rVr* PropagateNanALLr+create_minimumfrkr]NONEcreate_minnumfr)r create_minsiis_int_unsigned create_minuir<rrrr#r9s rminimumrH '1g 6 6DAq GE  5 BO/ / /9W44QXqxHH!&QQ Q bo2 2 29W33AHahGGPP PHHHII I     5y--ahAA16JJJ    5y--ahAA16JJJ3E33444rc t|||\}}|j}|r|tjjkr8tj||j|j|j S|tjj kr8tj| |j|j|j Std|| r8tj||j|j|j S|r8tj||j|j|j St#d|r)rfr9rVr*rrr+create_maximumfrkr]rcreate_maxnumfr)r create_maxsir create_maxuir<rs rmaximumrrrminmaxcTt|||\}}t|||\}}t|||\}}|j}|r?tj||j|j|j||jStd|d)Nrz(. Only floating point clamp is supported) rfr9rVr*r+ create_clampfrkr]r<)rrrrr#r9s rclampr%s+Cg>>HC )!S' : :FAs )!S' : :FAs GE ]y..qxSZQ^__abaghhh[E[[[\\\rcTt|||ddd\}}|jj}|jj}|r|st ||t ||}||krt |||}||krt |||}||fS)NF)rfr]r^rOrrAr_)rgrhr# input_sca_ty other_sca_tyres rbitwise_op_type_checking_implr6s/ugueUZ[[LE5:$L:$L    D (;(;(=(=D' lCCC%lLAAJ\!!UJ00\!!UJ00 %<rct|||\}}tj||j|j|jSrT)rr*r+ create_andrkr]rgrhr#s rand_rE?0wGGLE5 9W'' elCCUZ P PPrct|||\}}tj||j|j|jSrT)rr*r+ create_orrkr]rs ror_rJs?0wGGLE5 9W&&u|U\BBEJ O OOrct|||\}}tj||j|j|jSrT)rr*r+ create_xorrkr]rs rxor_rOrrc|js#t|tjd|}|js#t|tjd|}t |||SNint1)r]is_int1bitcastr*r9rrs r logical_andrTsv :    :rx//99 :    :rx//99 ug & &&rc|js#t|tjd|}|js#t|tjd|}t |||Sr)r]rrr*r9rrs r logical_orr\sv :    :rx//99 :    :rx//99 ueW % %%rc|js#t|tjd|}t ||Sr)r]rrr*r9invert)rgr#s rnot_rdsC :    :rx//99 % ! !!rct|||\}}tj||j|j|jSrT)rr*r+ create_lshrrkr]rs rlshrrj?0wGGLE5 9W((u|DDej Q QQrct|||\}}tj||j|j|jSrT)rr*r+ create_ashrrkr]rs rashrrorrct|||\}}tj||j|j|jSrT)rr*r+ create_shlrkr]rs rshlrtrrc|SrT)rgs rplusr~s Lrc(|jj}|r'td|zdzt j||||}t|||S)Nz$wrong type argument to unary minus ()) r]r^rUr)rr*r+get_null_valueto_irrx)rgr#r_0s rrsrss:$La?,BWBWBYBYY\__``` 7)),*<*??? GE ]28U + +F 9W..uc::F C CCrr List[int]r9ct|tjr,|jjdks Jdt |||}nx|t d|dkr)|||}n#t|d|j }||}tj||}t|||S)Nr zonly accepts size-1 tensorz2dtype must be specified when value is not a tensorrget_) r r*r+numelvaluer_r)rrgetattrnamesplat)rrr9r# get_value_fns rfullrs%## ({ A%%%'C%%%UE7++ =QRR R A::**5;;w+?+?@@EE"7,?5:,?,?@@L L''E %'' w ' ''rrc|jr Jdt|dkr|Stj|j|}tj||j||S)NzCannot splat a block tensorr) r]rlenr*rr9r+ create_splatrk)rrr#rs rrrspz""$$CC&CCCC 5zzQ ]5; . .F 9W))%,>> G GGr dst_shape can_reordercd}|D]}||z}|jj|krtdtj|jj|}tj||j|||S)Nr z:reshape() cannot change total number of elements in tensor) r]rr)r*rr^r+create_reshaperk)rgrrr#rsrs rreshaper!%s} E    z5  UVVV ]5:,i 8 8F 9W++EL)[QQSY Z ZZrc@d|jD}||d|jst |||St j|jj|}t j| |j ||S)Nc6g|]}tj|Srr*_constexpr_to_value).0rs r zexpand_dims..0s#@@@q'**@@@rr )rr#) rinsertr]rrr*rr^r+create_expand_dimsrk)rgr!r#rrs r expand_dimsr*/s@@EK@@@I T1 :   >U)W==== ]5:,i 8 8F 9W// dCCV L LLrc&|s Jdt|jdksJtj|jj|jd|jdzg}tj||j|j|S)Nz;current implementation of `cat` always may reorder elementsr r) rrr*rr]r^r+ create_catrk)rXrYrr#ret_types rcatr.:s UUUUUU sy>>Q    }SX_sy|cil/J.KLLH 9W'' CJ?? J JJrabct|||\}}|jgk}|r"t|d|}t|d|}t|jdtjrt jd}nd}|j|gz}t j|jj|}t j | |j |j |}|rt|dgd|}|S)Nrr Frr#) r\rr*r r* constexprrr]r^r+ create_joinrkr!)r/r0r# was_rank_1two new_shaper-rs rjoinr9As 1g . .DAqBJ' 1g & & 1g & &!'"+r|,,l1oo3%I}QV]I66H )G''!(;;X F FCDcA3E7CCC Jrcjt|jdksJtj|jddksJ|jdd}tj|jj|}||j\}}tj ||tj ||fS)Nrr2r ) rrr*r%rr]r^ create_splitrkr+)r/r#r8r-outLHSoutRHSs rsplitr>Zs LL1      "172; / /1 4 4 4 4 I}QV]I66H))!(33NFF &(## &(## rdims Tuple[int]ctjt|krtdtd|Dt t t|krtd|t jjj fd|D}t j | j ||S)Nz5permute dims must have the same length as input shapec3>K|]}tj|VdSrTr$)r&ds r zpermute..js- 6 6Ab$Q'' 6 6 6 6 6 6rz?permute dims must be a permutation of 0, 1, ..., n-1, but were c*g|]}j|Sr)r)r&rCrgs rr'zpermute..ms0N0N0NAQ0N0N0Nr) rrr)sortedlistr r*rr]r^r+ create_transrk)rgr?r#r-s` rpermuterIgs 5;3t99$$PQQQ 6 6 6 6 666$uSYY?O?O:P:PPPa[_aabbb}UZ.0N0N0N0N0N0N0NOOH 9W))%,==x H HHrc |jsHtj|j|}tj||j||S|j}t|t|krtd|d|||kr|St|D];\}}|||kr*|dkr$td||d|d|d|d| >C NN * *3y>>3y>>:: 6 6i : :3:q I I " fmaS9_ M MOO"3355  6 ^^c)nn , ,3y>>3y>>:: 6 6i : :3:q I I " fmaS9_ M MOO"3355 9~~Y////  ++ a aGAtaLEqyy  ''''1**%4--  &&&& "-/21vv"68<"=?B4yy"IKR"SUXY^U_U_"`aaa  ! !]6=)<?? ? 9W++EL&,,w:O:OPPRX Y YYrNfp_downcast_roundingc|j}t|tjr|j}t|tjr|j}|r1tj|j|j}||kr|S|j}|j}t|}d}| rP| r<|j |j kr,|tj j}nL|tj jkrd}n4|2tdt!|zdzt!|z|s|r|jjs Jd|s|r?|jd Jd|jd||||S|r| s*| r|s|rBtj||j||||S|r|r(|r>|s*t?t?|tj |||S| o#| o|j |j k}|rAtj|!|j|||S| o#| o|j |j k} | rAtj|"|j|||S|#r |#r|j$|j$ks|j%|j%kr|&o|' } |'rX|j(|} tj|)| |j(} tU|| |Stj|+|j||| |S|,r|#r|'rX|j(|} tj|)| |j(} tU|| |S|&rAtj|-|j|||Stj|.|j|||S|#r|,r|'s|&sAtj|/|j|||Stj|0|j|||S|1r|#r|j$} | d krAtj|2|j|||S| d krZtUt?|tj3|tj|4d tj3|S|#rU|1rAtj|5|j|||S|1rU|1rAtj|6|j|||SJd |d |)NFTz]fp_downcast_rounding should be set only for truncating fp conversions. Source scalar type is z and destination type is z4fp8e4nv data type is not supported on CUDA arch < 89convert_custom_typesz0target doesn't provide conversion for this type.r@r rz cannot cast z to )7r]r r*r4rrrr^rMrgrVrjr rdrer)rV is_fp8e4nvoptions allow_fp8e4nv is_fp8e4b15 codegen_fnsgetis_fp8r+create_fp_to_fprkrrKrIrMr_rJcreate_fp_trunc create_fp_extrOr7r8ris_boolr9rrcreate_int_castis_standard_floatingcreate_fp_to_sicreate_fp_to_uicreate_ui_to_fpcreate_si_to_fprUcreate_ptr_to_intint64 get_int64create_int_to_ptrrk)rgrhr#rqrlrmrnuse_custom_rounding truncate_fpext_fp sign_extendtyrbitwidths rr_r_ss ZF&",''& 55:39 Mv}ej.I.I.K.KLL  JJ11EFFyJ$:$:%%y  '**G G G  '@P@U)=)= !R%5%: : :RVW[e[t>t>t ..00M9K9K9M9M5M      p""7++B711"55u{CCBUB00 09W44U\6<->-@-@c      c""7++B711"55u{CCBUB00 0  % % ' ' c9W44U\6<>@@c      cz'?'?'A'A c9W44U\6<>9W66u|V\\RYEZEZ[[]cdd d q==T%7;;RYwGXGXYZG[G[]_]e=f=fhopp paz0022ay225<gAVAVWWY_```^z0022^y// fll7>S>STTV\]]]4444F44444rctjj}|rC|dkrtjj}n+|dkrtjj}nt d|d|S)Nz.ca.cgCache modifier  not supported)r CACHE_MODIFIERrCACGr)cache_modifiercaches r_str_to_load_cache_modifierrQse   "EO U " "%(EE u $ $%(EEM~MMMNN N Lrctjj}|rs|dkrtjj}n[|dkrtjj}nC|dkrtjj}n+|dkrtjj}ntd|d|S)Nz.wbrz.csz.wtrr)r rrWBrCSWTr)rs r_str_to_store_cache_modifierr]s   "E O U " "%(EE u $ $%(EE u $ $%(EE u $ $%(EEM~MMMNN N Lrctjj}|rC|dkrtjj}n+|dkrtjj}nt d|d|S)N evict_last evict_firstzEviction policy r)r EVICTION_POLICYNORMAL EVICT_LAST EVICT_FIRSTr))eviction_policyevictions r_str_to_eviction_policyrmse!(HQ l * *)4HH  - -)5HHOOOOPP P Orcd}|rC|dkrtjj}n+|dkrtjj}nt d|d|S)NzeronanzPadding option r)r PADDING_OPTIONPAD_ZEROPAD_NANr))padding_optionpaddings r_str_to_padding_optionrys_GO V # #'0GG u $ $'/GGM~MMMNN N Nrctjj}|rs|dkrtjj}n[|dkrtjj}nC|dkrtjj}n+|dkrtjj}nt d|d|S)Nacquirereleaseacq_relrelaxedMemory semantic r)r MEM_SEMANTICACQUIRE_RELEASEACQUIRERELEASERELAXEDr)) sem_optionsems r _str_to_semrs / )C L  " "/)CC 9 $ $/)CC 9 $ $/1CC 9 $ $/)CCJ JJJKK K Jrctjj}|r[|dkrtjj}nC|dkrtjj}n+|dkrtjj}nt d|d|S)Ngpuctasysrr)r MEM_SYNC_SCOPEGPUCTASYSTEMr)) scope_optionscopes r _str_to_scopers}   !EN 5 %)EE U " "%)EE U " "%,EEL LLLMM M Lrcn|rt|ds|g}d|D}|D]5}t|trd|cxkrt|ksnJ6t|dksJt|tt |ks Jdt |SdS)N__iter__cTg|]%}t|tjr|jn|&Srr r*r4rr&elems rr'z0_canonicalize_boundary_check..s0lllUY 4(F(FP$**Dlllrrz'Duplicate dimension in `boundary_check`r)hasattrr r"rsetrF)boundary_check block_shapedims r_canonicalize_boundary_checkrs&~z22 .,-Nll]klll! H HCc3'' GA,G,G,G,Gs;7G7G,G,G,G,G,G,G,G>""Q&&&&>""c#n*=*=&>&>>>>@i>>>n%%% 2rc ||td|jjj} | tjks Jd| r$|t jjkrtd|jj} t|| }tj | |j |||||| S)NK`mask` and `other` arguments cannot be specified for loading block pointers3`tl.int1` should be rewrited in `tl.make_block_ptr`z@Padding option `nan` is not supported for integer block pointers)r)r] element_tyr*rrOr rrrrMr+create_tensor_pointer_loadrk) ptrmaskrhrrrr is_volatiler#elt_tyrhs r_load_block_pointerrs 5,fggg X +F RW   S    }}]7b&7&???[\\\X F2.&BYBYB[B[\\N 9**3:~wPUW_almmou w wwrc |jjs*td|jd||td|s|rtd|jsT|r(|jrtd|r(|jrtd|jrT|(t ||j|}|(t ||j|}|jj} | j} | tj kr7tj } tj | | j } t|| |}|t|| |}|jr/|j} tj| | } n| } |0tj||j|||| Stj||j|j|r|jnd|||| S)NUnsupported ptr type z in `tl.load`z)`other` cannot be provided without `mask`z`padding_option` or `boundary_check` argument is not supported for loading a tensor ofpointers or loading a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadEMask argument cannot be block type if pointer argument is not a blockzFOther argument cannot be block type if pointer argument is not a block)r]r^rUr)rrrSrMrr*rint8 pointer_type address_spacer_rr+ create_loadrkcreate_masked_load) rrrhrrrrrr#ptr_tyrrrhs r _load_legacyrs 8? ! ! # #US1B1B1D1DSSSTTT |)DEEEU.UTUU U 8    g  fDI&&(( fdee e  gUZ((** geff f xV  'ch.G.G.I.I7SSD  (0I0I0K0KWUUEX_F  F)=>>3(( UFG,, x))++vu-- |y,,SZ+VVX^___y  & &sz4;PU@_ [_afhp'2 4 45;== =rrrOptional[tl.tensor]rrrrVrrrc &t|} t|} t|} |jr5|jjrt||||| | | || St||||| | | || SrT) rrrr]rUrrrr) rrrhrrrrrr#rrrs rloadrs ( 7 7E&77H$^44G xnSX099;;n"3e^WeU]_jlstttCunguhXcelmmmrdesc_ptrc t||d}||j|||t |t |}t j||SNF require_i64)_convert_to_ir_valuescreate_descriptor_loadrkrrrr*r+)roffsetsrrr]r#rs rdescriptor_loadrsk#GW%HHHG&&xGATAT'B>'R'R'>'O'O Q QA 9Q  rct||d}tj||j|j|tjSr)rr*r+create_descriptor_storerkvoid)rrrr#s rdescriptor_storersC#GW%HHHG 9W44X_elT[\\^`^e f ffrc .|td|jj}|jst |||}|js Jd||jks(Jd|d|jd|jjj|jjks*Jd|jjjd|jjd|jjj}|t jks Jdt||}t|||}t j | |j |j |||t j S) Nrz-Value argument must be block type or a scalarz Block shape(z) and value shape(z ) mismatchzBlock element type(z) and value element type(r)r)r]rrMrrSr*rrr_r+create_tensor_pointer_storerkr) rvalrrrrr#rrs r_store_block_pointerrs fggg(%6688K 8    >"3 W== 8    OO OOOO #(33   \k\\SX5N5N5P5P\\\    8  )SX-@ @ @ @CqX[X`XkXvCqCqRURZReCqCqCq @ @ @ X +F RW   S   2.+NNN sFG $ $C 9W88SZQ_afhpqqW  rc |jjs*td|jd|rtd|jsR|jrtd|r(|jrtd|jrRt ||j|}|(t ||j|}|jj}|j}|tj kr7tj }tj ||j }t|||}t|||}|s?tj||j|j||tjS|jjstdtj||j|j|j||tjS)Nrz in `tl.store`z`boundary_check` argument is not supported for storing a tensor of pointers or storing a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadzFValue argument cannot be block type if pointer argument is not a blockrz"Mask must have boolean scalar type)r]r^rUr)rrrSrMrr*rrrrr_r+ create_storerkrrcreate_masked_store) rrrrrrr#rrs r _store_legacyr;s' 8? ! ! # #VT1B1B1D1DTTTUUUBABB B 8    f 8     geff f  fDI&&(( fdee e xT"3(A(A(C(CWMM  'ch.G.G.I.I7SSD X_F  F)=>>3(( sFG $ $C ay--cj#*eXVVXZX_``` 9  # # % %?=>>> 9W00SZV[]effhjho p pprrc t|}t|}|js|jjrt d|jr3|jjrt|||||||St|||||||S)N"Cannot store to a constant pointer) rrr]is_constr^r)rUrrrr) rrrrrrr#rrs rstorergs ) 8 8E&77H x?cho6688?=>>> xWSX099;;W#CdNE8U\]]]S#t^UHgVVVrcmprrc t|}t|}|jjj}|jdvrt dtj| |j |j |j |||jS)N)rrtz9atomic_cas only supports elements with width {16, 32, 64}) rrr]r^rrjr)r*r+create_atomic_casrk)rrrrrr#rs r atomic_casr }sy c  C % E+J$L88TUUU 9W..sz3:szSVX]^^`c`h i iirop&Tuple[tl.tensor, tl.tensor, tl.tensor]c|jjs)td|jz|js|jjrtd|jjj}|tjur|dkrtd|zdz|tj tj tj tj fvr%td|zdzt|z|jrT|(t||j|}|(t||j|}t#||jjj|}|s|d}tj }|jrc|||j}tjtj |j}tj||}|||fS)Nz)Pointer argument of store instruction is rrqatomic_z does not support fp16z does not support T)r]r^rUr)rrrr*rLrrint16rNrVrrSrMr_get_int1rrr+)rrrr r#rmask_irmask_tys ratom_red_typechecking_implrs 8? ! ! # #\DsxGXGXGZGZZ[[[ x?ch1::<<?=>>>+JRZB%KKR*BBCCCbgrw"+>>>R*>>ZPQQQ xR  'ch.G.G.I.I7SSD ?&sCH,E,E,G,GQQC sCHO. 8 8C +""4((' 8     J**7CH4M4M4O4OPPGmBGSX-F-F-H-HIIGy'** T>rc  t|||d|\}}}t|}t|}|jj}|r|rPtj| tj j |j |j |j |||jStj| tj j|j |j |j |||jS|tjtjhvrt#d|t%gd||}|tjkr tjn tj}t+|||} t+|tj|d|} |tjkr tjn tj} t+|| |} t+|tj| d|} t3|||}t5|||}tj| tj j | j | j t7|||j ||| j}tj| tj j| j | j t7|||j ||| j}t;||||}t+|||S)Nrz#atomic_max not supported for dtype r )rrrr]r^rOrr*r+create_atomic_rmwr ATOMIC_OPMAXrkUMAXrJrHr<rr-rrruint32uint64rrrUMINwhererrrrrr#sca_tyri_typei_vali_ptrui_typeui_valui_ptrposnegpos_retneg_retrs r atomic_maxr,/S$wOONCd c  C % E X_F }}y    ! ! y9))",*:CJ TXT_adfkllnqnvxx x9))",*;SZUYU`beglmmorowyy y bj"*---FfFFGGG C ) )D2:--RXX28F C ) )E C33W = =E!RZ//biiRYG S'7 + +F S"/'155w ? ?F T7 + +C Cw ' 'Ci!!","2EL%,"&tS'":":"A3 O OPUPZ\\Gi!!","3V]FM"&tS'":":"A3 O OPVP[]]G Wgw / /C 3 ( ((rc  t|||d|\}}}t|}t|}|jj}|r|rPtj| tj j |j |j |j |||jStj| tj j|j |j |j |||jS|tjtjhvrt#d|t%gd||}|tjkr tjn tj}t+|||} t+|tj|d|} |tjkr tjn tj} t+|| |} t+|tj| d|} t3|||}t5|||}tj| tj j | j | j t7|||j ||| j}tj| tj j| j | j t7|||j ||| j}t;||||}t+|||S)Nrz#atomic_min not supported for dtype rr )rrrr]r^rOrr*r+rr rMINrkrrJrHr<rr-rrrrrrrrrrr s r atomic_minr0r-rc rt|||d|\}}}t|}t|}|jj}|rt jjnt jj }tj | ||j |j |j |||jS)Nrq)rrrr]r^rVr rFADDADDr*r+rrk)rrrrrr#r!r s r atomic_addr4s/S$wOONCd c  C % E X_F$0022 H   8HB 9W..r3:sz4;X[]bccehem n nnrc  t|||d|\}}}t|}t|}tj|t jj|j |j |j |||j S)Nand) rrrr*r+rr rANDrkr]rrrrrr#s r atomic_andr9y/S$wOONCd c  C % E 9W..r|/?SZY]YdfikpqqX  rc  t|||d|\}}}t|}t|}tj|t jj|j |j |j |||j S)Nor) rrrr*r+rr rORrkr]r8s r atomic_orr>sx/S$gNNNCd c  C % E 9W..r| CJX\XcehjoppX  rc  t|||d|\}}}t|}t|}tj|t jj|j |j |j |||j S)Nxor) rrrr*r+rr rXORrkr]r8s r atomic_xorrBr:rc  t|||d|\}}}t|}t|}tj|t jj|j |j |j |||j S)Nxchg) rrrr*r+rr rXCHGrkr]r8s r atomic_xchgrFsy/S$PPNCd c  C % E 9W..r|/@#*cjZ^ZegjlqrrX  rc||jjvsJd|jjd||}|dkrd}t t j|S)Nzinput_precision must be one of z. Got TF32X3TF32x3)lowerrvallowed_dot_input_precisionsupperrr INPUT_PRECISION)input_precisionr#s r_str_to_dot_input_precisionrOsz  " "go&R R R Ro'/*Voo^moo S R R%++--O(""" 2% 7 77raccrNmax_num_imprecise_acc out_dtypec d}|jr|jsJ||j|j|j|js|jr6t |t j|}t |t j|}| |jj}t||}t|j }t|j } || cxkrdks,n|| cxkrdksnJd|j d|j d|j dj |j dj ksAJd |j d |j d |j dj d |j dj d |j dj d kr,|j dj d kr|j dj d ksJd|j d|j d|jj rf|jj t jks Jd|j dj dks Jd|d} t j} n|rt'd|jj s|jj r"|d} t j} n@|r|dn|d} |} |jj d} |jj d} |dkr|jj dnd}t j| |r|| | gn| | g}| || |r|| | gn| | g}n|j}|j|ksJ|A|jr&|jr |jj}nd}t j||j|j||||S)Nc@|jsu|s|r Jd|r|rdS||ksJd|d|ddS|s|rQ||ksJd|d|d|s"|sJd|ddSdS|s|r/|jrgd }nd d g}d }|||d |||ddS|sI|s5| s!| s Jd||sI|s5| s!| s Jd|||ksJd|d|ddS)Nz1Dot op does not support fp8e4nv on CUDA arch < 90z First input (z) and second input (z) must have the same dtype!z0Both operands must be same type. First operand (z) and second operand (rz:Both operands must be either int8 or uint8. Operand type ()fp8e4nvfp8e5fp8e4b15rUrVc tfd|Ds.d|}td|d|dddS)Nc3NK|]}td|V dS)is_N)r)r& dtype_namer9s rrDzLdot..assert_dtypes_valid.._validate_dtype..=s?dd Awu.@J.@.@AACCddddddrrKzOnly supports z. z (r)anyr9AssertionError)r9 allowed_types operand_namesupported_typess` r_validate_dtypez9dot..assert_dtypes_valid.._validate_dtype<ssddddVcdddddk*.))M*B*B,-io-i-iQ]-i-iaf-i-i-ijjjkkrz First operandzSecond operandzUnsupported dtype ) rwrur{rOis_int8is_uint8allow_fp8e4b15rKrMrIr) lhs_dtype rhs_dtypervr^ras rassert_dtypes_validz dot..assert_dtypes_valid*s\$ E ++-- Ci6J6J77 C CB C C C!! i&6&6&8&8  ))),A9,A,AZc,A,A,A)))))!! EY%5%5%7%7 E I---0Pbk0P0PDM0P0P0P--- ((**]i.@.@//]]\PY\\\]]]]]]]!!## Ey'7'7'9'9 E)9$D$D$DMM%.$8Mkkk   =/JJJ =:JKKKKK ((**4i.?.?.A.A4YEVEVEXEX4\e\m\m]]443 33444 ((**4i.?.?.A.A4YEVEVEXEX4\e\m\m]]443 33444 I---0Ey0E0E^g0E0E0E-----rr z+Both inputs must be either 2D or 3D; (lhs: z vs rhs: rr2zFirst input shape (z) and second input shape z= are not compatible for matmul (second index of first shape (z0) must be equal to first index of second shape (r z0All non-batch values in both first input shape (z) and second input shape (z) must be >= 16!zonly int8 supported!r rzsmall blocks not supported!rzhout_dtype=bfloat16 is unsupported. Please use out_dtype=float32/float16 and cast with `.to(tl.bfloat16)`) r]rr9rvrxr_r*rLdefault_dot_input_precisionrOrrrr^rOr get_int32r-rMr)rIget_fp32rJrKget_fp16rrrkr{max_num_imprecise_acc_defaultr+ create_dot)rXrYrPrNrQrRr#rglhs_rankrhs_rankr ret_scalar_tyMNBr acc_handles rdotrw'sEEE@ 8    638#4#4#6#6666 39go>>> y-#)"7"7"9"9-3 G,,3 G,,!/E1/7KKO39~~H39~~H x $ $ $ $1 $ $ $ $H(A(A(A(A(A(A(A(A(ADRqtqzDRDRFIFODRDRDR(A(A(A 9R= #) #    qqqSYqqVYV_`bVcViqq[^[deg[h[nqqq    9R= " $ $2)<)B)B IbM 2 % % %{39{{`c`i{{{ & % % x"x"')))+A)))y|!R''')F'''   q ! !     " vxx x  " ""cho&=&=&?&?"   a  $-$5$5$7$7 PW  a W=M=Ma=P=P!  rA rA%]]qA ]=q*D1a))q!f E EF {))"1.Hq!Qii1a&II Z x6!!!!$ 9     &#)"2"2"4"4 &$+O$Q ! !$% ! 9W'' CJ O]rss  r conditionct|tj|}|jr " " $ $C+Iq'BB 1 VF 9W**9+;QXqxPPRX Y YYrc`|rtj||}n|}tj||SrT)r*rr+)rrwr\res_tys r wrap_tensorr}s6y)44 9Q  rinputsSequence[tl.tensor]Tuple[tl.tensor, ...]c"tfdDddjjt}|ksJd|dfdt Dt fdDs JddD|tfd ttDS) Nc3TK|]"}t||jjgdV#dS)Tr3N)r!rr)r&tr#s rrDzreduction..s;ffZ[wq17=/tWUUUffffffrrz&reduction axis must be < inputs rank (rc&g|] \}}|k |Srr)r&rQr r!s rr'zreduction..s"===tq!199999rc38K|]}|jjkVdSrT)r]r)r&rrs rrDzreduction..s,55qv|u$555555rz-all reduction inputs must have the same shapecg|] }|j Srrkr&rs rr'zreduction..s&@&@&@Aqx&@&@&@rc3K|]8}t||jjV9dSrTr} get_resultr]r^)r&rQr~ reduce_opr\s rrDzreduction..sGtt\]Y11!44fQin6KYWWttttttr) tupler]rrrNall create_reduceverifyr )r~r!region_builder_fnr#rankrr\rs`` ` @@@r reductionrsC |ffff_efffff 1IN E u::D $;;;HHHH;;;====y//===I 5555f555 5 5ff7ffff%%&@&@&@&@&@$GGIi     ttttttafgjkqgrgrasasttt t ttrreversecdjjt}| |cxkr|ksnJd|d|d|dkr||z }D]}|jjks Jd|dD|||t fdt tDS)Nrz scan axis z must be < inputs rank (rz(all scan inputs must have the same shapecg|] }|j Srrrs rr'z$associative_scan..s"<"<"<18"<"<".sGnnVWW//22F1IN4I5QQnnnnnnr)r]rr create_scanrrr ) r~r!rrr#rrrrs ` @@rassociative_scanrs( 1IN E u::D 5D    4     !Sd!S!SD!S!S!S    axx   QQv|u$$$&P$$$$!!"<"R>R>T>T)U)UVVV Hrct|jt|krtd|jdt j||j|S)NzDShape of input to max_contiguous does not match the length of valuesz tt.contiguityrrr)rkrr rrrs rmax_contiguousrsa 17||s6{{""_```Hor|FAH.s+++s +++r)endswithr startswithr*r+ create_printr)rrrr#new_argss r device_printrs ??3  D#  ??4 $T$t# 6{{Qv0055v++d+++H 9W))&#x@@"' J JJrcondmsg file_namelinenoc >|j}|sHtj|jd}tj||jd|}tj||j||||tj S)N)r ) r]rr*rr^r+rrk create_assertr)rrr func_namerr#cond_tys r device_assertrsiG     L-66y--dk5AA7KK 9W**4;Y SYZZ\^\c d ddrczt|trtj|}t|tjr|rAd|jcxkrdksnJd|jd||jSd|jcxkrdksnJd|jd||jSt|tjr|jjdks Jd |j s Jd |j tj krG|rE| |j ||j S|j tjkr |s Jd |j SJd t#|)Nllz@Block pointers only support 64 bit `shape/strides`, got a value z which is out of the rangezFBlock pointers only support 32 bit `offsets/block_shape`, got a value r z*Expected a scalar in shape/strides/offsetsz8Expected an integer scalar type in shape/strides/offsetsFzzBlock pointers only support 32 bit `offsets/block_shape`, add a `.to(tl.int32)` or use regular indexing for 64 bit supportz3Unsupported element type in shape/strides/offsets: )r r"r*r4rrrkr+rr9rOrrrk get_int64_tyrr-r])r#rrs r_convert_elem_to_ir_valuer s$"|D!!$ %%  1TZ////%/////2F#z2F2F2F///$$TZ00 0TZ////%/////2F#z2F2F2F///$$TZ00 0 D") $ $z1$$$&R$$$z  ""^^$^^^^ : ! !k !**4;8L8L8N8NPTPZPhPhPjPjkk k Z28 # #K # SS S S S{TTT TTTTTrcht|drfd|DSt|gS)Nrc2g|]}t|Sr)r)r&rr#rs rr'z)_convert_to_ir_values..%s&\\\$)'4EE\\\r)rr)r# list_likers` `rrr#sJy*%%]\\\\\R[\\\\ %gy+ F F GGrbasec t||}t||}t||d}|jr|jjrt d|jjt jkr8t|t j t j |jj |}tdsgdDtdDs Jdt|ds|g}d|D}t|tt!t#|ks Jd tfd ||||fDs Jd ||j||||}t j|t j t j|jjS) NFrzMExpected `base` to be a pointer type (but not a block pointer type or others)rcTg|]%}t|tjr|jn|&Srrrs rr'z"make_block_ptr..;s/bbbdD",!?!?I4::Tbbbrc3`K|])}t|tod|cxkodkncV*dS)rrN)r r"rs rrDz!make_block_ptr..<sMXXDz$$$?4)?)?)?)?%)?)?)?)?XXXXXXrzGExpected a list of constant integers (`int32_t` range) in `block_shape`cTg|]%}t|tjr|jn|&Srrrs rr'z"make_block_ptr..Bs/ V V V:dBL99 CTZZt V V Vrz.Fs6ddis;3y>>1ddddddrzBExpected shape/strides/offsets/block_shape to have the same length)rr]rUrrr)r*rr_rrrrrrFrGr rcreate_make_block_ptrrkr+r)rrstridesrrorderr#rks ` rmake_block_ptrr)s "'5 1 1E#GW55G#GW%HHHG 9    j!5!>!>!@!@jhiii yrw&&D"/"'493JKKWUU ; + +$"m bbVabbbK XXKXXX X XRRQRRR 5* % % V VPU V V VE %==Ds5zz!2!233 3 3 35s 3 3 3 ddddE7T[]bCcddd d dMMLMMM  * *4;wQ\^c d dF 9VR_R]49;OQ\-]-]^^ _ __rct||d}tj||j||jSr)rr*r+create_advancerkr])rrr#s radvancerPs>#GW%HHHG 9W++DKAA49 M MMr)r!r"r#r$r%r&)r3r4r5r4r%r4)r3r4r5r4rBrCr%r4)rr4rr4rQrCr%rR)FFTF)rXr&rYr&r#r$r%rZ)rgr&rhr&r#r$r%r&) rgr&rhr&rrCr#r$r%r&)rr&rr&rrr#r$) rr&rr&rr&rrr#r$)rgr&rhr&r#r$r%rZ)rgr&r#r$)rgr&r%r&)rgr&r#r$r%r&)rgr&r#r&r%r&)rr&r%r)rr"rr"r#r$r%r&)rrr9r4r#r$r%r&)rr&rrr#r$r%r&) rgr&rrrrCr#r$r%r&)rgr&r!r"r#r$r%r&) rXr&rYr&rrCr#r$r%r&)r/r&r0r&r#r$r%r&)r/r&r#r$r%rZ)rgr&r?r@r#r$r%r&)rgr&rrr#r$r%r&)rXr&rYr&r#r$r%r&)r_r`)rgr&rhr4r#r$r%r&rT) rgr&rhr4r#r$rqr`r%r&)rr&rrrhrrrrrVrrVrrVrrCr#r$r%r&) rr&rrVrrVr#r$r%r&)rr&rr&r#r$r%r&)rr&rr&rrrrVrrVr#r$r%r&)rr&rr&rr&rrVrrVr#r$r%r&) rr&rr&rr&r rVr#r$r%r)rr&rr&rr&rrVrrVr#r$r%r&)rXr&rYr&rPr&rNr`rQr"rRr4r#r$r%r&) rxr&rr&rr&r#r$r%r&)r~rr!r"r#r$r%r) r~rr!r"rrCr#r$r%r)rgr&rr"r#r$r%r&)rr&rrr%r&)r#r$r%r&) rrVrrrrCr#r$r%r&) rr&rrVrrVrr"r#r$r%r&)T)rr&r#r$r%r&)l __future__rtypingrrrrr _C.libtritonr r r*r r Exceptionrr/r2rArPrWrfrqrxr}rrrrrrrrrrrrrrrrrrrsrrrrrrrrrrrr!r*r.r9r>rIrSr\rgrr_rrrrrrrrrrrrrrrr rr,r0r4r9r>rBrFrOrwrr}rrrrrrrrrrrrrrrrrsW"""""";;;;;;;;;;;;;; GCLLFFFFF FFFDDDD FFFF @ @ @ @ , , , ,P < < < <ejGK,1"::::0 4 4 4 4 4 4 4 4RRRR4 : : : :&&&&444465555$5555$ ] ] ] ]"    QQQQ PPPP QQQQ ''''&&&&"""" RRRR RRRR QQQQ####$$$$)))) 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4"DDDD"((((.HHHH[[[[MMMMKKKK2    IIIILLLL$2222tppppZZZZ(04o5o5o5o5o5n                     www,7=7=7=t n n n n gggg :)q)q)qXWWWW,jjjj6$)$)$)$)N$)$)$)$)Noooo888XXXX@ Z Z Z Z&   uuuu.oooo2nnnn            8888 K K K KeeeeUUU0HHHH $`$`$`$`NNNNNNNr