çg_.zddlmZddlmZddlmZmZddlmZdZ Gdde Z eGdd Z Gd d Z d Zd Zdedee defdZde defdZde defdZdedee defdZde defdZdede defdZdede defdZde defdZde defdZdZedkrddlmZeeZed d!d"#ed$d%ed&'ed(ed)d*+e Z!e ZgZ"e!j#D]PZ$ee$Z%e%&Z'e"(e%j)e*e'Qd,ej+,DZ-d-ej+,DZ.e.ddZ/ee/Z0ee/Z1e!j23d.4d/5Z5d0Z2e2d16e-z Z2e2d1z Z2e2e0z Z2e2d1z Z2e2e1z Z2e57e2d2d2d2n #1swxYwYd3ej+,DZ8d4ej+9DZ:ee:e/Z;ee/Z<ee:e/Z=ee/Z>ee/Z?e!j23d54d/5Z5d)Z2e2d0z Z2e2d6z Z2e2d7z Z2e2d1z Z2e2d16e8z Z2e2d1z Z2e2e;z Z2e2d1z Z2e2e>z Z2e2d1z Z2e2e????%rrBkerc Z||jvr|j|d}t|j|jD]S\}}||krHtd|dd|jdd|jT|j||dS)NrSz Mismatched signature for kernel z: existing sig is: rOz current is: )r,ziprrr=rI)r-rBr[lastcurnew_s r r<zHeaderParser._add_kernelms 4<  %)\$%7%;D #.AA   T$;;%^4^^X[X`X`aeapXqXq^^DGDLDLMPM[D\D\^^ T!!#&&&&&r)r#N) rrrr.rrDr9r:r;rr<r rr r"r"s ) ) ) )#.DCDDDD H# H H H H C     . ' '*: ' ' ' ' ' 'rr"cpddt|j|jDS)N, c"g|] \}}|d| S r .0rLargs r z0gen_signature_with_full_args..{s&RRRCmmcmmRRRr)r=r]rrr?s r gen_signature_with_full_argsrkzs1 99RR3q|Q[3Q3QRRR S SSrcdt|j|jD}dt|j|jD}ddt||D}|S)Nc$g|] \}}|dk |Sr1r )rgrLhints r riz!gen_signature..s!LLLD$!)))))rc$g|] \}}|dk |Srnr rgrhros r riz!gen_signature..s!MMMd4199999rrbc"g|] \}}|d| Srdr rfs r riz!gen_signature..s&LLLwr3]]S]]LLLr)r]rrrr=)r? arg_typesrsigs r gen_signatureru~ssLLC ag$>$>LLLIMMc!+qw&?&?MMMI ))LL#i2K2KLLL M MC JrrBmetasr#c Fd|dt|dd|d|d S)N CUresult (CUstream stream, rSz ); void load_(); void unload_(); )rk)rBrvs r make_algo_declsr|sS    ">uRy"I"I        rmetac d|jdt|d|jdt|d|jd|jd S)Nrx_default(CUstream stream, z ); CUresult ryz, int algo_id); void load_rzr{)rrk)r}s r make_global_declrs     ;WX\;];]      3OPT3U3U       "    rcd|jdt|d}|d|jdd|jdz }|dz }|S) N CUresult rz){ return (stream, rbz, 0); } rrkr=rr}srcs r make_default_algo_kernelrsd pd+ p pGcdhGiGi p p pC Z- Z Z $.8Q8Q Z Z Z[C5LC Jrc d|d}t|dD]0}|d|jd|jd|jdt |d z }1|dz }|d|dt |d d z }|dz }t|d D]}d d fdt|j|j D}|t|j rd|dndz }dt|j|j D}|d|jd|jd|jdd|d z }|dz }|dz }|dz }dD]}|d|d|dz }t|dD]#}|d|d|jd|jd|jd z }$|d|d|dz }|dz }t|dD]#}|d |d|jd|jd|jd z }$|dz }|S)!Nz// launcher for:  c|j Sr rrjs r z.make_kernel_hints_dispatcher.. Q[Lr)keyrr4ry); rSz){c|j Sr rrjs r rz.make_kernel_hints_dispatcher..rrcB|dkr d|d|dn|dkr d|d|dndS)NrP(z % z == 0)r1z == )r )valros r rz.make_kernel_hints_dispatcher..sTrzz7#66$6666qyy&S%%d%%%% rz && c2g|]\}}|||Sr r )rgrrocond_fns r riz0make_kernel_hints_dispatcher..s9   T GC  rz if (z) zif (1) c$g|] \}}|dk |Srnr rqs r riz0make_kernel_hints_dispatcher..s"WWWYS$TUVYYSYYYrz return rrbz# return CUDA_ERROR_INVALID_VALUE; rloadunloadz // z for: c|j Sr rrjs r rz.make_kernel_hints_dispatcher..  rvoid (); z() {c|j Sr rrjs r rz.make_kernel_hints_dispatcher..rr ) sortedrrrrurkr=r]rrany)rBrvrr}condsrmoders @r make_kernel_hints_dispatcherrs &d & & &Cu"8"8999|| {40{{4={{4;{{boptbubu{{{{4KC \ \ \0LUSUY0W0W \ \ \]C4KCu"8"8999vv          <<      s4:F#####J WW#dndj*I*IWWW  uT2uuT]uuT[uu[_[d[den[o[ouuuu4KC 11C5LC" +t++4++++5&<&<=== ] ]D \4\\$"7\\$-\\$+\\\ \CC )t))d)))) t 5&<&<=== \ \D ZZZ 5ZZ ZZ ZZZ [CC u  Jrcd|jdt|d}|d|jdz }|d|jdd|jd z }|d z }|S) Nrryz, int algo_id){ z assert (algo_id < (int)sizeof(z _kernels)); rz_kernels[algo_id](stream, rbrrrrs r !make_kernel_meta_const_dispatcherrs ud+ u u?[\`?a?a u u uC Rd.C R R RRC gt, g g RVR`HaHa g g ggC5LC Jrnamescndt|d}|d|jdz }|D] }|d|dz } |dz }|S)Nz3typedef CUresult (*kernel_func_t)(CUstream stream, rzkernel_func_t z_kernels[] = { rz, z}; )rkr)rr}rrBs r make_func_pointersrsf h@\]a@b@b h h hC DD1 D D DDC ~D~~~6MC Jrcbd}dD])}|d|d|jdz }|D]}|d|d|dz }|dz }*|S) Nrrr4z(void){ rrz} r)rr}rrrBs r make_kernel_load_defrsv C" ?t??d3???? + +D ***t*** *CC w Jrcd|jd}|S)Nint z_get_num_algos(void);rrs r make_get_num_algos_declrs =& = = =C JrcRd|jd}|d|jd|jdz }|dz }|S)Nrz_get_num_algos(void){ z return (int)(sizeof(z_kernels) / sizeof(z_kernels[0])); rrrs r make_get_num_algos_defrsG @& @ @ @C uD$9 u udNc u u uuC5LC Jra0 Triton ahead-of-time linker: This program takes in header files generated by compile.py, and generates a single entry-point responsible for dispatching the user's input to the right kernel given the specializations that were compiled. Example usage: python link.py /path/to/headers/*.h -o kernel_name __main__)ArgumentParser) descriptionheaders+z_Paths to header files to link. Must include linker directive annotations (autogenerated by ttc))nargshelpz--outz-oz Out filename)typerz--prefixrz(String to prefix kernel dispatcher names)rdefaultrc4g|]\}}t||Sr )r|rgrBr}s r riris&WWW*$/$--WWWrcg|]\}}|Sr r rs r riris@@@:4$@@@rz.hwz#include rNc4g|]\}}t||Sr )rrs r riri*s' ^ ^ ^t (t 4 4 ^ ^ ^rcg|]}|Sr r )rgrBs r riri+s 4 4 4dT 4 4 4rz.cz#include z#include )@ collectionsrpathlibrtypingrr dataclassesrr Exceptionrrr"rkrurr|rrrrrrrrdescrargparserparser add_argument parse_argsrKincludesrr/h_path read_texth_strrIrBrDr,items algo_decls meta_listsr}get_num_algos_decl global_declout with_suffixopenfpr=writedefskeysrfunc_pointers_defmeta_const_defload_unload_defget_num_algos_defdefault_algo_kernelr rr rs######""""""""!!!!!!     )     , , , , , , ,  ,Z'Z'Z'Z'Z'Z'Z'Z'zTTT#h/?&@S+#3%s%8