çgFddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZejejeZejedgZdZejd Zd ZGd d eZd ZdZGddeZGddeZdS)N)Path)_build)get_cache_manager) GPUTarget) GPUDriverincludec( ddl}|dkrdSddl ddlm}m}m}m m m}G fdd j } |||||||} dj }n#YdSxYw| g|_ ||_d dz} fd } ||| | r't!j | SdS) NrLinux)c_charc_intc_size_tc_void_pc_char_pPOINTERc eZdZdfdfgZdS)8_find_already_mmapped_dylib_on_linux..DlPhdrInfo dlpi_addr dlpi_nameN)__name__ __module__ __qualname___fields_)rrsV/var/www/html/ai-engine/env/lib/python3.11/site-packages/triton/backends/amd/driver.py DlPhdrInfors& ( # ( # rz libc.so.6ic |jj}ttj|}|jvr4||tt|dSdS)Nrr) contentsrrosfsdecodenamememmoveminlen)infosizedatarpctypeslib_namemax_path_lengths rcallbackz6_find_already_mmapped_dylib_on_linux..callback2saM+ Y'' ( ( qv   NN4CY,P,P Q Q Q1qr)platformsystemr)r r r rrr Structure CFUNCTYPECDLLdl_iterate_phdrargtypesrestypecreate_string_bufferrr string_at)r*r-r r r rr callback_tr2pathr,rrr)r+s` @@@@r$_find_already_mmapped_dylib_on_linuxr9sOOOG##t MMMKKKKKKKKKKKKKKKK        V%   !!%)<)O>OQXQXY_Q`Q`aaJ ++k22Btt *H5O#OO  & &': ; ;Dzz(++T223{6++D11222 4s BB"cxdtjd}|rK|r!tj|r|St d|dt }|r6tj|r|St d|dg}ddl}|}| }|j r|g|z}|D]\}tj |dd }tj|r|cS| |]tjd }|rp| d D]Z}tj |} tj| r| cS| | [tjd d g} fd| D} | D]:} tj| r| cS| | ;tj d} tj| r| S| | t dd|)Nzlibamdhip64.soTRITON_LIBHIP_PATHzTRITON_LIBHIP_PATH 'z' does not point to a valid zmemory mapped 'z'' in process does not point to a valid rtorchlibLD_LIBRARY_PATH:z/sbin/ldconfigz-pcg|]C}|)|dDS))stripendswithsplit).0liner*s r z2_get_path_to_hip_runtime_dylib..psB ^ ^ ^djjll>S>ST\>]>] ^DJJLL  ^ ^ ^rz/opt/rocm/lib/zcannot locate z after attempted paths )rgetenvrCr8exists RuntimeErrorr9sitegetsitepackagesgetusersitepackagesENABLE_USER_SITEjoinappendrD subprocess check_outputdecode splitlines)env_libhip_path mmapped_pathpathsrK site_packages user_siter8env_ld_library_pathdflibslocsloccommon_install_pathr*s @r_get_path_to_hip_runtime_dylibra@sHi 455Ok  # #H - - #"'..2Q2Q #" "i/ii_giijjj8AALn 7>>, ' '  l\llbjllmmm EKKK((**M((**I 4" m3 w||D'5(;; 7>>$   KKK T)$566$**3//  A Q))Aw~~a    LLOOOO  "$4d#; < < C C E ED _ ^ ^ ^):): ^ ^ ^D 7>>#   JJJ S',,'7BB w~~)**#"" LL$%%% PPPPP Q QQrc \tj|d}t |}||d}|t j5}tj |d}t|d5}| |dddn #1swxYwYt|||gtg}t|d5}|||dd}dddn #1swxYwYdddn #1swxYwYddl} | j||} | j| } | j| | S) Nzutf-8z.sozmain.cwrbT)binaryr)hashlibsha256encode hexdigestrget_filetempfileTemporaryDirectoryrr8rOopenwriter include_dirputreadimportlib.utilutilspec_from_file_locationmodule_from_specloader exec_module) srcr!keycache cache_pathtmpdirsrc_pathr\so importlibspecmods rcompile_module_from_srcrsK .G,, - - 7 7 9 9C c " "E4 --J  ( * * Lfw||FH55Hh$$                 hKDDBb$ L1"YYqvvxxDdYKK  L L L L L L L L L L L L L L L  L L L L L L L L L L L L L L L > 1 1$ C CD . ) )$ / /CKC   JsZ61E'C = E C EC ,E=.D7+ E7D; ;E>D; ?EEEc$eZdZfdZdZxZS)HIPUtilsct|ds-tt|||_|jS)Ninstance)hasattrsuperr__new__r)cls __class__s rrzHIPUtils.__new__s<sJ'' = 3//77<"""r)rrrrr __classcell__rs@rrrsG ? ? ? ? ? ? ?rrcN|ddkrdSddddddd d dd d d d d d d|S)Nr*hipDeviceptr_tint32_tint8_tint16_tint64_tuint32_tuint8_tuint16_tuint64_tfloatdoublei1i8i16i32i64u1u8u16u32u64fp16bf16fp32f32fp64tys r ty_to_cpprs[ !u||    !  rc t|}dd|D}d d d fd|D}d|z}t|dkr4ddd |Dznd}t } fd |D} d | d t|dkrd|zndd dd| Dd|dd fd|Dd|d|ddd|Ddt|dkr4ddd|Dzndd} | S)N, c3DK|]\}}t|d|VdS)z argN)rrEirs r z make_launcher..s:SS2Yr]]3333SSSSSSrcN|ddkrdSddddddd d dd d d d d d d|S)Nrr PyObject*rrrrrrrrrrrrrs r_extracted_typez&make_launcher.._extracted_typesZ a5C<<;  ! rc ,dddddddddd d d d |S) NOr\r[lbhrBHIK) rrrlongrrrrrrrrrrs r format_ofz make_launcher..format_ofs?      rc8g|]}|Srr)rErrrs rrGz!make_launcher..s-WWWb99__R%8%899WWWr iiiKKOOOOrc3&K|] \}}d|V dS)z&_argNrrs rrz make_launcher..s, L LB L L L L L Lrcg|]}|v| Srr)rEr constantss rrGz!make_launcher..s# @ @ @AQi-?-?a-?-?-?ra; #define __HIP_PLATFORM_AMD__ #include #include #include #include #include // The list of paths to search for the HIP runtime library. The caller Python // code should substitute the search path placeholder. static const char *hipLibSearchPaths[] = {"a"}; // The list of HIP dynamic library symbols and their signature we are interested // in this file. #define HIP_SYMBOL_LIST(FOR_EACH_ERR_FN, FOR_EACH_STR_FN) \ FOR_EACH_STR_FN(hipGetErrorString, hipError_t hipError) \ FOR_EACH_ERR_FN(hipModuleLaunchKernel, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipPointerGetAttribute, void *data, \ hipPointer_attribute attribute, hipDeviceptr_t ptr) // The HIP symbol table for holding resolved dynamic library symbols. struct HIPSymbolTable { #define DEFINE_EACH_ERR_FIELD(hipSymbolName, ...) \ hipError_t (*hipSymbolName)(__VA_ARGS__); #define DEFINE_EACH_STR_FIELD(hipSymbolName, ...) \ const char *(*hipSymbolName)(__VA_ARGS__); HIP_SYMBOL_LIST(DEFINE_EACH_ERR_FIELD, DEFINE_EACH_STR_FIELD) }; static struct HIPSymbolTable hipSymbolTable; bool initSymbolTable() { // Use the HIP runtime library loaded into the existing process if it exits. void *lib = dlopen("libamdhip64.so", RTLD_NOLOAD); if (lib) { // printf("[triton] chosen loaded libamdhip64.so in the process\n"); } // Otherwise, go through the list of search paths to dlopen the first HIP // driver library. if (!lib) { int n = sizeof(hipLibSearchPaths) / sizeof(hipLibSearchPaths[0]); for (int i = 0; i < n; ++i) { void *handle = dlopen(hipLibSearchPaths[i], RTLD_LAZY | RTLD_LOCAL); if (handle) { lib = handle; // printf("[triton] chosen %s\n", hipLibSearchPaths[i]); } } } if (!lib) { PyErr_SetString(PyExc_RuntimeError, "cannot open libamdhip64.so"); return false; } // Resolve all symbols we are interested in. dlerror(); // Clear existing errors const char *error = NULL; #define QUERY_EACH_FN(hipSymbolName, ...) \ *(void **)&hipSymbolTable.hipSymbolName = dlsym(lib, #hipSymbolName); \ error = dlerror(); \ if (error) { \ PyErr_SetString(PyExc_RuntimeError, \ "cannot query " #hipSymbolName " from libamdhip64.so"); \ dlclose(lib); \ return false; \ } HIP_SYMBOL_LIST(QUERY_EACH_FN, QUERY_EACH_FN) return true; } static inline void gpuAssert(hipError_t code, const char *file, int line) { if (code != HIP_SUCCESS) { const char* prefix = "Triton Error [HIP]: "; const char* str = hipSymbolTable.hipGetErrorString(code); char err[1024] = {0}; snprintf(err, 1024, "%s Code: %d, Messsage: %s", prefix, code, str ); PyErr_SetString(PyExc_RuntimeError, err); } } #define HIP_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, hipStream_t stream, hipFunction_t functionz>) { // printf("_launch hip kernel\n"); void *params[] = { c3 K|] }d|V dS)z&argNr)rErs rrz make_launcher..Ms( < < < < < < < 0) { HIP_CHECK(hipSymbolTable.hipModuleLaunchKernel(function, gridX, gridY, gridZ, aw*num_warps, 1, 1, shared_memory, stream, params, 0)); } } typedef struct _DevicePtrInfo { hipDeviceptr_t dev_ptr; bool valid; } DevicePtrInfo; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ptr = PyObject_GetAttrString(obj, "data_ptr"); if(ptr){ PyObject *empty_tuple = PyTuple_New(0); PyObject *ret = PyObject_Call(ptr, empty_tuple, NULL); Py_DECREF(empty_tuple); Py_DECREF(ptr); if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; return ptr_info; } ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(ret); if(!ptr_info.dev_ptr) return ptr_info; uint64_t dev_ptr; hipError_t status = hipSymbolTable.hipPointerGetAttribute(&dev_ptr, HIP_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == hipErrorInvalidValue) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; } ptr_info.dev_ptr = (hipDeviceptr_t)dev_ptr; Py_DECREF(ret); return ptr_info; } PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); return ptr_info; } static PyObject* launch(PyObject* self, PyObject* args) { // printf("launch\n"); int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL;  c8g|]\}}|d|dS)z _arg; r)rErrrs rrGz!make_launcher..s8 O O OEAr## / /! / / / O O Orz if(!PyArg_ParseTuple(args, "z", &gridX, &gridY, &gridZ, &_stream, &_function, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hook a=)) { return NULL; } // extract kernel metadata int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ; if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) { return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* args = Py_BuildValue("(O)", launch_metadata); PyObject* ret = PyObject_CallObject(launch_enter_hook, args); Py_DECREF(args); if (!ret) return NULL; } // raise exception asap rc Ng|]"\}}|ddkrd|d|d|d|d nd#S)rrzDevicePtrInfo ptr_infoz = getPointer(_argrz); if (!ptr_infoz.valid) return NULL;rrrs rrGz!make_launcher..s~dddINIJLNoqrsotx{o{o{kqkkAkkkkTUkkkkBDdddrz; _launch(gridX, gridY, gridZ, num_warps, num_ctas, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (hipStream_t)_stream, (hipFunction_t)_functionc3JK|]\}}|ddkrd|dnd|VdS)rrptr_infoz.dev_ptr_argNrrs rrz make_launcher..sj~j~chcdfhEGHIEJLOEOEOkAvwkAkAkAkAU_\]U_U_j~j~j~j~j~j~ran); if(launch_exit_hook != Py_None){ PyObject* args = Py_BuildValue("(O)", launch_metadata); PyObject* ret = PyObject_CallObject(launch_exit_hook, args); Py_DECREF(args); if (!ret) return NULL; } if(PyErr_Occurred()) { return NULL; } // return None Py_INCREF(Py_None); return Py_None; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { if (!initSymbolTable()) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } )r$rOitemsvaluesrakeys)r signatureids warp_size start_desc arg_decls args_formatformat args_listrparamsrxrrs` @@r make_launcherrsmYJ SSARARSSSSSI* ''WWWWWIDTDTDVDVWWWXXK ; &FPST]P^P^abPbPbtyy L L)//:K:K L L LLLLLhjI022KA @ @ @)) @ @ @F^ .9^ ^ zX[\eXfXfijXjXjDHKTDTDTpr{^ ^ ~yy < z&HIPLauncher.__init__..s/As9K9KRCF,221555QRrc.i|]\}}||SrrrEryvaluecst_keys r z(HIPLauncher.__init__..s'MMMZS%WWS\\5MMMrc.i|]\}}||Srrrs rrz(HIPLauncher.__init__..s'QQQZS%WWS\\5QQQr__triton_launcher) rr constexprstuplerdictrrrrrlaunch)rrxmetadatarrrrrs ` @rrzHIPLauncher.__init__s#'#t:L:L%YSV%6%6RWRYRYZ%,S+%>%>JCMMDFF RRRRMMMM9??;L;LMMM QQQQ3=;N;N;P;PQQQ Iy#x7IJJ%c+>??j rc |j|i|dSr)r)rargskwargss r__call__zHIPLauncher.__call__s T$V$$$$$rN)rrrrrrrrrrs2!!!%%%%%rrc:eZdZfdZedZdZxZS) HIPDriverctt|_t|_dSr)rrrutilsr launcher_cls)rrs rrzHIPDriver.__init__s2 ZZ 'rc&ddl}|jjduS)Nr)r<versionhip)r<s r is_activezHIPDriver.is_actives } ,,rc|}|j|}|d}|d}td|dd|S)NarchwarpSizer r?r)get_current_devicerrrrD)rdevicedevice_propertiesrrs rget_current_targetzHIPDriver.get_current_targets^((** J<>>r)rrrr staticmethodr rrrs@rrrsb((((( --\-???????rr) functoolsrrfrQrkpathlibrtriton.runtime.buildrtriton.runtime.cachertriton.backends.compilerrtriton.backends.driverrr8rrealpath__file__rOror9 lru_cacherarobjectrrrrrrrrrs ''''''222222......,,,,,, '//"'**844 5 5w||GY//0 ---`;R;R;R|&?????v???(   ,QQQh % % % % %& % % % ????? ?????r