Ngd#ddlZddlZddlZddlmZmZddlZddlmZ ddl Z ddl m Z m Z mZmZddlmZddlmZddlmZejeZGdd Zd Zed kreZejreejejZ ej!Z"ej#$e"r(e%d e"d e&d e"d e j'e Z(ee(ej)ej*ej+Z,e,-e,j(.e"ddSdS)N)ListTuple) GraphProto ModelProto NodeProto TensorProto)quantize_matmul_bnb4) ONNXModel)attribute_to_kwargceZdZdZdZdZddededefdZe d e e d e e e ffd Zd ejd ejfd Zdede e d efdZde e fdZdZdS)MatMulBnb4QuantizerzMPerform 4b quantization of constant MatMul weights using FP4 or NF4 data typerr Nmodel quant_type block_sizec|pg}|tjtjfvsJt||_||_||_t||_dSN) rFP4NF4r rrrsetnodes_to_exclude)selfrrrrs j/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/quantization/matmul_bnb4_quantizer.py__init__zMatMulBnb4Quantizer.__init__&s_+1r157J7NOOOOOu%% $$ #$4 5 5 graph_pathreturnctt|dz ddD]'}||}|jD]}|j|kr||fccS(dS)Nr )NN)rangelen initializername)r#rgidgraphtensors r__get_initializerz%MatMulBnb4Quantizer.__get_initializer.swZ1,b"55 ) )CsOE+ ) );$&&!5=(((((' )zrfpweightc t|jdkrtd|}|j\}}||z}|j}||zdz |z}|dzdz}t j|d} t j||j} t| || ||j ||| | fS)z4b quantize fp32/fp16 weightz9Current bnb4 block quantization only supports 2D tensors!r uint8)dtype) r!shape ValueError transposecopyrnpzerosr,r r) rr( fpweight_trowscolsnumelr num_blocksquantized_numelpackedabsmaxs rbnb4_block_quantz$MatMulBnb4Quantizer.bnb4_block_quant7s x~  ! # #XYY Y''))..00 ^ dt _ j(1,;  19*/999*HN;;;VZT_VZ\`aaarnode graph_stackc|jdkr|Std|jd|j|jvr%td|jd|S|jd}t ||\}}|td|Stj |}t|j d krtd |S| |\}}tj |} |jd z| _|jD])} | j|kr|j| n*tj |} |jd z| _|j| | gi} |j \} }| | d <|| d<|j| d<|j| d<tjj d|jd| j| jg|jdg|jr |jd znddd| }td|jd|S)zdIf the node is MatMul with fp32 const weight, quantize the weight with int4, and return the new nodeMatMulzstart to quantize z ...zexclude to quantize z$ as specified by nodes_to_exclude...r Nz2MatMul doesn't have const weight. Skip to quantizer*z)MatMul weight is not 2D. Skip to quantize_Bnb4_absmaxKNrr MatMulBnb4r com.microsoft)inputsoutputsr#domainzcomplete quantization of )rD)op_typeloggerdebugr#rinputr%_MatMulBnb4Quantizer__get_initializeronnx numpy_helperto_arrayr!r-r; from_arrayremover"extendrrhelper make_nodeoutput)rr<r=inputBBBs_graphB_arrayr9r:B_quantrM absmax_tensorkwargsr4r5matmul_bnb4_nodes r_bnb4_matmul_node_weightz,MatMulBnb4Quantizer._bnb4_matmul_node_weightMss <8 # #K 9$)999::: 9- - - LL_ ___ ` ` `KA);;FKPP 8 9 LLM N N NK#,,Q// w}   " " LLD E E EK..w77#..v66v' ^  EzV##%%e,,,$)44V<< Vi/ ##Wm$<===] ds s #|#|;0  JqM7<1CD[^$(, 9W$$r"        @@@@AAArcg}|d}|jD]k}d|jD}t|ri}|jD]}|jtjjkr7||j|j | |i}n}|jtjj krTg} |j D]@} || | | |gA|j | i}nt|}||t jj|j|j|jfd|j i|}||||m|d|j |||S)Nrczg|]8}|jtjjks|jtjjk6|9S)typerOAttributeProtoGRAPHGRAPHS).0attrs r z9MatMulBnb4Quantizer._process_subgraph..sG9 3 999TY$J]Jd=d=d=d=d=drr#r<)r< attributer!rdrOrerfappendgr#_process_subgraphrggraphsrTr updaterUrVrJrMrWr` ClearFieldpop) rr= new_nodesr%r< graph_attrsr^rikvvaluesubgraphs rrnz%MatMulBnb4Quantizer._process_subgraphs BJ O OD NK ;  N&&DyD$7$===#**46222"i)?)? )L)LMd&9&@@@ "(, PPH'..x888!LL$*@*@*M*M)NOOOO"i//55MM"%%%%{,L$*dk@D MS   T::4MM N N N N     )$$$ rc\|jg}|j}d}|D]}|jdkrd}|s4|t jddg|||j dS)NFrFTr ) rr% opset_importrIrTrOrU make_opsetidrnclean_initializers)rr=ry has_ms_domainopsets rprocesszMatMulBnb4Quantizer.processsz''))* z..00  ! % %E|.. $  P   !9!9/1!M!M N O O O {+++ %%'''''rr)__name__ __module__ __qualname____doc__rrrintr staticmethodrrrrrNnpt ArrayLiker1ndarrayr;rr`rnr~rcrrrrsWW C C66j6c6s6666D,<{T^G^A_\  2:    ,5 Y5 T*EU5 Zc5 5 5 5 n$T*-=$$$$L ( ( ( ( (rrctjd}|ddd|ddd|d d d tjtjgd |dd dd|ddd d|d |ddtd gd|S)NaBlockwise FP4/NF4 quantization for MatMul 2D weight matrices. A weight matrix is partitioned into blocks, where each block is a contiguous subset inside the flattened transposed weight matrix. Each block is quantized into a set of 4b integers with an absolute value scaling factor. ) descriptionz --input_modelTzPath to the input model file)requiredhelpz--output_modelzPath to the output model filez --quant_typeFr z&Quantization data type. 0: FP4, 1: NF4)rdefaultchoicesrz --block_size@zVBlock size for blockwise quantization. Note: bnb.nn.Linear4bit only uses block_size=64)rrrz-vz --verbose store_true)raction)verbosez--nodes_to_exclude+zBSpecify the nodes to be excluded from quantization with node names)nargsrdrrr) argparseArgumentParser add_argumentrrr set_defaultsstr parse_args)parsers rrrs5  $F $=[\\\ (4>]^^^ $(*=*AB 5   e   kE,OOO &&&   Q      r__main__zfile z already exists)rT)/rloggingostypingrrnumpyr1 numpy.typingrrO onnx.onnx_pbrrrronnxruntime.capi._pybind_stater onnx_modelr quant_utilsr getLoggerrrKrrargsrsetLevelDEBUG input_modelinput_model_path output_modeloutput_model_pathpathexistserror Exceptionloadrrrrquantr~save_model_to_filercrrrs  GGGGGGGGGGGG??????!!!!!!++++++  8 $ $^(^(^(^(^(^(^(^(B$$$N z :<