Ng8 ddlmZddlZddlZddlZddlmZddlmZddl Z ddl Z ddl m Z m Z m Z ddl mZddlmZmZmZmZddlmZdd lmZmZmZ dd lmZn #e$rdZYnwxYw dd lmZmZn#e$rdZdZYnwxYw dd l m!Z!n #e$rdZ!YnwxYwd Z"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+iZ,de-e DZ.GddeZ/GddeZ0GddeZ1GddeZ2ej j3e j4d ej j5e j4d!ej j6e j4d"ej j7e j4d#ej j8eej j9eej j:eiZ;ej j5e j<de j=$e j<d%e j=$fej j3e j<d&e j>$e j<d'e j>$fej j7e j<de j?$e j<d(e j?$fej j6e j<d)e j@$e j<d*e j@$fej j:e j<de$e j<d+e$fej j9e j<d,e$e j<d-e$fiZAej j5e j<de j=$e j<d.e j=$fej j3e j<d/e j>$e j<d'e j>$fej j7e j<de j?$e j<d0e j?$fej j6e j<d1e j@$e j<d*e j@$fiZBej j5e j<de j=$e j<d'e j=$fej j3e j<d2e j>$e j<d3e j>$fej j7e j<de j?$e j<d*e j?$fej j6e j<d4e j@$e j<d5e j@$fej j:e j<de$e j<d-e$fej j9e j<d6e$e j<d7e$fiZCd8d9d:ZDdd;ZEdd=ZFd>ZG dddLZH dddNZI dddWZJddXZKddYZLdd]ZMddaZNGdbdcZOGdddeZPGdfdgZQdhZRdiZSdjZTdkZUddpZVdqZWddsZXdduZYddwZZddyZ[dd|Z\dd}Z]dd~Z^ddZ_ddZ`ddZaddZbddZcddZdddZeddZfddZgddZhdS)) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4)to_array_extendedz onnx.quantizez0.1.0zai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedlci|]@}ttt|t*tt||AS) isinstancegetattrrint).0ks `/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/quantization/quant_utils.py r$8sAqqqq SZ[fhiSjSjloHpHpq Q''qqqc2eZdZdZdZdZedZdS)QuantizationModerc|jSNnameselfs r#__str__zQuantizationMode.__str__C yr%cV t|S#t$rtwxYwr*)r'KeyError ValueError)modes r# from_stringzQuantizationMode.from_stringFs7 #D) )   ,,   (N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr/ staticmethodr5rr%r#r'r'?sHJJ\r%r'c2eZdZdZdZdZedZdS)QuantizedValueTyperr(c|jSr*r+r-s r#r/zQuantizedValueType.__str__Rr0r%cV t|S#t$rtwxYwr*)r>r2r3)vs r#r5zQuantizedValueType.from_stringUs7 %a( (   ,,  r6N)r7r8r9Input Initializerr/r<r5rr%r#r>r>NsH EK\r%r>c\eZdZdZdZdZdZdZdZdZ dZ e d Z e d Zd S) QuantTyperr(c|jSr*r+r-s r#r/zQuantType.__str__fr0r%cV t|S#t$rtwxYwr*)rEr2r3)ts r#r5zQuantType.from_stringis6 Q<    ,,  r6c|tjkr tjS|tjkr tjS|tjkr tjS|tjkr tj S|tj kr tj S|tj kr tj S|tjkr tjSt!d|d)NzUnexpected value qtype=.)rEQInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r3r-s r# tensor_typezQuantType.tensor_typeps 9? " "# # 9# # #$ $ 9$ $ $% % 9# # #$ $ 9* * *+ + 9# # #$ $ 9? " "# #<4<<<===r%N)r7r8r9rPrRrXrVrTr\rZr/r<r5propertyr^rr%r#rErE]s| E FM FG E F\ >>X>>>r%rEc2eZdZdZdZdZedZdS) QuantFormatrr(c|jSr*r+r-s r#r/zQuantFormat.__str__r0r%cV t|S#t$rtwxYwr*)rar2r3)formats r#r5zQuantFormat.from_strings7 v& &   ,,  r6N)r7r8r9 QOperatorQDQr/r<r5rr%r#rarasHI C\r%raint8uint8int16uint16dtypeiiiiiiiii@ii@rGzero_point_indexcTg}t|D]\}}tjt|tjr(|tj|nEt|tjr||ntd|d|||krI|d}|j tj ks|j tj krtd|j t|dkrt|n|dS)Nzarg z is not an array: rtzzero_point cannot be r(r) enumeratenumpy issubdtypetypenumberappendarrayrndarray TypeErrorrlfloat32float16lentuple)rvargsnew_argsiarAs r# _check_typersH$ C C1  DGGU\ 2 2 = OOEKNN + + + + 5= ) ) = OOA    ;1;;;;<< <   Aw%-''17em+C+C A A ABBB!(mma//5???Xa[@r%c|tvsJd|d|tjjtjjtjjtjjfvrF|dkrtd|d|jtj kr tj }n:|jtj kr tj }ntd|jdtt!t#dgdgt$jd|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|d||ddSt|} t3|dd\} } |t5| |n| } |t7| |n| } tj|tj |z |z}tj|| | |t/|| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rOzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrF) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protorrYFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrlryrFLOATrFLOAT16r3r r r onnxhelper make_tensorr rrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefrlqminqmaxcliplowcliphigharr_fp32s r#quantize_nparrayrst %%%%duddd &%% +-)-  ??%&j[e&j&j&jkk k 9 % %#)II Y%- ' '#+II====>> > "Bdk>U>UVbdikmpqor>s>s.0L0L0LseTT  *3 4@@*7ItDD(UD99:    !,,3774sU)C)CDDQGHHH%U+,URWXXX d$'O#dC...&*&63tT???D=#**U]";";e"C!J!J!L!Lz!YZZ 8WhH====8??511222r%Fc |dks|dkrtd|d|tj|tjd|j}tj|tjd|j}|,t ||tj||jz}|r?tjtj|tj|}| }| }||ksJd|d|tj||z tj }tj|tj tj|tj z }tj||z } | dks Jd| tj |jj kr7tjd |j} tjd|j} n|rRtjtj ||ztjd tj z |j} n3tjtj ||| z z |j} | |j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rkNzqmin=z > qmax=z scale isse?g@)r3ryminimumr~rlmaximumrrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r#compute_scale_zprsQ( axx4!88r^brrlprrsss =u{1DJ??? @ @D =u{1DJ??? @ @D!4 nDJ O O OOPPuy $@@ww 4<<<555t55<<< TD[ 6 6 6B T / / /%+d%-2X2X2X XB KR E A:::|::: u{4:&&+++ Ctz222[$*555  Y  TD[EK5=,Q,Q,QQRRZ^ZdJJU[u 1D%E%ETZXXXJ TZ((  r%c& d}|tvr|tjkrTddlm ddlm}|} fdtdD}tj d|Dtj }ntd |d |t|<n|tjkrddlm}|}|td |d tj t|}tj d|}tj ||z |j}||gS) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)float8e4m3_to_float32rc&g|] }|Srr)r!rrs r# z+compute_scale_zp_float8..Js%GGGq//22GGGr%cbg|],}tj|tj|*|-Sr)ryisnanisinf)r!fs r#rz+compute_scale_zp_float8..Ls3TTTqek!nnTU[QR^^TTTTr%rkzQuantization to element_type=z not implemented.zUnexpected element_type rO)FLOAT8_DISTRIBUTIONSrrYonnx.numpy_helperr#onnx.reference.custom_element_typesrrangeryr~rr3rstdrl) element_typerzp_dtyper all_valuesvaluesstd_f8zerorrs @r#compute_scale_zp_float8r9sWH/// ;3 3 3 ? ? ? ? ? ? H H H H H H#HGGGGE#JJGGGJ[TTJTTT\a\iFF\\\\\]] ]-3\** 1 1 1DDDDDDB<BBBCCC Y+L9 : :F ;q ) ) )D Kf CI 6 6 6E %=r%data numpy.ndarray quant_typeonnx.TensorProto.DataTyperboolrr float | None rmin_override rmax_overridereturn#tuple[numpy.ndarray, numpy.ndarray]ct|tjs tdt |d||}n%t |r|nd}||}n%t |r|nd}tj||j }tj||j }tjd|j } |tj krJ|rtdtj |} t|| \} } t| | d S|tjtjtjtjtjtjfvrit-||| \} } t |rt/||| | ||\} } ntjd| j } t| | d St1d |d) a Returns the zero_point and scale for the given data. :param data: The data for which to compute quantization parameters. :param quant_type: The quantization data type. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: zero point and scale z%Weight must be given as an array not rONgrkrz1Unsupported option reduce_range=True for float 8.rrurz Unexpected value for quant_type=)rryrrr{rrrr~rlrrY RuntimeErrorrrrrQrSrWrUr]r[rrr3)rrrrrrrrrrrrrrs r#compute_data_quant_paramsr^s* dEM * *OMT MMMNNN  YY/txxzzzC  YY/txxzzzC ;t4: . . .D ;t4: . . .D K4: . . .E[---  TRSS Sioo3JDD E:uqAAAA -ZQZ[[[ d t99 : 0tT4Tb c c JQdj999J:uqAAAA E EEE F FFr%2tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]c t|||||||\}}|tjkrt||||} t | t jdzdkrxt j |} td| d| d| d| d ||| fS|tj tjtjtjtjtjfvrt||||} ||| fSt'd|d)al :param data: data to quantize :param qType: data type to quantize to. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point rnz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rO)rrrYranyrryrhravelrrrrrQrSrWrUr]r[r3) rrrrrrrrrquantized_datanp_datas r# quantize_datars82  J ((()%ujII %%ek2288::S@SH I I mD))GWgkkmmWWw{{}}WW&4&8&8&:&:WW>L>P>P>R>RWWW 5.00  *%ujII5.00 ;5;;; < <JJJRRSYS^__ #0;;M=YY r%c|tjjkrtdd}|rt|}n3|r|t vrt |}nt|}|std|d|\}}|dks|dkr&td|d|d|j d |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rrrYrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr3rl)rrrqrangerrs r#rr(s#  &333!"_``` F0,0077 0u ===.u5$((// xvvvvwwwJD$ axx4!88 4 4 4"& 4 404  4 4KW 4 4" 4 4,1 4 4   Mr%c6t|||\}}||z S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r)rrrrrs r#get_qrange_for_qTyper&Hs& ) RRRJD$ $;r%r ranktuple[bool, int]c<|dkr||zn|}|dko||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr)rr' axis_normis_valids r#normalize_axisr,Rs7 $axxt TIA~2)d"2H Y r%src_8bitr  bytearrayc"t|}|dkrtS|dzdz}t|}d}d}||dz kr3||dzdzdz||dzz||<|dz }|dz }||dz k3||kr||dz||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rr(rFrorH)rr.)r- num_elemsdst_sizedstsrc_idst_is r#r r ^sH IA~~{{A !#H H  CEE )a-   *S0Q68E?S;PQE      )a-    ye_s*E Jr%c eZdZdZggdfdZdS)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_dSr*) r, initializerrminsrmaxs zero_pointsscalesrrr) r.r,r8r9r:r;r<rrrs r#__init__zQuantizedInitializer.__init__sJ &  &  , r%r7r8r9__doc__r=rr%r#r6r6|s= r%r6c"eZdZdZ ddZdS)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_dSr*) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r.r,new_quantized_namerEzero_point_namequantized_value_typerrHrIrJs r#r=zQuantizedValue.__init__sH"( $& . "$$r%)NNNNr>rr%r#rArAs@%%%%%%r%rAceZdZdZdZdS)BiasToQuantizez+ Represents a bias to be quantized c0||_||_||_dSr*) bias_name input_name weight_name)r.rQrRrSs r#r=zBiasToQuantize.__init__s"$&r%Nr>rr%r#rOrOs-'''''r%rOc|jdkrtd|jd|jdkr|j}n|jdkr|j}n|jdkr|j}n|jdkr|j}n|jdkr|j}n|jd kr|j}nl|jd kr|j }nY|jd kr|j }nF|jd kr|j }n3|jd kr|j }n td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.r(rFrGrHrIrJrp rz has unsupported type rO) r{r3r,rrsrMgfloatsintsstringstensorsgraphs) attributers r#attribute_to_kwargr_sB ~TinTTTUUU~  1    1    1    1    1    1   1  ! 1  ! 2   ]in]]IN]]]^^^ NE ""r%cZfd|D}t|dkr|dndS)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. c*g|]}|jk |Srr+)r!item item_names r#rz find_by_name..s% B B Bd49 +A+AT+A+A+Ar%rN)r)rc item_listitemss` r# find_by_namerfs; C B B Bi B B BE5zzA~~5884/r%cdd}tt|D]}|||kr|}|S)zC Helper function to return index of an item in a node list rt)rr) elem_name elem_listelem_idxrs r#get_elem_indexrks@H 3y>> " " Q<9 $ $H Or%cHtjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr )inputsoutputr,s r# get_mul_noderps" ; $ ? ??r%filenamer identifierr cV|j|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)rqrrs r#generate_identified_filenamerxs( ? # #HMJ$>$P Q QQr%cddl}ddlm}ddl}|j|jt dt |t dt ||||d|d| d| d | dS) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotryset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesr}pltrys r# apply_plotrsJJJ######LLLES[1111 , $KKK  *JJtZdJ+++JJ~JJxII()))HHJJJJJr%rOc  ddlddl}ddlddlmcmcm}ddlmcmcm}ddl m m m tjd|Gfddj}||}t#t$j|dd 5}||dddn #1swxYwYd}|d } g} t1|D]:} || } | } t7| d |t7| d |g}t=t?|}| | }| |}|!| |"| ||#| ||$| }| %|<|&| tO| | D]}| (|| )}|*| |+| ||,| }| -|| .}t#t$j|d d5}||dddn #1swxYwYt$j/dddvr|j0|d}|1}te|D]c}|3|}tj|4tj|5dt#t$j|dd 5}t1|D]} || } | } t7| d |t7| d |g}| dzt=t?|z}|||d ddddS#1swxYwYdS)z> Helper function to write calibration table to files. rN)CalibrationMethod TensorData TensorsDatazcalibration cache: c"eZdZfdZdS)*write_calibration_table..MyEncoderc\t|fr|St|jr*|t |jddSt|r|jjt |dSj ||S)Nz numpy.array)rrlCLS)rr) rto_dictrtolistr rl __class__r7 JSONEncoderdefault)r.objrrrjsonnps r#rz2write_calibration_table..MyEncoder.default2s# K899 %{{}}$#rz** ] # s39~~m\\\#011 J"}5CIII#++D#66 6r%N)r7r8r9r)rrrrrsr# MyEncoderr1sB 7 7 7 7 7 7 7 7 7 7 7r%r)clszcalibration.jsonwihighestlowestzcalibration.flatbufferswbQUANTIZATION_DEBUG)r(1zcalibration.cache  )6r flatbuffersry5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTable"onnxruntime.quantization.calibraterrrlogginginfordumpsopenospathjoinwriter~Buildersortedkeysrfloatr!rbr r CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndr}TrtTableStartDictVectorrPrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrrrr json_datafilerbuilderkey_value_listkeyrd_valuesrYrflat_key flat_value key_value main_dict cal_tablebufdict_lenrrrrrrs @@@@@r#write_calibration_tabler!s! KKKLLLLLLLLLLLLLLLLLLLLLLLL]]]]]]]]]] L:'8::;;;77777777777D$777 ,) <>## (,,y$//4466 7 7 (,,x..3355 6 6 CKK  '',,))%00 w'''222!!':666((11 i(((( $$Wc..A.ABBB#33 '' 2222!!##I 7### Wi000$$W--I NN9 ..  C bgll3 9::D A AT 3 z~~*A..(::%77Q?? ''))x , ,A!q))I L ) ) ) L** + + + + bgll3 344c : : d+002233  C&s+F~~''Hhll9d3388::;;hll8T227799::F#ICKK 0 00E JJu    JJt                       s7)C  CCM))M-0M-&C7U++U/2U/-C6?c|dktj}|dktj}|}|j|z }|sdS|t |zt |z }|dksJd|||fz|tj}|||z| |zzz }|dkdksJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrz"n_zeros=%d, n_nonzeros=%d, eps1=%f)rryrsumsizer)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rs r#smooth_distributionr}sQu}--H6//%-00KllnnG'!J t w % "3"3 3D #:::; ?::: 88EM " "DC(Nte{2 22D AI??   ! ! ! ! Kr% model_pathctj|d}|jjD]}t j|rdSdS)NF)load_external_dataT)rloadas_posixgraphr8ruses_external_data)rmodel intializers r#model_has_external_datarsZ Ij))++ F F FEk-  2: > > 44  5r%opt_model_pathct}||_tj|_i}dg|d<t ||fddgi|}dS)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)rr sess_optionkwargs_s r#optimize_modelrsr!""K+9+B+B+D+DK(+A+RK( F%6$7F !,,.. jjH^G_jcijjAAAr%rrcddi}|jr+|jD]#}||j|ji$tj||dS)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrrrset_model_props)rrprops r#add_pre_process_metadatarsh.0CDN :( : :D  ! !48TZ"8 9 9 9 9K~66666r%cZ|jr#|jD]}|jdkr|jdkrdSdS)zCCheck the model whether it went through quantization pre-processingrrTFrrr)rrs r#model_has_pre_process_metadatar sG (  Dx333 FY8Y8Ytt 5r%cddi}|jr+|jD]#}||j|ji$tj||dS)N onnx.inferrr)rrrs r#add_infer_metadatarsh"$78N 4% 4 4A  ! !15!'"2 3 3 3 3K~66666r%cZ|jr#|jD]}|jdkr|jdkrdSdS)Nr rTFr )rrs r#model_has_infer_metadatarsF %  Au $$4G)G)Gtt 5r%c,t|d}tjt |t |tj|}t|||S)Nz -inferred) rxrshape_inferenceinfer_shapes_pathr rrrunlink)rinferred_model_pathrs r#load_model_with_shape_inferrs{6z;OO**3z??C@SrErarQrlrSrWrUrYr]r[rr~rhrgrjrir#r"r rrrrrrrrr&r,r r6rArOr_rfrkrprxrrrrrrr rrrrrr&r)r,r/r2r5rr%r#rEs@  #"""""  >>>>>>>>>>&&&&&&QQQQQQQQQQQQ------PPPPPPPPPP@@@@@@@LLL ????????? D EEE7777777      ,$2'!qqCC 4D4Dqqq      t           #>#>#>#>#>#>#>#>L     $    V!4!4 +%+g"6"6 +%+g"6"6!;5;x#8#8' % ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+d%*"E"E"E{u{SV^c^hGiGiGi!j!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ;5;q#>#>#>  BV[@\@\@\"]+%+b"="="={u{1TX?Y?Y?Y!Z  ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+d%*"E"E"E{u{SV^c^hGiGiGi!j!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ! ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+c"D"D"DkekRT\a\fFgFgFg!h!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ;5;q#=#=#={u{1TX?Y?Y?Y"Z+%+b"="="={u{1TX?Y?Y?Y!Z )+ A A A A A 13131313h<<<<~"""R#'"&"&;G;G;G;G;G~hl:=:=:=:=:=D$( L L L L L ^@    <>%%%%%%%%8''''''''"#"#"#J000@@@RRRR$YYYYx: k k k k 777777777777++++,,,,2222----3333//////s6AA%$A%)A22 A>=A>B BB