Ng ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZddlZddlZddlmZmZmZmZddlZddlmZmZmZdejd ejd ejfd Z d-dejd ejd e ed ed ejf dZGddZ GddZ!GddeZ"Gddej#Z$GddZ%Gdde%Z&Gdde%Z'Gdde'Z(Gd d!e'Z)Gd"d#e'Z*Gd$d%ej#Z+Gd&d'e+Z,dd(e"j-d)ifd*ee.efd+e e e.fd,Z/dS).N)Enum)Path)DictOptionalSequenceTupleUnion) ModelProto TensorProtohelper numpy_helper) apply_plotload_model_with_shape_infersmooth_distributionpkqkreturnctj|j|j}|ddtj|dd|ddz z|dd<|dk|dkz}d||<|dk|dkz}tj||<|S)z See https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.rel_entr.html#scipy.special.rel_entr. Python implementation. dtypeNr)npemptyshaperloginf)rrresc2c1s ^/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/quantization/calibrate.pyrel_entrr!s (2828 , , ,C URVBqqqEBqqqEM** *CF 'bAg BCG q&R!V BvCH JbaseaxiscX||dks Jd| Jdtj|tj}d|ztj||dz }tj|tj}tj||\}}d|ztj||dz }t ||}tj||}||tj|z}||jS) z Simplifeied version of entropy. Source: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.entropy.html. This avoids taking a dependency on scipy just for this function. Nrz0base={base} must be a positive number or `None`.z qk is None?T)r$keepdimsr$) rasarrayastypefloat32sumbroadcast_arraysr!rr)rrr#r$vecss r entropyr0's <4!888%W888 >><>>> B  rz * *B rBF2D4888 8B B  rz * *B  R ( (FB rBF2D4888 8B 2r  C sA  RVD\\ 88BH  r"czeZdZegdZegdZdZedZedZ dZ dS) TensorData)avgstdlowesthighesthist hist_edgesbins)r3r4r5r6r8c t||_|D]\}}|tjvr t d|dtjd|tjvrkt|ds"t dt|d||j tj tj fvrt d|j d|t|||dS)NzUnexpected value z not in .rUnexpected type z for k=zUnexpected dtype )listkeys_attrsitemsr2_allowed ValueError_floatshasattrtyperrfloat16r+setattr)selfkwargskvs r __init__zTensorData.__init__Gs6;;==)) LLNN  DAq +++ !XQ!X!X*BU!X!X!XYYYJ&&&q'**O$%MQ%M%M%M%MNNN72:rz":::$%N%N%N%N%NOOO D!Q      r"ct|drt|ds tdt|d|j|jfS)Nr5r6z0Attributes 'lowest' and/or 'highest' missing in r;)rDAttributeErrordirr5r6rHs r range_valuezTensorData.range_valueSsZtX&& bgdI.F.F b !`TWX\T]T]!`!`!`aa a T\**r"ct|drt|ds tdt|d|j|jfS)Nr3r4z)Attributes 'avg' and/or 'std' missing in r;)rDrNrOr3r4rPs r avg_stdzTensorData.avg_stdYsWtU## [74+?+? [ !YSQUYY!Y!Y!YZZ Z$(##r"cLfdjD}jj|d<|S)Nc2i|]}|t|S)getattr).0rJrHs r z&TensorData.to_dict..as%99974##999r"CLS)r? __class____name__rHdatas` r to_dictzTensorData.to_dict_s19999T[999n-U  r"N) r\ __module__ __qualname__ frozensetrArCrLpropertyrQrSr_rVr"r r2r2CsyZZZ[[HiIIIJJG    ++X+ $$X$ r"r2cjeZdZdeeeeefffdZdZ dZ dZ dZ dZ dZd Zd Zd S) TensorsDatar^c ||_i|_|D]F\}}t|ts t dt |dt|tr|tj kr9t|dkr&t|d|d|j|<t|dkr4t|d|d|d|d |j|<t d |d d t|d |dt|ts t dt |d||j|<HdS)NzKeys must be strings not r;rrr5r6)r5r6r7r9zUnexpected tuple for rz , it has z elements: zValues must be TensorData not ) calibration_methodr^r@ isinstancestr TypeErrorrEtupleCalibrationMethodMinMaxlenr2)rHrlr^rJrKs r rLzTensorsData.__init__gss"4 JJLL  DAqa%% H FDGG F F FGGG!U## _%):)AAAc!ffPQkk#-QqT1Q4#H#H#HDIaLq66Q;;#-QqT1Q4aPQdYZ[\Y]#^#^#^DIaL ] ] ] ]c!ff ] ]YZ ] ] ]^^^a,, M Ka K K KLLLDIaLL  r"c#$K|jEd{VdSNr^rPs r __iter__zTensorsData.__iter__ys&9r"c||jvSrurvrHkeys r __contains__zTensorsData.__contains__|sdir"c|j|Srurvrys r __getitem__zTensorsData.__getitem__sy~r"cR||jvrtd|d||j|<dS)Nz)Only an existing tensor can be modified, z is not.)r^ RuntimeError)rHrzvalues r __setitem__zTensorsData.__setitem__s8 di  Z3ZZZ[[ [ #r"c4|jSru)r^r>rPs r r>zTensorsData.keyssy~~r"c4|jSru)r^valuesrPs r rzTensorsData.valuessy!!!r"c4|jSru)r^r@rPs r r@zTensorsData.itemssy   r"c:|jj|j|jd}|S)N)rZr^rl)r[r\r^rlr]s r r_zTensorsData.to_dicts*>*I"&"9    r"N)r\r`rarrnr r2rrLrwr{r}rr>rr@r_rVr"r rerefsc5UAR;S6S1T$       """!!!r"receZdZdZdZdZdZdS)rqrrrgrjN)r\r`rarrEntropy Percentile DistributionrVr"r rqrqs" FGJLLLr"rqcleZdZedZejdefdZdZ dZ dZ de de fd Z d S) CalibrationDataReadercXt|drt|jptS)Nget_next)rDcallablerNotImplemented)clssubclasss r __subclasshook__z&CalibrationDataReader.__subclasshook__s(x,,L(:K1L1L^P^^r"rct)z9generate the input data dict for ONNXinferenceSession runNotImplementedErrorrPs r rzCalibrationDataReader.get_nexts "!r"c|SrurVrPs r rwzCalibrationDataReader.__iter__s r"c@|}|t|Sru)r StopIteration)rHresults r __next__zCalibrationDataReader.__next__s >  r"ctrurrPs r __len__zCalibrationDataReader.__len__!!r" start_index end_indexctrur)rHrrs r set_rangezCalibrationDataReader.set_rangerr"N)r\r`ra classmethodrabcabstractmethoddictrrwrrintrrVr"r rrs__[_ "$"""" """"S"S""""""r"r) metaclassceZdZ ddeeefdeeefdZdgfdZ d Z d e fd Z d Z d ZdefdZdefdZdS)CalibraterBaseNaugmented_model.onnxF model_pathop_types_to_calibratecXt|tr"tt||_n9t|trt||_nt d||_||_||_||_ ||_ d|_ d|_ dg|_ dS)a :param model_path: ONNX model to calibrate. It should be a model file path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param symmetric: make range of tensor symmetric (central point is 0). :param use_external_data_format: use external data format to store model which size is >= 2Gb. :param per_channel: whether to compute ranges per each channel. z model_path should be model path.NCPUExecutionProvider)rmrnrrmodelrBraugmented_model_path symmetricuse_external_data_format per_channel augment_model infer_sessionexecution_providers)rHrrrrrrs r rLzCalibraterBase.__init__s" j# & & A4T*5E5EFFDJJ  D ) ) A4Z@@DJJ?@@ @%:"$8!"(@%&!!$:#;   r"rc<||_|dS)zz reset the execution providers to execute the collect_data. It triggers to re-creating inference session. N)rcreate_inference_session)rHrs r set_execution_providersz&CalibraterBase.set_execution_providerss$$7  %%'''''r"ctj}tjj|_tj|j||j|_dS)z9 create an OnnxRuntime InferenceSession. ) sess_options providersN) onnxruntimeSessionOptionsGraphOptimizationLevelORT_DISABLE_ALLgraph_optimization_levelInferenceSessionrrr)rHrs r rz'CalibraterBase.create_inference_sessionsO#133 0;0R0b -(9  %%.   r"rcd|jjD}|d|jjD|d|jjDd|jjD}t }tjtj h}|jj D]}|j r|j |j vrttj|j|jD]T}||vrN||}|jdr,|jjj|vr||vr||U||fS)z select input/output tensors of candidate nodes to calibrate. returns: tensors (set): set of tensor name. value_infos (dict): tensor name to value info. ci|] }|j| SrVnamerXvis r rYz>CalibraterBase.select_tensors_to_calibrate..sDDDrrwDDDr"ci|] }|j| SrVr)rXots r rYz>CalibraterBase.select_tensors_to_calibrate..sEEEBBGREEEr"ci|] }|j| SrVr)rXits r rYz>CalibraterBase.select_tensors_to_calibrate..sDDDBBGRDDDr"ch|] }|j SrVr)rXinits r z=CalibraterBase.select_tensors_to_calibrate..sEEETtyEEEr" tensor_type)graph value_infoupdateoutputinput initializersetr FLOATFLOAT16noderop_type itertoolschainrEHasFieldr elem_typeadd) rHr value_infosrtensors_to_calibratetensor_type_to_calibrater tensor_namers r select_tensors_to_calibratez*CalibraterBase.select_tensors_to_calibrates`EDU[-CDDD EE%+2DEEEFFFDD%+2CDDDEEEEEU[-DEEE "uu$/$5{7J#K K$ B BD- BA[1[1[#,?4:t{#K#KBBK"k11(5G,,];;B!#!4!>BZ!Z!Z!,K!?!?044[AAA#[00r"c|jS)zP return: augmented onnx model. Call after calling augment_graph )rrPs r get_augment_modelz CalibraterBase.get_augment_models zr"ct)z abstract method: augment the input model to prepare for collecting data. It will: 1. augment the model to be able to collect desired statistics data 2. save augmented model to augmented_model_paths rrPs r augment_graphzCalibraterBase.augment_graphs "!r" data_readerct)z abstract method: collect the tensors that will be used for range computation. It can be called multiple times. r)rHrs r collect_datazCalibraterBase.collect_data "!r"rct)ze abstract method: compute data based on the calibration method stored in TensorsData rrPs r compute_datazCalibraterBase.compute_data"rr")NrFFF)r\r`rar rnrrrrLrrr rrrrrrerrVr"r rrs:>3!& < <#t)$ < ( 6 < < < = 2Gb :param moving_average: compute the moving average of the minimum and maximum values instead of the global minimum and maximum. :param averaging_constant: constant smoothing factor to use when computing the moving average. :param max_intermediate_outputs: maximum number of intermediate outputs before an intermediate range is computed. :param per_channel: whether to compute ranges per each channel. )rrrrrNch|] }|j SrVrrXrs r rz,MinMaxCalibrater.__init__..L&Y&Y&Yvv{&Y&Y&Yr"rrz;Invalid averaging constant, which should not be < 0 or > 1.)superrLintermediate_outputscalibrate_tensors_rangersrrrnum_model_outputsmodel_original_outputsmoving_averagerBaveraging_constantmax_intermediate_outputs) rHrrrrrrrrrr[s r rLzMinMaxCalibrater.__init__*s.  "7!5%=#    %'!'+$!$TZ%5%.get_op_version^s] % 2 0 0 9==,*=>>0'////0SSSSTT Tr"cd}|dz|z}|dz}tj||g|g||}tjd|g|g|}djjjD}|djjjD|d jjjD||vr||j j j }ntd |d j rt||j j jj} d gt#d | }  |jdkr.|jtjd| nt+t-j} t1jt5j| t4j| } |j| jjj| jjj||gjjjtj ||dgdS)Nr__Reshape)r'rReshape)inputsoutputsrci|] }|j| SrVrrs r rYzNMinMaxCalibrater.augment_graph..add_reduce_min_max..wsMMM227BMMMr"ci|] }|j| SrVr)rXos r rYzNMinMaxCalibrater.augment_graph..add_reduce_min_max..xsKKKaKKKr"ci|] }|j| SrVr)rXis r rYzNMinMaxCalibrater.augment_graph..add_reduce_min_max..ysJJJaJJJr"z'Unable to guess tensor type for tensor zE, running shape inference before quantization may resolve this issue.rrgaxesr)!rr make_noderrrrrrrErrrBrrsrdimrange attributeappendmake_attributernuuiduuid4r from_arrayrarrayint64rrextendmake_tensor_value_info)rreduce_op_namer' reduce_outputintermediate_output reduce_node reshape_noder onnx_type tensor_rank reduced_axesreduce_axes_name reduce_axesr reshape_shape_namerHs r add_reduce_min_maxz:MinMaxCalibrater.augment_graph..add_reduce_min_maxdsH(#->M"/*"< +// 0C/Dx^k0K ;00+-?@&( 1LNM1A1LMMMK   KK4:3C3JKKK L L L   JJ4:3C3IJJJ K K Kk))' 49EO  [k[[[  E!+k":"?"K"Q"UVV !:E![$9$9: !>.$*==BB)001Fv|1\1\]]]]'*4:<<'8'8$"."9"(>>J$077 DDD J  ! ( (+|)D E E E J  # * *6+HXadhci+j+j k k k k kr" ReduceMin ReduceMaxsave_as_external_dataN)rrrnrrr rrr r!rrrrsaverr)rHtensorsr  reshape_shaper/tensorr r.s` @@r rzMinMaxCalibrater.augment_graphSs! 55djAA  ..$/"RX0N0N0NPbcc  $++M::: U U U , l, l, l, l, l, l, l\ 4 4F  v{ 3 3 3  v{ 3 3 3 3 J  %"&"?      r"cg|_dSrurrPs r clear_collected_dataz%MinMaxCalibrater.clear_collected_data$&!!!r"rc$ |}|snl|j|jd||j1t |j|jkr|t |jdkr|jtd| }t|ts tdt|d|dS)NTrNo data is collected.z+compute_data must return a TensorsData not r;)rrrrrunrrsr:rrBrrmrerorE)rHrrts r rzMinMaxCalibrater.collect_datas , ))++F   % , ,T-?-C-CD&-Q-Q R R R-9122d6SSS))+++ , t( ) )Q . .43O3W455 5     ![)) VT$q''TTTUU U !!#####r"ct|s|S|D]\}}t|tr|jd}|jd}n|\}}t||tr'||jd}||jd}n ||\}}|jr!||j||z zz} ||j||z zz} n t ||} t||} t|tst||trt| | ||<| | f||<|S)Nrrrh)r@rmr2rQrrminmax) rH old_range new_rangerzrold_minold_maxnew_minnew_max min_value max_values r merge_rangezMinMaxCalibrater.merge_ranges]  #//++ 8 8JC%,, )+A.+A.#( )C.*55 2#C.4Q7#C.4Q7#,S> " 2#d&=7AR&SS #d&=7AR&SS 11 11 %,, 8 9S>:0V0V 8!+9i!P!P!P #"+Y!7 #r"rc ` tjdkrjSfdttjdDfdjD}i|D]E}|D].\}}|g|/Fjd fdtdt dD}fdD}g}tdt dD]'}jrHtj | |d} tj | |d zd} nGtj | |d} tj | |d zd} j rctj tj| tj| gd} |t| | g|t| | g)t!t"jt't)||} jr!j| _n| _jS) z Compute the min-max range of tensor :return: dictionary mapping: {added node names: (ReduceMin, ReduceMax) pairs } rcXg|]&}j|j'SrV)r get_outputsr)rXrrHs r z1MinMaxCalibrater.compute_data..s0sssQ*6688;@sssr"cJg|]}tt| SrVrziprXr& output_namess r rOz1MinMaxCalibrater.compute_data..9   =PD\#677 8 8   r"NcRg|]#}|dd$S)r r) rpartition)rXradded_output_namess r rOz1MinMaxCalibrater.compute_data..s?" " " 9: q ! , ,S 1 1! 4" " " r"rgc4i|]}|jv ||SrV)r)rXrmerged_output_dictrHs r rYz1MinMaxCalibrater.compute_data..s5$ $ $ )*ATMhDhDhA!!$DhDhDhr"r(r)rsrrrr@ setdefaultrrrrmeanrArBrabsrprerqrrrrRrK)rHoutput_dicts_listdrJrKcalibrate_tensor_namesmerged_added_output_dictpairsrmin_value_arraymax_value_arraymax_absolute_valuenew_calibrate_tensors_rangerXrZrTs` @@@r rzMinMaxCalibrater.compute_datas t( ) )Q . ./ /ssss%PSTXTmnoTpPqPqJrJrsss     TXTm    " ? ?A  ? ?1"--a44;;A>>>> ?)$*@*B*BC" " " " >CAsK]G^G^`a>b>b" " " $ $ $ $ $ .@$ $ $  q#011155 H HA" f"$'*BCUVWCX*Y`a"b"b"b"$'*BCUVWZ[V[C\*]de"f"f"f"$&)ABTUVBW)X_`"a"a"a"$&)ABTUVYZUZB[)\cd"e"e"e~ H%'VRVO-D-Dbf_F]F],^ef%g%g%g" U%7$79K#LMMNNNN UO_#EFFGGGG&12C2JDQTUkmrQsQsLtLt&u&u#  ' G+/+;+;D3!&!%'A'A#t)$'A ( 6'A'A'A'A'A'ARG G G R'''$(=$$$$(B0,k0,0,0,0,0,0,0,0,r"rceZdZ dd eeefd eeeffd Zd Z d Z de fdZ de fdZxZS)HistogramCalibraterNrF percentile-X@samerrc Xt|||||g|_d|_t |jjj|_d|jjjD|_ d|_ ||_ ||_ ||_ | |_d|_| |_dS)a= :param model_path: ONNX model to calibrate. It is a model path. :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. :param scenario: see :class:`DistributionCalibrater` )rrrrNch|] }|j SrVrrs r rz/HistogramCalibrater.__init__..+rr")rrLrrrsrrrrr collectormethodnum_binsnum_quantized_binsrkrscenario) rHrrrrrsrrtrurkrvr[s r rLzHistogramCalibrater.__init__s2  "7!5%=    %'!'+$!$TZ%5%.KsYYYX8=YYYr"cg|] }|j SrVrr{s r rOz4HistogramCalibrater.collect_data..LsWWW( WWWr"TNrr=cJg|]}tt| SrVrQrSs r rOz4HistogramCalibrater.collect_data..brUr"c4i|]}|jv ||SrV)r)rXr merged_dictrHs r rYz4HistogramCalibrater.collect_data..ks,fff1qDLeGeGeQ AGeGeGer")rsrrtrurkrv)r get_inputsrNrr> enumeratercopyrrsrBr@r[rrHistogramCollectorrsrrtrurkrvcollectr:)rHrinput_names_setrr fixed_outputs output_indexrr^r_rJrKclean_merged_dictrrTs` @@r rz HistogramCalibrater.collect_dataGsLZY9K9V9V9X9XYYYWWd6H6T6T6V6VWWW  < ))++F (,,T6::GM(1'(:(: 1 1$ f -@@!((6):):;;;;!((0000  % , ,] ; ; ; <" t( ) )Q . .455 5    TXTm    " 8 8A  8 81&&q"--44Q7777 8gffff fff~ /{.#'#:? DN 0111 !!#####r"rc|jstdt|tr tj}ndt|t r tj}nBt|tr tj }n tdt|dt||j S)z Compute the min-max range of tensor :return: dictionary mapping: {tensor name: (min value, max value)} z9No collector created and can't generate calibration data.zUnknown calibrater z". This method must be overwritten.)rrrBrmEntropyCalibraterrqrPercentileCalibraterrDistributionCalibraterrrorErecompute_collection_result)rHcals r rz HistogramCalibrater.compute_datazs ~ ZXYY Y d- . . b#+CC 2 3 3 b#.CC 4 5 5 b#0CC`$t**```aa a3 H H J JKKKr") NrFrkFrlrmrnro)r\r`rar rnrrrrLrr:rrrerrgrhs@r rjrjs:>3!&*!*!#t)$*! ( 6*!*!*!*!*!*!X   '''1$(=1$1$1$1$fLkLLLLLLLLr"rjc`eZdZ d deeefdeeeffd ZxZ S) rNrFr0rlrrc Zt||||||||dS)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. )rsrrtruNrrL) rHrrrrrsrrtrur[s r rLzEntropyCalibrater.__init__sH*   ! $1  r")NrFr0Frlrl r\r`rar rnrrrrLrgrhs@r rrs}:>3!&  #t)$  ( 6          r"rc`eZdZ d deeefdeeeffd ZxZ S) rNrFrkrmrnrrc Zt||||||||dS)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. )rsrrtrkNr) rHrrrrrsrrtrkr[s r rLzPercentileCalibrater.__init__sH*   ! $!  r")NrFrkFrmrnrrhs@r rrs}:>3!&  #t)$  ( 6          r"rc^eZdZ d deeefdeeeffd ZxZ S) rNrF distributionrlrorrc Xt|||||||dS)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param scenario: for float 8 only, if `scenario="same"`, the algorithm weights and float 8 follow the same distribution, if `scenario="p3"`, it assumes the weights follow a gaussian law and float 8 ~ X^3 where X is a gaussian law )rsrtrvNr) rHrrrrrsrtrvr[s r rLzDistributionCalibrater.__init__sE.   ! $      r")NrFrrlrorrhs@r rrsz:>3!&  #t)$  ( 6          r"rcReZdZdZejdZejdZdS)CalibrationDataCollectorzL Base class for collecting data for calibration-based quantization. ct)z Generate informative data based on given data. name_to_arr : dict tensor name to NDArray data rrH name_to_arrs r rz CalibrationDataCollector.collects "!r"ct)z? Get the optimal result among collection data. rrPs r rz2CalibrationDataCollector.compute_collection_results "!r"N)r\r`ra__doc__rrrrrVr"r rrs\ """ """""r"rcleZdZdZdZdZdZdZdZdZ dZ d Z d Z e dd Zd ZdZdS)ra` Collecting histogram for each tensor. Percentile and Entropy method are supported. ref: https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py ref: https://docs.nvidia.com/deeplearning/tensorrt/pytorch-quantization-toolkit/docs/_modules/ pytorch_quantization/calib/histogram.html chi|_||_||_||_||_||_||_dSru)histogram_dictrsrrtrurkrv)rHrsrrtrurkrvs r rLzHistogramCollector.__init__s9  "  "4$  r"c|jSru)rrPs r get_histogram_dictz%HistogramCollector.get_histogram_dicts ""r"ctd|jdvr||S|jdkr1|jr||S||St d)Nz/Collecting tensor data and making histogram ...>r0rrkDOnly 'entropy', 'percentile' or 'distribution' methods are supported)printrs collect_valuercollect_absolute_valuerBrs r rzHistogramCollector.collects ?@@@ ;5 5 5%%k22 2 [L ( (~ 722;???))+666cdd dr"c |D]m\}}t|tr|D]9}t|tjsJdt |d|:t d|D}t|dksJd|d|tj|}n>t|tjs"tdt |d||}| }|j dkr)tj |}tj |}n6tjd|j}tjd|j}tj|}||jvrgtj||j\} } | |j} |jtjks Jd | | ||f|j|<|j|} | d } | d } t-| d sJd t | t-| d sJd t | | d}| d}tj |}||dkrI|d|dz }tj|d|z||z|}tj||f}tj||\} } | |j} | dt|xx|z cc<|jtjks Jd | | t| |t| |f|j|<odS)z5 Collect histogram on absolute value r<z for tensor=c3$K|] }|jV dSrur)rXas r z.6s$77QW777777r"rz6The calibration expects only one element type but got rr)r9zMonly float32 or float16 is supported, every constant must be explicitly typedrgrjrz'old_min should be a numpy array but is rN)r@rmr=rndarrayrErrsr)rBflattensizerArBr rabsoluter histogramrtr*float64rDarangehstack)rHrr7data_arrarrdtypes data_arr_nprIrJr7r8 old_histogramrErFold_histold_hist_edges temp_amaxwidth new_bin_edgess r rz)HistogramCollector.collect_absolute_value.s!, 1 1 3 34 s4 s FH(D)) '#mmC%c2:66ll8l4PS998l8lbh8l8lllll77h77777KK1$$$jFjj`fjj%$$ j22 "*55 ' !ZDNN!Z!ZPV!Z!Z[[[& %--//K!##F;// F;// HQk.?@@@ HQk.?@@@ +k22KT000#%< $-#P#P#P j'..{/@AA %333b433/3ZI.V#F++ $ 3F ; '*'*w00kk2k\`ah\i\i2k2kkkkw00kk2k\`ah\i\i2k2kkkk(+!.q!1F;// ~b111*1-q0AAE$&InR.@5.H)V[J[]b$c$cM%'Y /N%O%ON#%< .#Q#Q#Q j'..{/@AA _s8}}_%%%1%%%%333b433/3ZWiAXAXZ]^egpZqZq.r#F++i4 s4 sr"c |D]E\}}tj|}|}|jdkr)tj|}tj|}n6tjd|j}tjd|j}tjt t|t||j}||j vr0|j |}| ||||||j |<tj ||j | |f\}} || |||f|j |<GdS)z1 Collect histogram on real value rrrN)r@rr)rrrArBr rr]rmerge_histogramrrt) rHrr7rrIrJ thresholdrr7r8s r rz HistogramCollector.collect_valuehsc!, 1 1 3 3   FHz(++H''))H}q  F8,, F8,, HQhn=== HQhn=== S^^S^^!D!DHN[[[I,,, $ 3F ; .2.B.B!8Y 9//#F++$&<$-QZPZ\eOf#g#g#g j /#F++)  r"cR|\}}}} } || krPtj|t|| | f\} } | |z|t||t | || fS| dkr0tj|t|| |f\} }| |z } nqt|}d| z|z }t || z |zdz}|d|zz}||z| z}tj||| |f\} }| |||z xx|z cc<| |t||t | ||fS)Nrrrgr)rrrsrArBr)rHrrrGrH new_thresholdrrrErF old_thresholdnew_histr r7r8 old_num_bins old_stridehalf_increased_bins new_num_binss r rz"HistogramCollector.merge_histogramsFSC>7G] M ) ),xX ~WdFefffKHa8#GW%%GW%%  !!#%<#h--Q^P^`mOn#o#o#o j "8}} .= &)==+HZ*WZ[*[&\&\#+a2E.EE 3j @= P #%<,P]~_lNm#n#n#n j(<:M+MMNNNRZZNNNGW%%GW%%  r"cf|jrt|jdkrtdtd|jd|jdkr|S|jdkr|S|jdkr|Std) Nrz=Histogram has not been collected. Please run collect() first.z0Finding optimal threshold for each tensor using z algorithm ...r0rkrr)rrsrBrrscompute_entropycompute_percentilecompute_distributionrPs r rz,HistogramCollector.compute_collection_results" ^c$*=&>&>!&C&C\]] ] ^^^^___ ;) # #'')) ) [L ( (**,, , [N * *,,.. .cdd dr"c|jdks |jdkrtd|j}|j}i}tdt |td|jtdd|z d|d |D]\}}|d}|d }|}tj ||z } |j r_tj | |dz } tj || |j  tj || |j f||<nzd|z d z } tj | d | z } tj | | } tj || |j tj || |j f||<|d} |d}||d| kr| ||d f||<||d |kr||d|f||<g|||ddR||<tjdddvrt#|||S)Nrdze>>>?? ?SbS/JqrrN2c9 A::&=%%''$((**4C619$))++dhhjj836AcIC8Cz'7888"(3jN^:_:_:__ _ u::  3u::>Q#6#6&%-',,..;CFEMC/A55::<RZ////>RZ////#rz1111&0%!~~''"(( '''OF #z~~2A66(BB4,,,r"c |d}|d}|j}|dz}|dz}|djtj||z dz}fdt |jD} t ||dzdD]:} || z } t || zdz|} || || f| | |z <t j|| | } | }t|d| }t|| d}|dxx|z cc<|dxx|z cc<|dk tj }tj|tj }| j|z}t |D]&}||z}||z}t| ||||<'|dxxt| ||zdz cc<tj|jtj }t |D]9}||z}||z}t|||}|dkr|||z |||<:t|}t|}||!tj tj }n$tj t||}||| |z <.Es<nnnqrx///!51I1I1IJnnnr"Nrrrjr)rrrzerosrrArdeepcopyr,r*r!rr rr0argminrD)rHrrur7r8rtzero_bin_indexnum_half_quantized_bin kl_divergence thresholdsrrrsliced_distributionpleft_outliers_countright_outliers_countnonzerosquantized_binsnum_merged_binsindexstartendqnormdivmin_kl_divergence_idxrrIrJrs @r rz(HistogramCollector.get_entropy_threshold7s |q\ 9!Q!3q!8! "2H!H1!LMM nnnnTYZgZlTmTmnnn -~/A1EE. <. rs  99999999999999 >>>>>>>>>>>>UUUUUUUUUU       "!    5/   Z 8        F11111111h"""""ck""""4k"k"k"k"k"k"k"k"\[,[,[,[,[,~[,[,[,|CLCLCLCLCL.CLCLCLL     +   D     .   D      0    F""""""""",E!E!E!E!E!1E!E!E!T 6:/&-" KKKK d KK#HSM2KKKKKKKKKKKKr"