Ng_ddlZddlZddlmZddlZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZmZmZmZmZdd lmZmZmZd ejd <ejeZd ej iZ!d.d Z"dZ#dZ$ej%fdZ&dZ'dZ(dZ)dZ* d/dZ+de,de,de-de.de.de de.de.fdZ/de,d e,d!e,fd"Z0d#Z1 d/d$Z2d%Z3d0d'Z4d(Z5d)Z6d*Z7d+Z8d,Z9d-Z:dS)1N)Path)AffinitySetting) OptimizerInfo Precisioncreate_onnxruntime_session) MODEL_CLASSES)QuantizeHelper)torch_onnx_export) AutoConfigAutoFeatureExtractor AutoTokenizer LxmertConfigTransfoXLConfig)PRETRAINED_GPT2_MODELSGPT2ModelNoPastStateTFGPT2ModelNoPastState2TF_CPP_MIN_LOG_LEVELtriuc|Jt|jdkr,|d|dksJtd}|t jdtj|}|d|dd|df}t j||t j |S)Nrr)rdtype) lenshapesize torch_functorchonesuint8wherebool zeros_like)xdiagonalout torch_triutemplatemasks b/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/transformers/onnx_exporter.py triu_onnxr-#s ;;; qw<<1  affQii!7!7!7!7F#Jz%*\EEExPPH KaffQiiK166!99, -D ;tyy{{Au'7':': ; ;;c(tt_dSN)r-r rr.r,replace_torch_functionsr2-s EJJJr.c4tdt_dS)Nr)rr rr1r.r,restore_torch_functionsr41sF#EJJJr.ch|jdvrOtj|d|j|jtj}d|i}|Stjd|dz ||f|}d|i}d|vrtj||g| }||d<d |vrtj ||g| } | |d <|j r||d <t|trtj dd|jtj|d <tj dd|jtj|d <t|t r)tj |jgtj |d<|S)Nvitswin pixel_valuesrrlowhighrr input_idsattention_maskrtoken_type_idsdecoder_input_ids visual_feats visual_posz@tf_transfo_xl_model/transformer/pos_emb/einsum/Einsum/inputs_1:0) model_typenumpyrandomrand image_sizeastypefloat32randintr!zerosis_encoder_decoder isinstancerrandnvisual_feat_dimvisual_pos_dimr hidden_size) vocab_size batch_sizesequence_length input_namesconfig data_typer>inputsr? segment_idss r,create_onnxruntime_inputr[5s O++L%%j!V5FHYZZaabgbopp  ),  $$azSbFckt$uuI9 %F;&&Z$ASSS#1 ;&&k:"?yQQQ #.   0&/"#&,''e!&!3!3Aq&:P!Q!Q!X!XY^Yf!g!g~$|11!Q8MNNUUV[Vcdd|&/** UZU`    V V V QR Mr.c2i}|D]}||vr ||||<|Sr0r1)rYrVremaining_model_inputs input_names r, filter_inputsr_Ss;!DD   17 1C ": . !!r.cXt|ttfr d|Dn|gS)Nc,g|]}t|Sr1)flatten.0is r, zflatten..\s ( ( (AWQZZ ( ( (r.)rNlisttuple)rYs r,rbrb[s3,6ve},M,M Y ( ( ( ( ( (SY ZZr.c|D]C}t|ttfs||nt ||D|Sr0)rNrgrhappendupdate_flatten_list)rYres_listres r,rkrk_sN ee",Qu ">">dDWXY[cDdDd Or.cV|djd}d|D}dtt|D}t|D]Q\}}ddi||<||j}t|D](\}} | |kr|||di)R||fS)Nr>ci|]}|ddd S)rTseq_len)rrr1rdkeys r, z&build_dynamic_axes..hs"SSSSC\i88SSSr.c8g|]}dt|dzzS)output_r)strrcs r,rfz&build_dynamic_axes..js'PPPqIAE *PPPr.rrTrp)rranger enumerateupdate) example_inputsoutputs_flattenrU dynamic_axes output_namesre output_namedimsjdims r,build_dynamic_axesres$[17;OSSNSSSLPPE#o:N:N4O4OPPPL#L11AA;%& $5 [!q!'oo A AFAso%%[)00!Y@@@ A  %%r.c 0t||d}|t|ddSt|dd|D}|||}t |t |kri|]\}}||Sr1)rE)rdkts r,rsz'validate_onnx_model..s&JJJ41a!QWWYYJJJr.z"Number of output tensors expected z, got g-C6?zMax absolute diff=z for output tensor g?g?)rtolatolzOutput tensor z is not close: rtol=z, atol=z0inference result of onnxruntime is validated on T) rloggererrorinfoitemsrunrrwrEamaxabscpuallclose) onnx_model_pathrzexample_outputs_flattenuse_gpufp16r} test_sessionexample_ort_inputsexample_ort_outputsreabs_diffrrs r,validate_onnx_modelrtsA.ow`efffL BBBCCCu KK?:::;;;KJ>3G3G3I3IJJJ&**<9KLL "##s+>'?'??? o5L1M1M o oUXYlUmUm o o   u 3.// 0 0:ei(;A(>AXYZA[A_A_AaAaAgAgAiAi(ijjkk d?? KKMXMM!MM N N N&uu$&uu$~  " #A & * * , , 2 2 4 4      LLT!TTTTdTT U U U55  KKT?TTUUU 4r.onnx_dir model_name input_countoptimized_by_scriptr precisionoptimized_by_onnxruntimeuse_external_datachddlm}|dd|} |s| d|} n|rdnd} | d|d|d| } |r| dz } |} |rU|sStj|| } tj| stj| tj| | dS) Nr)subz [^a-zA-Z0-9_]_gpur_ortz.onnx)rerospathjoinexistsmakedirs) rrrrrrrrrnormalized_model_namefilenamedevice directorys r,get_onnx_file_pathrsC 0#zBB Q+;;k;;!,u+PPkPPIPPPPFI#!9#GLL844 w~~i(( # K " " " 7<< h#5#5#5 6 66r. file_pathsuffixreturnct|}t|j|j|z|jS)a Append a suffix at the filename (before the extension). Args: path: pathlib.Path The actual path object we would like to add a suffix suffix: The suffix to add Returns: path with suffix appended at the end of the filename and before extension )rrvparentjoinpathstem with_suffixr)rrrs r,add_filename_suffixrsD  ??D t{##DI$677CCDKPP Q QQr.c"|stj|sPt|jddddlm}m}||||d}||||<dSt d|dS)NTparentsexist_okr)get_fusion_statisticsoptimize_by_onnxruntimec)roptimized_model_path opt_level'Skip optimization since model existed: ) rrrrrmkdir optimizerrrrr)rort_model_pathr overwritemodel_fusion_statisticsrrrs r,optimize_onnx_model_by_ortrs P~66 P ^#))$)FFFLLLLLLLL $ # !/     3H2G2W2W/// NnNNOOOOOr.c |s tj|st|jddddlm} ddlm } | | |} | ||tj krd| _ |tjkrd| _|dkrd}d}| ||||d| |d}|d ks|d kr||| |<|tj kr|d ||| dSt(d |dS) NTrr) FusionOptions)optimize_modelFr8) num_headsrRroptimization_optionsronly_onnxruntime bert_kerasbert_tf) keep_io_typesr)rrrrrrfusion_optionsrrruse_raw_attention_maskrFLOAT16enable_gelu_approximationINT8enable_embed_layer_normuse_dynamic_axesget_fused_operator_statisticsconvert_float_to_float16save_model_to_filerr)rrrDnum_attention_headsrRrrrrruse_external_data_formatrrr opt_models r,optimize_onnx_modelrs*V';<<*V !"")//t/LLL000000,,,,,,  '#0=#<#< 334JKKK  ) ) )=A :   & &;@ 8   "# K #N  )#!5"      % %y)@)@  & & ( ( (8A8_8_8a8a 45  ) ) )  . .T . B B B$$%9;STTTTT T>RTTUUUUUr.c"|5|tvr|Stddtz|tvrdSddl}|d|dS|d|dS|d |d Sd S) NzValid model class:  rrz-squad$AutoModelForQuestionAnsweringz-mprc$"AutoModelForSequenceClassificationgpt2AutoModelWithLMHead AutoModel)r Exceptionrrrsearch)rcustom_model_classrs r,modelclass_dispatcherrs%  . .% %1CHH]4K4KKLL L+++%% III yyJ''3.. 8Z ( ( 433 6: & & 2$$ ;r.FcJt||}|dkr0|rtj|||Stj|||S|rd|z}t d|g}t d|t||}||||S)Nr)rW cache_dirTF transformers)fromlistzModel class name: )rrfrom_pretrainedr __import__rrgetattr)rrWrr is_tf_modelmodel_class_nametransformers_module model_classs r,load_pretrained_modelr/s,Z9KLL111  h)9*V_hiii i'7 6]fggg g3"22$^?O>PQQQ KK7%577888-/?@@K  & &z&I & V VVr.ctj||}t|drd|_||t ||||}||fS)Nr return_dictF)rWrr)r rhasattrrmodifyr)rrrconfig_modifierrWmodels r, load_pt_modelrBsf  ' i H H HFv}%%#"6""" !*Vyep q q qE 5=r.ctj||}||t}|t ||||d}|||fS)NrT)rWrrr)r rrr get_affinityr set_affinity)rrrrrWaffinity_settingrs r, load_tf_modelrNs  ' i H H HF6"""'((!!### !&    E!!### 5=r.c2ddlm}||\}}||fS)Nr)tf2pt_pipeline)convert_tf_models_to_pytorchr )rr rWrs r,load_pt_model_from_tfr cs3<;;;;;"N:..MFE 5=r.cd}|rt| |||d|}|tjkr | || jfS|tjks |t jks|t jkrt||t|d||d|}t| ||| j | j ||| | | || |} |r"t| ||||t jk|}|t jkrPtd| tj| | |td| |tjkr%|r#t%| d}t'| ||| | | ||dvr| jn| jfS)NTFzQuantizing model: zFinished quantizing model: rr6)rrNOOPTrSBYSCRIPTrrrrrrrrRrrr quantize_onnx_modelBYORTrr num_labels)rrrDrrVrr optimize_info validate_onnxrrrWrrrzrr}ris_valid_onnx_modelrrs r,validate_and_optimize_onnxrms( 1   #       +++ 3V5FFF ///9 @Q3Q3QU^bkbpUpUp1          $      &     "  # $  /  "5'Y.. ##    & & KK>_>> ? ? ?  .Qi j j j KKGoGG H H H +++  0&IIN &'     '?::@Q r.ct||||\}}|d}d}|dvrtj||}tjdd|j|jzdztj |j|jd}||d}nHtj||}|j |d }| d d}t||}|di|}t|t t"fsJd t%|t'|}t)|g}t+||t-|d | | d |}|st.j|st4d |t9|jddd}d}|dvrd|Ddg}}nt?||\}}tAtC|t#|"|t!|#||d|| tInt4d|tK|||||| | | | | ||||||d|\}}}||||fS)Nr6rrr9r;pt)return_tensorsrThis is a sample inputz%type of output is not list or tuple: FExporting ONNX model to Trci|]}|ddiS)rr:r1rqs r,rsz-export_onnx_model_from_pt..s)])])]s#>/B)])])]r.logits) rargsfrVr}r|do_constant_folding opset_versionr!Skip export since model existed: r1)&rrr rrErFrKrHr"reshaper max_model_input_sizesget encode_plusr_rNrgrhtyperbrkrrrrrrrrrrrr2r valueskeysr4r) rr!rrDrrrrrVrroptimizer_inforrrrrrWrrzmax_input_sizeimage_processordata tokenizerexample_outputsrrr|r}onnx_model_filerrSs r,export_onnx_model_from_ptr1sF&"*k9oVVMFE IIKKKNN_$$.>zU^___|##&"3f6G"G!"KSXS^$  '&#V%6 : : )dCCC!1* RRR "8<;??Ne--n--O oe} 5 5vv7v_cds_t_t7v7vvvv&o6612I2NN( K     OK77K @@@AAA _$**4$*GGG   ( ()])]n)])])]`h_i,LL);NLc)d)d &L,!!!~,,..//^002233%% $'%=  !!!! IIIJJJ7Q  %884O(** /^ KKr.cddl}|jgdtj||}|j|ddi|j|d}t||||\}}| t|| dd|d d }t||}|jr"| dd|d d j|d <|d krJ|jdd|jg|d<|jdd|jg|d< |jrd|_n#t*$rYnwxYw||d}d}|dks|dkr dg}|d}ddlm}||}t3||t|d| | d|}|r |ddn|}|s t4j|st:d||s)t?|j !d d ddl"}ddl#}|j$%|j$j&g}|'D]Z\} }!dgt|!j(z}"|)|*tW|"|!j,| [|j-.|tW||||\}#}#|r|/|d5}$|$0t4j1|dddn #1swxYwYt4j2t4j1|d}t4j|rt5j3|t5j4||nt:d||dz}tk|||||| | | | | ||||||||\}%}&}'|%|&|'|fS) NrGPUr pad_tokenz[PAD]rrtf max_lengthT)rr6padding truncationrAzunc-nlp/lxmert-base-uncasedrrBrCF)trainingzxlnet-base-casedzxlnet-large-casedlast_hidden_state)nestrr)name)input_signatureopset large_model output_pathrz__MODEL_PROTO.onnxr"_tf)6 tensorflowrWset_visible_devicesr rr4add_special_tokensr$r%rresize_token_embeddingsrr&r_rMr>rFnormalrPrQ use_cachertensorflow.python.utilr;rbrrrrrrrrrzipfiletf2onnxlogging set_levelERRORrrrj TensorSpecrhrconvert from_kerasZipFile extractalldirnamerremoverenamer)(rr!rrDrrrrrVrrr*rrrrrr5r.r+rWrrzr/r}r;rrtf_internal_model_pathrKrLspecsr=valuerrzoptimized_onnx_pathrrSs( r,export_onnx_model_from_tfr];s2(I!!"e,,,-jINNNI"$$k7%;<<<488TJJN!*k9oVVMFE !!#i..111** ! +N#>;??N  .7.C.C $%  /D/ /  *+222)+)9)91aAW:X)Y)Y~&')y'7'7Av?T8U'V'V|$   %$F       eNU;;;OL''':9L+L+L+, )*=>,+++++"ll?;;( K     O6Nb_SbS11SbK'=>>K @@@AAA' S ' ( ( / 5 5dT 5 R R R!!'/"7888)//11 M MKD%6C ,,,D LLuT{{EKdKK L L L L)) !%LL0. *  1 $ ?!7== F RW__-CDDEEE F F F F F F F F F F F F F F F%'W\\"'//BX2Y2Y[o%p%p "w~~o.. + /*** I,o > > >  IIIJJJe#J;U %<<8,j,   s$ E E&%E&)3M((M,/M,)rNr0)F);rMrpathlibrrEr affinity_helperrbenchmark_helperrrrhuggingface_modelsrquantize_helperr torch_onnx_export_helperr rr r r rr0onnxruntime.transformers.models.gpt2.gpt2_helperrrrenviron getLogger__name__rrrr-r2r4int64r[r_rbrkrrrvintr$rrrrrrrrr rr1r]r1r.r,rjs3  ++++++QQQQQQQQQQ,,,,,,******666666gggggggggggggg &) !"  8 $ $ej ! <<<<$$$fkep<"""[[[ & & &* ))))X7777 7  7  7#77777B R3 R R R R R RPPP:8V8V8V8Vv,WWWW&   *\\\~lLlLlL^RRRRRr.