NgcddlmZddlmZmZmZmZmZddlm Z ddl m Z ddl m Z mZmZddlmZeeZGdde ZGd d eZd S) ) getLogger)DictListOptionalTupleUnion)Fusion) FusionUtils) NodeProto TensorProtohelper) OnnxModelcLeZdZdZd"dedeffd Zdedede eefffd Z d ed e ee efd e de fd ZdZdZdZdZdZdede eedefffdZ d#ded ededededefdeef dZdZdZ d$dZdZd Zd!ZxZS)%FusionEmbedLayerNoMaskz Fuse embedding layer into one node (EmbedLayerNormalization). It supports the following model types: BERT, DistilBert, ALBert. no maskmodel descriptionct|dddg|t||_d|_d|_d|_d|_dS)NEmbedLayerNormalizationLayerNormalizationSkipLayerNormalizationF)super__init__r utils shape_infershape_infer_done attention embed_node)selfrr __class__s f/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/transformers/fusion_embedlayer.pyrzFusionEmbedLayerNoMask.__init__sf   % !#; <     !''  %addreturnNc|j|dgdg}|dS|j|dgdg}|dS|d|dfS)NGatherr)rmatch_parent_path)rr# gather_0_path gather_1_paths r!match_two_gatherz'FusionEmbedLayerNoMask.match_two_gather&sg 44S8*qcJJ  4 44S8*qcJJ  4Qq!111r" layernorminput_name_to_nodesis_distil_bertc$|j|d|d|_|jdS|jd|vrdS||jd}t d|D}|gdkrd|D]a}|jd krT|j|gd gd }|2|d jd|jdkr|d |_dSbt|dkr|djdkr|djd|vr||djd}t|dkrr|djdkra|djd|vrL||djd} | D]}|jdkr ||_dSt d| D}|r5|gdkr,|gdkr$|gdkrt ddSn,|gdkr$|gdkrt ddSdS)aCheck that LayerNormalization has a child of Attention node or subgraph like Attention. Args: layernorm (NodeProto): LayerNormalization node input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes is_distil_bert (bool): whether it is DistilBert or not Returns: bool: whether there is Attention node or subgraph like Attention AttentionF) recursiveNTrcg|] }|j Sop_type.0childs r! zCFusionEmbedLayerNoMask.check_attention_subgraph..Ks E E E5 E E Er")MatMulr:r:rr)Addr:MultiHeadAttentionr:)NNrrr'r:r;cg|] }|j Sr3r4r6s r!r9zCFusionEmbedLayerNoMask.check_attention_subgraph..hs(J(J(J5(J(J(Jr")r:r:r:Shaper)r;r:r:r:r@r@)r;r:r:r:r@z %4  A &9 9 95&y'7':; E EH E E EFF UUU U U $ $<#;;; J88III***E (U2Y_Q-?9CSTUCV-V-V/4Qx,#tt x==A  (1+"5"A"AhqkFXYZF[_rFrFr/ 0B10EFMM""a''!!$,55!!$+A.2EEE+M!,<,CA,FG!$$D|{22)-#tt3"((J(JE(J(J(J!K!K  "c"c"ccc"&]&]&]]]"&T&T&TTT [\\\u""" !%%%  [\\\utr"cB|j|ddgddg}|$|j|gdgd}|dS|d|d }}|jd|krdS|j|gd gd fgd gd fg|\}}}|dS|d} |j| ddr|j| ddsdS|d} |j| ddsdS|d } | jd|krdSdS)az Match position embedding path from input_ids to Gather for DistilBert. Pattern is like the following: (input_ids) | Shape | | Gather (indices=1) | | | Cast (optional) | | | Range (start=0, end=*, delta=1) | | | Unsqueeze | / Expand | Gather Expandr@r'N)rQWhereReshaper@)r'r'r>rFrr=) UnsqueezeRangeCastr&r@)rrr'rr)rTrUr&r@)rrr'rr>T)rr(rDmatch_parent_pathsrcheck_node_input_value) rposition_embedding_gather input_idsoutput_name_to_noderLexpandshape_path2 range_node gather_node shape_nodes r!#match_position_embedding_distilbertz:FusionEmbedLayerNoMask.match_position_embedding_distilberts* ,,-FSZH[^_ab]cdd =J00)777 E }ua%) ;q>Y & &5j33 BBBOOOT:::LLLI    5! =51X J - -j!Q ? ? DHJDeDefprsuvDwDw 5Bi  11+q!DD 52Y  A ) + +5tr"cdS)aYMatch position embedding path from input_ids to Gather for Roberta. Roberta Embedding Layer Pattern (* is optional since it might be removed by ORT, ? is the padding word id): (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Mul -- Cast(to=7) -- Add(B=1) -- Cast(to=7)* --> Gather | ^ V | +------------------------------+ Roberta new pattern from transformers v4.9: (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Add(B=0) -- Mul -- Cast(to=7) -- Add(B=1) --> Gather | ^ V | +-------------------------------------------+ start_node = position_embedding_gather start_index = 1 # match optional Cast node. parent = self.model.get_parent(start_node, start_index, output_name_to_node) if parent is None: return if parent.op_type == "Cast": if OnnxModel.get_node_attribute(parent, "to") != 7: return start_node = parent start_index = 0 i, path, return_indices = self.model.match_parent_paths( start_node, [ (['Add', 'Cast', 'Mul', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0]), (['Add', 'Cast', 'Mul', 'Add', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0, 0])], output_name_to_node) if path is not None: # constant input of Add shall be 1. i, value = self.model.get_constant_input(path[0]) if value != 1: return False _, self.padding_word_id = self.model.get_constant_input(path[-1]) return input_ids == path[-1].input[0] Fr3rrZr[r\s r! match_position_embedding_robertaz7FusionEmbedLayerNoMask.match_position_embedding_robertas Zur"c*|j|ddgddg|}|dS|\}}|j|jd}|t |jdkr|jddkro|j|ddgrR|j|ddgr5t |jd ks|j|d dgsdS|j}|d krtj |d dgsdSn|j|ddgsdS|j |d|} | dS| j d kr;|j| ddsdS|j | d|} n| } | | j d krdS|j| ddsdS|j | d|} | | j dkrdS|| jdkS)a  Match position embedding path from input_ids to Gather for BERT. BERT Embedding Layer Pattern: (input_ids) / / Shape / | / Gather (indices=1) / | / Add (optional, B=0) / | Gather (segment_ids) Unsqueeze (axes=0) \ | | \ Gather Slice (data[1,512], starts=0, ends=*, axes=1, steps=1) \ / | Add Gather \ / Add | LayerNormalization SlicerTr'r>NFr axesr;r&r@) rr(get_constant_valuerDrFr^rrYget_opset_versionr check_node_attribute get_parentr5) rrZr[r\pathslice unsqueeze slice_weight opset_versionrKgatherr^s r!match_position_embedding_bertz4FusionEmbedLayerNoMask.match_position_embedding_bertsW,z++ % k " F     <5yz44U[^DD  $L&''1,,"1%** 11%QC@@+ 11%QC@@+U[!!Q&&$**K*KESTWXVY*Z*Z&5 4466 2  3IvsKK u :44YA3GG uz$$Y3FGG <5 <5 :44T1a@@ uZ**44GHHFFF >V^x775 11&!Q?? 5 %%fa1DEE =EMW445EKN**r"cj||||rdS||||rdSdS)NTF)rxrdrfs r!match_position_embeddingz/FusionEmbedLayerNoMask.match_position_embedding:sK  - -.GTg h h 4  3 34MyZm n n 4ur"c |jd}|r |jdnd}|jd}|js'|jd|_d|_|j|j|}|j|}|r|sJt |dkr%t |dkr|d|dks"td|d|dS|rU|j ||s:td |d |j|dS|j |jd } | t | j dkrtd dS|j |jd } | 4t | j dks| j d| j dkrtd dS|rw|j |jd } | 4t | j dks| j d| j dkrtddS| j d | j d krRt d|jd d| j d d|jd d| j d |r| j d | j d krRt d|jd d| j d d|jd d| j d | j d | j d krRt d|jd d| j d d|jd d| j d dS)zXSanity check of embedding weights, and match hidden_size of weights and shape of inputs.r'NT)updater>z^Cannot fuse EmbedLayerNormalization: input_ids and position_ids not matched in 2nd dimension: z vs FzYCannot fuse EmbedLayerNormalization: input_ids and segment_ids does not have same shape: z != rzICannot fuse EmbedLayerNormalization: word embedding table is not expectedzMCannot fuse EmbedLayerNormalization: position embedding table is not expectedzLCannot fuse EmbedLayerNormalization: segment embedding table is not expectedzword_embedding_table (z) size z <= position_embedding_table (z <= segment_embedding_table (zposition_embedding_table () rDrrinfer_runtime_shaperget_edge_shaperFrGinfo compare_shapernr^warning) rword_embedding_gathersegment_embedding_gatherrZr[ segment_ids position_idsinput_ids_shapeposition_ids_shapeword_embedding_tableposition_embedding_tablesegment_embedding_tables r!check_embeddingz&FusionEmbedLayerNoMask.check_embeddingHs>)/2 ;S].4Q77Y] 06q9 $ )#z==T=JJD $(D !   '".==iHHO!%!1!@!@!N!N " 9'9 9 9 9O$$))*++q00#A&*)DQ)G\\PdPjklPm\\NgNmnoNp\\yQyWXYyZ\\     #)!,0G0Ma0PPP]-B-H-K]]ThTnopTq]]QiQopqQr]]{R{XYZ{[]](-a04K4QRS4TTTi1J1PQR1Sii\t\z{|\}ii]u]{|}]~iiG^GdefGgiitr" input_namecd}|j|}|@|jjjt jkr|j|\}}n |}n|j|\}}||fS)aCast a graph input or node input to int32. Args: input_name (str): name of graph input or node input Returns: A tuple of casted input name and the cast node. int32_output (str): If input is int32, it is the input name, Otherwise it is output name of Cast node. input_cast_node (Union[None, NodeProto]): Cast node. It could be None if input is int32. N) rfind_graph_inputtype tensor_type elem_typer INT32rcast_input_to_int32)rrinput_cast_node graph_input int32_outputs r! cast_to_int32z$FusionEmbedLayerNoMask.cast_to_int32sj11*==  "+59JJJ04 0N0Nz0Z0Z- oo) ,0J,J,J:,V,V )L/_,,r"Fr[rrZrrc tg} ||\}} |jd} |jdkr|jd} |jd} n|jd} |jd} d}|N||jd\}} |||jd|jd|jd| | g}n|d|jd|jdd| | g}|B|d||\}} ||| d z| d zg}|r||n| d z}||t jd||| }d |_|j D](}|j dkr|j |g)t|j dkr.|j t j ddg| || D]}|j|j|j <|j | ||_|S)agCreate an EmbedLayerNormalization node. Note that segment embedding is optional. Args: input_ids (str): input_ids for word embeddings layernorm (NodeProto): LayerNormalization or SkipLayerNormalization node. word_embedding_gather (NodeProto): the Gather node for word embedding position_embedding_gather (NodeProto): the Gather node for position embedding segment_embedding_gather (Union[None, NodeProto]): the Gather node for segment embedding, or None. Returns: NodeProto: the EmbedLayerNormalization node created. rrr'r>rjNr_output_dummy_mask_index_embedding_sum)outputsnamez com.microsoftepsilong-q=)rrcreate_node_namer5rDappendr make_nodedomain attributerextendrFmake_attributethis_graph_namenode_name_to_graph_name nodes_to_addr)rr[r,rrZrrembedding_sum_outputembedding_sum_namerr_ node_namegammabetaembed_node_inputsrembed_node_outputsrrattrKs r!create_fused_nodez(FusionEmbedLayerNoMask.create_fused_nodes. )))44 1J//0IJJ   4 4 4OA&E?1%DDOA&E?1%D  # /!//0H0Nq0QRRNK%+A.)/2(.q1!  %+A.)/2!   #  $ $R ( ( ("00>>OL!  $ $\ 2 2 2')3YAT5TU  ,);)G%%YYiMiD  % %d + + +% % &    , & 3 3Cx9$$$++SE222 z# $ $ ) )  ' ')>y')R)R(S T T T J'''  K KD6:6JD ( 3 3   ...$r"cv|j|jd|jdd|_dS)NrT)rreplace_input_of_all_nodesrB prune_graph)rr,rs r! finish_fusionz$FusionEmbedLayerNoMask.finish_fusion s9 --i.>q.A:CTUVCWXXXr"c|jdko5t|jdkot|jddkS)Nrrjr)r5rFrB)rrKs r!"is_skip_layer_norm_with_sum_outputz9FusionEmbedLayerNoMask.is_skip_layer_norm_with_sum_outputsC 88nc$+>N>NQR>RnWZ[_[fgh[iWjWjmnWnnr"c r||}|dS|\}}|jd} |jd} |||dsdS||d|sdS|jdkrK||} d} |} | r |jdnd}|duo|j|du}n|} |jdkrdnd} t|j| kr |j| nd}|duo|j|du}|o||vot||dk}|duo|jdkp|p|} | | ||||| | |r|nd}| r2d | j| <|s&|j ||jd | ||d S) NFr'r.rrjr;r)rr_no_use__to_be_removed_r>T) r+rDrOrr5rrBrfind_graph_outputrFrrr)rr,add_before_layernormr-r\optional_segment_gather two_gatherrrZr[rneed_embedding_sum_outputsum_output_indexnode_with_sum_output sum_outputis_sum_graph_outputis_sum_used_by_multiple_nodesrs r! fuse_gpt2z FusionEmbedLayerNoMask.fuse_gpt2sn(**+?@@  5;E88)/2 06q9 ,,Y8K\a,bb 5##$94AZ[[ 5   8 8 8(,(O(OPY(Z(Z % #, 0IS)!,,tJ#-T#9"u @\@\]g@h@hpt@t  #7 $8$@E$I$Iqqq +2336FFF%+,<==  $.T#9"u @\@\]g@h@hpt@t o .A AosK^_iKjGkGknoGo **44)?)$,5m9LmPm & ++   ! % # !:-@Jzzd,    % X">?SV^U_bcad"e"e " *5$;A$>!,,-F Sfgg -001I9Vijj u+D'@ $(, %##$9;SUnoo 5++   ! % $    9j111tr"c4|j|dgdg}|jdkr|dS|d}d}n|j|dgdg}|j|dgdg}|||dS|d}|d}n;|5|3|j|dgdg}|dS|d}|d}n|}d}||||||rdS|||||rdS|||||rdSdS)Nr;rrr&r')rr(r5rrr) rrKr-r\first_add_pathrrr)r*s r!fusezFusionEmbedLayerNoMask.fuses55dUGaSII </ / /%#1!#4 &* # # J88zA3OOM J88zA3OOM$)B!)F'5a'8$*7*:''*}/D!%!=!=dUGaS!Q!Q!)F'5a'8$*7*:'''+$*.' >> &(;=PRi    F   &:>$ 46IK^ _ _  F  r")r)NFN)N)__name__ __module__ __qualname____doc__rstrrr rrr+rrboolrOrdrgrxrzrrrrrrrrrr __classcell__r s@r!rrsR  i c       2I 2%eIyDX>Y8Y2Z 2 2 2 2RR"#tI"67R R  RRRRh<<<|---^F+F+F+P   HHHT--c5y;Q6Q0R----<'+"```` ) ` $- ` #(i"8 `sm````D   ooorvOOOOb'''R000d"""""""r"rc6eZdZddeffd ZdZfdZxZS)FusionEmbedLayerNormalizationFrcZt|d||_dS)Nz with mask)rruse_mask_index)rrrr s r!rz&FusionEmbedLayerNormalization.__init__s+  ,,,,r"cj|j}t|jdkr;|j|td|jnrt|jdkr8|jds+||jd<td|jn"td|jdS|D]c}td|j|jdkr|jd|jd<C|jd kr|jd|jd <ddS) Nzappend mask to %szreplace mask in %szskip mask in %szupdate mask_index in %sr0r'rjr<rk) rrFrDrrGrHrr5rB)r mask_int32attention_nodesrattention_nodes r! replace_maskz*FusionEmbedLayerNormalization.replace_masks4_ z A % %   # #J / / / LL,jo > > > > ! " "Q & &z/?/B &",J Q  LL-z ? ? ? ? LL*JO < < < F- ? ?N LL2N4G H H H%44*4*;A*>$Q'''+???*4*;A*>$Q'  ? ?r"c*d|_d|_d|_t||||jdS|js1t d|ddS|j8|j1t d|ddS|jr|jj d}n|jj d}||}|j |r9d|D}| |||ddS||vr2t d||ddS||}|j d vrd |D}|j d krG|j d }t|t|kr|j|| |||ddSdS) NzG--use_mask_index is not set: EmbedLayerNormalization will not have maskz EmbedLayerNormalization(no mask)zLEmbedLayerNormalization will not have mask since attention node is not foundrjrkc$g|] }|jdv |S)r0r<r4r7rKs r!r9z6FusionEmbedLayerNormalization.fuse..%vvv$,RuBuButBuBuBur"z"EmbedLayerNormalization(with mask)zHEmbedLayerNormalization will not have mask since %s is not a node output) ReduceSumrVc$g|] }|jdv |Srr4rs r!r9z6FusionEmbedLayerNormalization.fuse..%rr"rr)rrErrrrrGrHincrease_counterrDrrrr5rFnodes_to_remover)rrKr-r\rchildren_nodesrr s r!rz"FusionEmbedLayerNormalization.fuses1#  T.0CDDD ? " F"  LLb c c c  ! !"D E E E F > !d&:&B LLg h h h  ! !"D E E E F > 7-a0JJ-3A6J,Z8 : & &z 2 2 vvvvvO   j/ : : :  ! !"F G G G F 0 0 0 LLceo p p p  ! !"D E E E F":. <0 0 0vvvvvO|{**!Z] ~&&#o*>*>>>(//555   j/ : : :  ! !"F G G G G G 1 0r")F)rrrrrrrrrs@r!rrsz--i------???*-H-H-H-H-H-H-H-H-Hr"rN)loggingrtypingrrrrr fusion_baser fusion_utilsr onnxr r r onnx_modelrrrGrrr3r"r!rs" 55555555555555$$$$$$////////// 8  P P P P P VP P P fGHGHGHGHGH$:GHGHGHGHGHr"