NgL0nddlmZddlmZGddZGddeZGddZd S) )ArgumentParser)EnumceZdZdZdZdZdZdS)AttentionMaskFormatrN)__name__ __module__ __qualname__ MaskIndexEndMaskIndexEndAndStart AttentionMaskNoMaskc/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/transformers/fusion_options.pyrr s)LMFFFrrc0eZdZdZdZdZdZdZdZdZ dS) AttentionOpType AttentionMultiHeadAttentionGroupQueryAttentionPagedAttentionc|jSNvalueselfs r__str__zAttentionOpType.__str__s zrc*t|jSr)hashrrs r__hash__zAttentionOpType.__hash__!sDJrc"|j|jkSrr)rothers r__eq__zAttentionOpType.__eq__$s{dj((rN) r r r rrrrr r#r&rrrrrsWI-/%N   )))))rrcdeZdZdZdZd dZdZdefdZe dZ e d e fd Z d S) FusionOptionsz'Options of fusion in graph optimizationcd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_|dkrd|_tj|_|dkrtj|_n|dkrtj|_d|_|dvr3d|_d|_d|_d|_d|_d|_d|_dSdS)NTFclipbertvitunetvaer*) enable_geluenable_layer_normenable_attentionenable_rotary_embeddingsuse_multi_head_attention!disable_multi_head_attention_biasenable_skip_layer_normenable_embed_layer_normenable_bias_skip_layer_normenable_bias_geluenable_gelu_approximationenable_qordered_matmulenable_shape_inferenceenable_gemm_fast_gelugroup_norm_channels_lastrrattention_mask_formatr rattention_op_typeenable_nhwc_convenable_group_normenable_skip_group_normenable_bias_splitgeluenable_packed_qkvenable_packed_kvenable_bias_add)r model_types r__init__zFusionOptions.__init__+s'!% $(,% ).%16.&*#'+$+/( $).&&*#&*#%*"(,%   +0D (&9%F"   )<)ID & & 5 )<)CD &!% 0 0 0$(D !%)D "*.D ')-D &%)D "$(D !#'D  1 0rTcR|rtj|_dStj|_dSr)rrr?r )r use_raw_masks ruse_raw_attention_maskz$FusionOptions.use_raw_attention_maskZs-  J)<)JD & & &)<)ID & & &rc(tj|_dSr)rrr?rs rdisable_attention_maskz$FusionOptions.disable_attention_mask`s%8%?"""r attn_op_typec||_dSr)r@)rrOs rset_attention_op_typez#FusionOptions.set_attention_op_typecs!-rct|j}|jrd|_|jrd|_|jrd|_|jrd|_ |j rd|_ |j rd|_ |j rd|_|jrd|_|jrd|_|jrd|_|jrd|_|jrd|_|jr|d|jr|d|jr||jdvrp|jrd|_|jrd|_|jrd|_ |j!rd|_"|j#rd|_$|j%rd|_&|j'rd|_(|j)rd|_*|S)NFTr-)+r(rH disable_gelur0disable_layer_normr1disable_rotary_embeddingsr3disable_attentionr2r4disable_skip_layer_normr6disable_embed_layer_normr7disable_bias_skip_layer_normr8disable_bias_gelur9r:disable_shape_inferencer<r=use_mask_indexrLno_attention_maskrNuse_group_norm_channels_firstr>disable_nhwc_convrAdisable_group_normrBdisable_skip_group_normrCdisable_bias_splitgelurDdisable_packed_qkvrEdisable_packed_kvrFdisable_bias_addrG)argsoptionss rparsezFusionOptions.parsefs00   ("'G   " .(-G %  ) 5/4G ,  ! -',G $  ( 4/3G ,  ' 3-2G *  ( 4.3G +  , 827G /  ! -',G $  ) 504G -  ' 3-2G *  % 1,0G )   2  * *5 1 1 1  & 1  * *4 0 0 0  ! -  * * , , , ?5 5 51 9380% 1+0(& 2,1)+ 716.* 605-& 2,1)% 1+0($ 0*/'rparsercL|dddd|d|dddd|d |d ddd |d |d ddd|d|dddd|d|dddd|d|dddd|d|dddd|d|dddd|d|dddd |d!|d"ddd#|d$|d%ddd&|d'|d(ddd)|d*|d+ddd,|d-|d.ddd/|d0|d1ddd2|d3|d4ddd5|d6|d7ddd8|d9|d:ddd;|d<|d=ddd>|d?|d@dddA|dB|dCdddD|dE|dFdddGdS)HNz--disable_attentionF store_truezdisable Attention fusion)requiredactionhelp)rVz--disable_skip_layer_normz%disable SkipLayerNormalization fusion)rWz--disable_embed_layer_normz&disable EmbedLayerNormalization fusion)rXz--disable_bias_skip_layer_normz2disable Add Bias and SkipLayerNormalization fusion)rYz--disable_bias_geluz)disable Add Bias and Gelu/FastGelu fusion)rZz--disable_layer_normz!disable LayerNormalization fusion)rTz--disable_geluzdisable Gelu fusion)rSz--enable_gelu_approximationz+enable Gelu/BiasGelu to FastGelu conversion)r:z--disable_shape_inferencez disable symbolic shape inference)r[z--enable_gemm_fast_geluzenable GemmfastGelu fusion)r=z--use_mask_indexzWuse mask index to activate fused attention to speed up. It requires right-side padding!)r\z--use_raw_attention_maskzuse raw attention mask. Use this option if your input is not right-side padding. This might deactivate fused attention and get worse performance.)rLz--no_attention_maskz1no attention mask. Only works for model_type=bert)r]z--use_multi_head_attentionzUse MultiHeadAttention instead of Attention operator for testing purpose. Note that MultiHeadAttention might be slower than Attention when qkv are not packed. )r4z--disable_group_normz9not fuse GroupNorm. Only works for model_type=unet or vae)r`z--disable_skip_group_normzPnot fuse Add + GroupNorm to SkipGroupNorm. Only works for model_type=unet or vae)raz--disable_packed_kvz[not use packed kv for cross attention in MultiHeadAttention. Only works for model_type=unet)rdz--disable_packed_qkvz[not use packed qkv for self attention in MultiHeadAttention. Only works for model_type=unet)rcz--disable_bias_addz0not fuse BiasAdd. Only works for model_type=unet)rez--disable_bias_splitgeluz6not fuse BiasSplitGelu. Only works for model_type=unet)rbz--disable_nhwc_convz:Do not use NhwcConv. Only works for model_type=unet or vae)r_z--use_group_norm_channels_firstznUse channels_first (NCHW) instead of channels_last (NHWC) for GroupNorm. Only works for model_type=unet or vae)r^z--disable_rotary_embeddingsz5Do not fuse rotary embeddings into RotaryEmbedding op) add_argument set_defaults)ris r add_argumentszFusionOptions.add_argumentss !+    e444 '8    E::: (9    U;;; ,E    ??? !<    e444 "4    u555 &    /// )>    e<<< '3    E::: %-    %888 j    5111 &e    5999 !D    e444 (d     U;;; "L    u555 'c    E::: !n    e444 "n    u555 C    U333 &I    5999 !M    e444 -B    %@@@ )H      rN)T) r r r __doc__rIrLrNrrQ staticmethodrhrrqrrrr(r((s11-(-(-(^JJJJ @@@./....33\3jw nw w w \w w w rr(N)argparserenumrrrr(rrrrvs $#####        )))))d)))"l l l l l l l l l l r