
     Ng}p                     ~    d dl Z d dlZd dlmZmZ d dlmZmZ d dl	m
Z
  e j        e          Z G d de          ZdS )    N)AttentionMaskFusionAttention)TensorProtohelper)	OnnxModelc                   F     e Zd ZdZdedededef fdZd Zd Z	d	 Z
 xZS )
FusionBartAttentionz?
    Fuse Bart Attention subgraph into one Attention node.
    modelhidden_size	num_headsattention_maskc                 P    t                                          ||||           d S )N)super__init__)selfr
   r   r   r   	__class__s        j/var/www/html/ai-engine/env/lib/python3.11/site-packages/onnxruntime/transformers/fusion_bart_attention.pyr   zFusionBartAttention.__init__   s)     	YGGGGG    c                    | j                             |dgdg          }|dS |d         }| j                             |g dg d          }	| j                             |g dg d          }
|	|
dS |	\  }}}|
\  }}}|j        d         |k    s|j        d         |k    rdS | j                             |g dg d          }| j                             |g dg d	          }||dS |d
         j        |j        k    s|d
         j        |j        k    rdS | j                             |g dg d          }| j                             |g dg d          }| j                             |g dg d          }|||dS |d
         }|d
         }|d
         }|j        d         }|j        d         |k    s"|j        d         |k    s|j        d         |k    rdS dS )NConcat   Fr   	UnsqueezeGatherShaper   r   r   )r   r   r   )r   r   r   )r      r   )r   r   MulT)r
   match_parent_pathinputnameoutput)r   reshape_qkv_2reshape_qkv_1reshape_q_2reshape_k_2reshape_v_2
root_inputconcat_qkv_2_pathconcat_qkv_2reshape_qkv_2_path_1reshape_qkv_2_path_2_gather_1shape_1gather_2shape_2reshape_qkv_1_path_1reshape_qkv_1_path_2reshape_q_2_pathreshape_k_2_pathreshape_v_2_pathmul_qmul_kmul_vgather_1_outs                            r   check_runtime_shape_pathz,FusionBartAttention.check_runtime_shape_path   sc    !J88
UVTWXX$5(+#z;;LJjJjJjlululuvv#z;;LJjJjJjlululuvv'+?+G538W38W=z))W]1-=-K-K5#z;;MKlKlKlnwnwnwxx#z;;MKlKlKlnwnwnwxx'+?+G5#(HM99=QRT=U=Z^f^k=k=k5:77EcEcEcenenenoo:77EcEcEcenenenoo:77EcEcEcenenenoo#'7'?CSC[5 $ $ $q);q>\))U[^|-K-Ku{[\~amOmOm5tr   c                    | j                             |g dg d          }|dS |d         j        d         |j        d         k    rdS | j                             |g dg d          }| j                             |g dg d          }||dS |d         }	|d         }
|	j        d	         |
j        d	         k    rdS |d         j        d         |j        d         k    r$|d         j        d         |j        d         k    rdS || j                             |d
gd	g          }|dS | j                             |d         g dg d          }| j                             |d         g dg d          }||dS |\  }}}}|\  }}}|j        d         |j        d         k    rdS |d         j        d         |j        d         k    r$|d         j        d         |j        d         k    rdS dS )N)r   Slicer   r   r   r   r   r   Fr   r   )r   PowCastDivr   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r>   )r>   r   r   r   )r   r   r   r   r   )r   r   r   T)r
   r    r!   r#   )r   r$   
matmul_qkvadd_qk	matmul_qkadd_qreshape_qkv_2_pathmatmul_qk_path_1matmul_qk_path_2mul_1mul_2add_qk_pathslice_q_path_1slice_q_path_2r.   unsqueeze_1unsqueeze_2s                    r   check_runtime_shape_path_openaiz3FusionBartAttention.check_runtime_shape_path_openaiM   se    "Z99AAA<<<
 
 %5!"%+A.*2CA2FFFu:77GGGI[I[I[
 
  :77GGGI[I[I[
 
 #'7'?5 # #;q>U[^++5B%a(ELO;;@PQS@T@Z[\@]afamnoap@p@p5 *66vy1#NNK"u!Z99A I I I<<< N "Z99+a.JjJjJjlululuvvN%.*@u#1 A{Aq .KA #{'8';;;ub!'*el1o==.QSBTBZ[\B]afamnoapBpBputr   c                    d}| j                             |g dg d          }| j                             |g dg d          }|
|\  }}}	}
}}n||}|\  }}}	}
}d}nd S g }|j        D ]4}||vr||d         j        d         k    r|                    |           5t          |          dk    rd S |d         }	 ||         }|j        d	k    r | j                             |          d         }|j        D ]6}|s||         }d
 |D             }|                    d          dk    r|} n7t          d | j         
                                j        D                       }t          d | j         
                                j        D                       }| j                             |g dg d          }| j                             |g dg d          }| j                             |g dg d          }| j                             |dgdg          }| j                             |g dg d          }d\  }}d\  }}||\  }}} }}!|j        d         }nV||}|\  }} }}!| j                             |ddg| g          }"| j                             |g d          }#|"!|"\  }$}%|%j        d         |v r|%j        d         }|#T|#\  }&}$}'|'j        d         |v r|'j        d         }| j                             |&ddgddg          }(|(\  }$})|)j        d         }n|'|\  }}&}} }}!|}|&j        d         }|&j        d         }nj||d         j        d         |v r|}|d         j        d         }|d         j        d         }||vrct          t          d | j                                         |                             }*t          |*          dk    r|*d         j        d         nd}n||d         j        d         |v r|}|d         j        d         }|d         j        d         }||vrct          t          d | j                                         |                             }*t          |*          dk    r|*d         j        d         nd}nt                              d            d S ||v r|nd}||v r|nd}| j                             |d!dgddg          }+| j                             |g d"g d#          },| j                             |g d$g d%          }-d }.|+|+\  }$}/|+}0n|,|,\  }$}$}.}$}/|,}0n|-	|-\  }$}.}/|-}0nd S | j                             |/g d&g d'          }1| j                             |/g d(g d)          }2d }3|1
|1\  }3}4}5}6}7}8n|2|2}1|1\  }6}4}5}7}8nd S | j                             |/g d*g d+          }9| j                             |/g d,g d          }:| j                             |/g d-g d.          };| j                             |/g d/g d0          }<| j                             |/ddgddg          }=| j                             |/g d1g d.          }>d\  }?}@d2\  }A}B}C|9|9\  }$}A}D}B}E}C|9}Fnd|:|:\  }G}D}B}C|:}F|Cj        d         }@| j                             |Cddg|Bg          }"| j                             |Cg d          }#|"!|"\  }$}H|Hj        d         |v r|Hj        d         }@|#T|#\  }I}$}J|Jj        d         |v r|Jj        d         }@| j                             |Iddgddg          }(|(\  }$}K|Kj        d         }?n|;|;\  }$}A}D}B}C|;}F|Dj        d         }@ny|<'|<\  }$}A}I}$}B}C|<}F|Ij        d         }?|Ij        d         }@nP|=|=d         j        d         |v r|=}F|Fd         j        d         }?|Fd         j        d         }@|@|vrct          t          d3 | j                                         |?                             }Lt          |L          dk    r|Ld         j        d         nd}@n|>|>d         j        d         |v r|>}F|Fd         j        d         }?|Fd         j        d         }@|@|vrct          t          d4 | j                                         |?                             }Lt          |L          dk    r|Ld         j        d         nd}@nd S |?|v r|?nd}?|@|v r|@nd}@|F|:|;|<fv r| j                             |j        d                   j        d         }Md5}N| j                             |N          }O|OF|                     |Nt(          j        |Mgt-          j        d6g|Mz  t,          j        7          8           | j                             d	          }Pt5          j        d	|N|Cj        d         g|Bj        g|P          }E|r|?s|                     |	||.|/|7          sd S |s|?s|                     |	||3|A||          sd S |?o|o|Cd u od9t?                      v}Q|Q o2|Cj        d         |k    o!|8j        d         |k    o|!j        d         |k    }R|Q oH|8j        d         |k    o7|Cj        d         |!j        d         k    o|Cj        d         |8j        d         k    }S|Ro|0|+k    }T|Ro|0|,|-fv }U|s|Tn|Uo|?o|}V|So|0|+k    }W|Qo|0|+k    }Xd }Y|Uri| j                             |.d:gdg          }Z| j                             |.g d;g d          }[|[|[d         j        d         }Yn|Z|Zd         j        d         }Y|Ts	|Us|Vs|Ws|Xr|	}\|                      |5          \  }]}^|]dk    s|^dk    s	|^|]z  dk    rt                              d<           d S d }_|Vs|Ws|XrX| j!        rN| "                    |8|Ws|Vr|Cn|?|Ws|Vr|!n||7|Ws|Vr|End |Ws|Vr|nd |]|^|\j        d         |Vr|?nd|Vr|nd|@||V=          nd }_nI| j!        }`d| _!        | #                    d |8|C|!|7|E||]|^||\j        d         |Ur|Ynd |?||@|>          }_|`| _!        |_d S | j$                            |_           | j%        | j&        |_j        <   | j'        (                    |\|
|g           | j'        (                    |0           |Vs|Ws|Xr|1d         j        dk    r|1)                                 |Fd         j        dk    r|F)                                 |d         j        dk    r|)                                 | j*        rs|Ws|Xro|1d         j        d	k    r|1)                                 |Fd         j        d	k    r|F)                                 |d         j        d	k    r|)                                 | j'        (                    |1           | j'        (                    |F           | j'        (                    |           d| _+        d S d S )?NF)AddMatMulReshape	TransposerV   rU   rC   )rT   rU   rV   rW   rU   )r   r   r   r   r   Tr   r   rT   c                     g | ]	}|j         
S  op_type).0childs     r   
<listcomp>z,FusionBartAttention.fuse.<locals>.<listcomp>   s    BBBemBBBr   rU   c                     g | ]	}|j         
S rY   r"   r\   nodes     r   r^   z,FusionBartAttention.fuse.<locals>.<listcomp>   s     P P Pt P P Pr   c                     g | ]	}|j         
S rY   r`   ra   s     r   r^   z,FusionBartAttention.fuse.<locals>.<listcomp>   s    !R!R!R$)!R!R!Rr   )rV   rW   rV   rT   rU   )r   r   r   r   N)rW   rV   rT   rU   )r   r   r   N)rV   r   rW   rV   rT   rU   )r   r   r   r   r   NrV   )rW   rV   rV   rW   r?   ) rd   )NNrW   )exclude)r   rV   rW   r   c                     | j         dk    S NIdentityrZ   rb   s    r   <lambda>z*FusionBartAttention.fuse.<locals>.<lambda>       
(B r   rd   c                     | j         dk    S rg   rZ   ri   s    r   rj   z*FusionBartAttention.fuse.<locals>.<lambda>,  rk   r   z&fuse_attention: failed to match v pathSoftmax)rm   rV   rT   rV   rU   )r   r   r   r   r   )rm   rT   rU   r   )rV   rW   rV   r   rT   rU   )r   r   r   r   r   r   )r   rW   rV   rT   rU   )r   r   r   r   r   )rW   rV   rW   rV   rT   rU   )r   r   r   r   r   r   )r   rW   rV   rU   )rW   rV   rW   rV   rU   )r   r   r   r   r   )rW   rV   r   rW   rV   rU   )r   r   r   r   r   r   )r   rW   rV   rV   rW   )NNNc                     | j         dk    S rg   rZ   ri   s    r   rj   z*FusionBartAttention.fuse.<locals>.<lambda>  rk   r   c                     | j         dk    S rg   rZ   ri   s    r   rj   z*FusionBartAttention.fuse.<locals>.<lambda>  rk   r   
empty_biasg        )dtype)dimsvalsmatmul_vWhere)Expandr   r   ru   z9fuse_attention: failed to detect num_heads or hidden_size)past_kpast_v	present_k	present_v
packed_qkv)
add_qk_strrw   rx   ry   rz   ),r
   r    r!   r#   appendlenr[   get_childrencountsetgraphmatch_child_pathlistfilterinput_name_to_nodesloggerdebugget_initializerrr   add_initializerr   FLOATnparrayfloat32create_node_namer   	make_noder"   rR   r<   localsget_num_heads_and_hidden_sizeuse_multi_head_attentioncreate_multihead_attention_nodecreate_attention_nodenodes_to_addthis_graph_namenode_name_to_graph_namenodes_to_removeextendpop!disable_multi_head_attention_biasprune_graph)ar   normalize_noder   output_name_to_nodemodel_impl_openai	qkv_nodesqkv_nodes_openaiadd_out
matmul_outr$   transpose_qkvr%   rD   other_inputsr!   r)   skip_layernormr#   childrenchildren_typesgraph_input_namesgraph_output_namesv_nodesv_nodes_openaiv_nodes_with_past_self_attnv_nodes_with_past_cross_attn#v_nodes_with_past_cross_attn_openairx   rz   r(   add_vtranspose_vreshape_v_1rt   reshape_pathconcat_pathr.   transpose_add_vconcat_vtranspose_concat_vconcat_nodestranspose_concat_v_inidentity_node_v
qk_nodes_1
qk_nodes_2qk_nodes_2_openairE   rF   qk_nodesq_nodesq_nodes_openair&   transpose_qreshape_q_1r8   rG   matmul_qk_nodes_with_biask_nodes_with_bias_openaik_nodes_no_bias#k_nodes_no_bias_with_past_self_attn$k_nodes_no_bias_with_past_cross_attn+k_nodes_no_bias_with_past_cross_attn_openairw   ry   r'   reshape_k_1matmul_ktranspose_k_1add_kk_nodesr9   transpose_matmul_kconcat_ktranspose_concat_ktranspose_concat_k_inidentity_node_kbias_dimempty_bias_nameempty_tensoradd_namethree_root_inputsone_root_inputtwo_root_inputsencoder_attentiondecoder_attentiondecoder_attention_with_pastdecoder_cross_attention!decoder_cross_attention_with_past
mask_indexmask_nodes_bartmask_nodes_whisperattention_last_noder   r   new_node%use_multi_head_attention_ground_truthsa                                                                                                    r   fusezFusionBartAttention.fuse   s   ! J00JJJ
 
	
  :77???OO
 

   
)(I  !%F#) 	' 	'E///	!+A...&&&&|!!F!!_
	 -Z8 !U**!Z44^DDQGN$+ 	 	F *62HBBBBBN##H--22#
 3   P Ptz7G7G7I7I7O P P PQQ !R!R
8H8H8J8J8Q!R!R!RSS*..@@@
 

 55555OO
 

 '+j&B&BJJJ!!!	'
 '
# (,z'C'CKC	(
 (
$ /3j.J.J<<<LL/
 /
+
 #	'UGND[+{E8#*1-II'$G:A7[+uh  :66K($ 7  L *55e=_=_=_``K''3$O")!,0BBB / 6q 9I&4?110%,Q/3EEE 2 9! <I#z;;HyR]F^abde`fgg+7((.4Q7(4QlN[(KeX1G^A&F *II(49UVX9Y9_`a9bfw9w9w2GR[&q)F*1-I 222"&BBDJDbDbDdDdekDlmm# # =@<P<PTU<U<UOA.5a88[]	/;3B7=a@DUUU9GR[&q)F*1-I 222"&BBDJDbDbDdDdekDlmm# # =@<P<PTU<U<UOA.5a88[]	LLABBBF!%666B!*.@!@!@IIb	Z11*y(>SVWYZU[\\
Z11JJJOOO
 

 !J88EaEaEaclclclmm!%LAy!HH#)3&Aq&!Y!HH*#4 Avy(HHF*..GGG
 

 55<<<OO
 

 LSIKk5%'$G?F<E;UHHF J88MMM
 

 $(:#?#?555LL$
 $
 
 *66FFFOO
 

 /3j.J.JPPP	/
 /
+ 04z/K/K)$F	0
 0
, 7;j6R6RCCCOO	7
 7
3 #	-=*[((J[GA{M;x'GG%1:R7E=+x.G *I  :66K($ 7  L *55h@b@b@bccK'*6'&%,Q/3EEE 2 9! <I&4?110%,Q/3EEE 2 9! <I#z;;HyR]F^abde`fgg+7((.4Q7(CR@A{M;%G%,Q/II0<Ad>A{Hah9G^A&F *II0<4R8>qAEVVV:GR[&q)F*1-I 222"&BBDJDbDbDdDdekDlmm# # =@<P<PTU<U<UOA.5a88[]	7C;B?EaHL]]]AGR[&q)F*1-I 222"&BBDJDbDbDdDdekDlmm# # =@<P<PTU<U<UOA.5a88[]	F!%666B!*.@!@!@IIb	/Befffz11%+a.AAFqIH*O:55oFFL#$$#%"3%("2"*EEE	 %    z22599H$U_hoa>P,QT_TdSegoppE 		 88 	 F!		 11 	 F"ava(d2BazY_YaYaGa!! 0q!Z/0q!Z/0 q!Z/	 	 "! 7q!Z/7q!X^A%667 q!X^A%66	 	 +Ex:/E*ZxJHY;Z/Z&7N=NeTZe_e 	$ #2"Lh*6L,=,X(jBX) 
 	;"j::	 O
 "&!=!====" "
 "-/29"=

 ,,Q/6r:
 ^	$ ^	$ +^	$ '	^	$
 1^	$ #0%)%G%G%T%T"I{A~~!1!1kI6MRS5S5SXYYYH* .V.E .VIj .V* 4!D88 $;f?Zf`f$;f?Zf`f!8a<Wa]a!8a<Wa]a!#+215)DLvv")DLvv""+"+#> 9   " % , 9=8U505-55'.q1->HzzD!!''! 6  $ 1V-$$X...:>:ND(7 '')<mZ(XYYY ''111 + &.E &Ij &2;&(22KKMMM2;&(22KKMMM2;&(22KKMMM9 &+&/P& r{*e33r{*e33r{*e33 ''000 ''000 ''000  $D}^	$ ^	$r   )__name__
__module____qualname____doc__r   intr   r   r<   rR   r   __classcell__)r   s   @r   r	   r	      s         HH H 	H
 &H H H H H H. . .`3 3 3j~$ ~$ ~$ ~$ ~$ ~$ ~$r   r	   )loggingnumpyr   fusion_attentionr   r   onnxr   r   
onnx_modelr   	getLoggerr   r   r	   rY   r   r   <module>r      s   
      ; ; ; ; ; ; ; ; $ $ $ $ $ $ $ $            		8	$	$q	$ q	$ q	$ q	$ q	$/ q	$ q	$ q	$ q	$ q	$r   