קg5A ddlmZddlmZmZddlmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZddlmcmZddlmZddlZddlmZddlmcmZddlZddl Z ddl!Z!ddl"m#Z#de$d e e$e$ffd Z%d e ed ej&d e e$e ffdZ'd ej&d e e$e fdejj(fdZ)d*dejj(d ejj(fdZ*dej(d ej(fdZ+dej(deej&deej&deej&fdZ,ej-ej.ej/ej0ej1ej2ej3ej4ej5ej6ej4ej7ej8g Z9ej:ej;gZej/diZ?deej&d e e$ej(ffdZ@deej&d e e$ej(fde ej(ej(ffdZAGddZBd+d"ZCd#eBd eDfd$ZEGd%d&ZFdejGfdejj(d'e e e$e fd(eejGd ejj(fd)ZHdS),N)ArgumentTarget)fuse_conv_bn_eval)TypeDictAnyTupleIterableOptionalListcast) ShapeProp) defaultdict)EnumtargetreturncP|dd^}}|r|dnd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnames ^/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/fx/experimental/optimization.py _parent_namers3 MM#q))MVT &6!99B ,,patternnodemodulescjt|jdkrdS|jd|f}t||D]x\}}t|tjsdS|jdkrdSt|jtsdS|j|vrdSt||j|urdSydS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)rrrnodes expected_type current_nodes rmatches_module_patternr.s 49~~u"&)A,!5E'*7E':':  # |,00 55 ?m + +55,-s33 55  g - -55  +, - -] B B55 C 4r new_modulect|jtsJt|j\}}|||j<t ||||dSN)r%rr)rsetattr)rrr/ parent_namers rreplace_node_moduler4-sW dk3 ' ''''$T[11K%GDK GK $ 33333rFmodelcntjtjftjtjftjtjfg}|stj|}|rt|tj j stj |}n|}t|}tj|j}|D]}|jD]}t%|||rt'|jdjdkr7||jdj} ||j} | jsdt1| | } t3|jd|| ||jd||ƌtj ||S)z Fuses convolution/BN layers for inference purposes. Will deepcopy your model by default, but can modify the model inplace as well. rr)nnConv1d BatchNorm1dConv2d BatchNorm2dConv3d BatchNorm3dcopydeepcopyr%torchr& GraphModulesymbolic_tracedict named_modulesgraphr+r.r"r#usersrtrack_running_statsrr4replace_all_uses_with erase_node) r5inplaceno_tracepatternsfx_modelr new_graphrrconvbn fused_convs rfuserR3s BN+BN+BN+-H % e$$ :eUX-ABB$U++8))++,,G hn--I + +O + +D%gtW== +ty|)**Q..ty|23T[)-.tR88 #DIaL':FFF**49Q<888$$T*** + >(I . ..rctj|}Gddtjj}||S)z5 Removes all dropout layers from the module. cPeZdZdedeedfdeeefdeffd Z xZ S)&remove_dropout..DropoutRemoverrr#.kwargsrct|j|tjrt |dksJ|dSt |||S)Nrr)r% submodulesr7Dropoutr"superr!)selfrr#rV __class__s rr!z2remove_dropout..DropoutRemover.call_moduleZsW$/&12:>> A4yyA~~~~Awww**64@@@r) __name__ __module__ __qualname__rr rrr)rr! __classcell__)r\s@rDropoutRemoverrUYs~ Av AeHcM6J AUYZ]_bZbUc Ahk A A A A A A A A A Arra)r&rBr@ Transformer transform)r5rMras rremove_dropoutrdSsi ''HAAAAA-AAA >( # # - - / //r orig_moduler+inputsoutputscPtj}i|D]!}||j}||<"|D] }||fd}||<!|fd|D|tj||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. c|Sr1)xenvs rz"extract_subgraph..ls s1vrc g|] }| Srjrj).0outputrls r z$extract_subgraph..ns888fc&k888r)r&Graph placeholderr node_copyrplintrA) rer+rfrgrNinputnew_noderrls @rextract_subgraphrxbs I"$C((44E &&t-=-=-=-=>>D  8888888999 NN >+y 1 11rc*tj|Sr1) th_mkldnnMkldnnBatchNorm)a_s rrmrms!:1!=!=rci}|D]}|jdkrt|jtsJ||j}t |t vrqt t ||t j}t|tj sJtj |||<t||||S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r!) r(r%rr)r* mkldnn_mapr@floatr7Moduler>r?r4)r+r old_modulesr cur_moduler/s rmodules_to_mkldnnrs /1K?? 7m # #dk3// / / / -JJ:--'Z(8(89*ekRR !*bi88888*.- *C*C J'#D':>>> rrc|D]Q}|jdkrDt|jtsJ||j}||vrt ||||RdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. r!N)r(r%rr)r4)r+rrrrs r reset_modulesrsu LL 7m # #t{C00 1 1 1 -J[((#D';z3JKKK LLrc$eZdZdejfdZdS) MklSubgraphfx_graphc>||_g|_g|_g|_dSr1)rr+ start_nodes end_nodes)r[rs r__init__zMklSubgraph.__init__s#  $& *,(*rN)r]r^r_r&rrrrjrrrrs1+++++++rr rcHdddtdtffd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) NrErc|j} :|jj |jj t d|Dt ttj d|j D}t |j || fd}|fd}tjj t |fd}||kS)Nc@g|]}tj|jSrj)r@randnshaperors rrqz@gen_mkl_autotuner..use_mkl_heuristic..s$IIITTZ00IIIrc(g|]}|jdS)r)r#rs rrqz@gen_mkl_autotuner..use_mkl_heuristic..s*T*T*TD49Q<*T*T*TrctD] }| tj}tD] }|} tj|z Sr1)rangetime)fr}beginoutiterswarmups r benchmarkz?gen_mkl_autotuner..use_mkl_heuristic..benchmarksc6]]  IKKE5\\  acc9;;& &rc6ddDDS)Nc6g|]}|Srj)to_denserois rrqzRgen_mkl_autotuner..use_mkl_heuristic....s %o%o%oqajjll%o%o%orc6g|]}|Srj) to_mkldnnrs rrqzRgen_mkl_autotuner..use_mkl_heuristic....s!GmGmGmZ[ GmGmGmrrj sample_inputs submodulesrrmz>gen_mkl_autotuner..use_mkl_heuristic..s-%o%oIIGmGm_lGmGmGmgen_mkl_autotuner..use_mkl_heuristic..s =(Ar)rr owning_modulerr propagater r r&r'rrxr+rrErCrD) rE input_nodes output_argsrmkl_time no_mkl_timerrexample_inputsrMrrrs @@ruse_mkl_heuristicz,gen_mkl_autotuner..use_mkl_heuristics'  ~3H.4K h   ) ). 9 9 9II[III 4=*T*TEO*T*T*TUU $Xu{KUU  ' ' ' ' ' '9oooooppio+T)2I2I2K2K-L-LkZZZi A A A A ABB +%%r)rbool)rrrrrMrs``` @@rgen_mkl_autotunerrs[HK&&&&&&&&&&&&0 rrEc2t|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )r"r+)rEs ruse_mkl_lengthrs u{  a rc@eZdZdZdefdZdedefdZdedefdZd S) UnionFindc2dg|z|_dg|z|_dS)Nrrsize)r[ns rrzUnionFind.__init__s ,06A: !sQw rvc.||j|<d|j|<dS)Nrr)r[rs rmake_setzUnionFind.make_sets A ! rrc|j|}||kr|S|J|||j|<tt|j|Sr1)rfindr int)r[rpars rrzUnionFind.findsOk!n 88H3 ACQ(((rr|bc||||}}||kr|S|j||j|kr||}}||j|<|j|xx|j|z cc<dSr1)rrr)r[r|rs rjoinzUnionFind.joinszyy||TYYq\\1 66H 9Q<$)A, & &aqA A !  ! $ rN)r]r^r_rrrrrrjrrrrs'''#)c)c))))%c%c%%%%%%rr pass_configtracercdddtid}|i}|||drt|}|drt|}|ddur|St |dt st d d|dvrt d |dd}|}|tj |tj |j }t | }Gd d t}tjD]} |j} | jd kr|| j} t)| t*vrq|j} t/| d} | F| jt4jks Jd| jt5jdks Jdn6| jdkr+| jt*vr|j} n| jt:vr|j} | |jkr| |jkr t?d| j Ds!| 5tj"| j fd} dddn #1swxYwYtGtHtj%j&| | _ '| 5(dd| f}| )|| f|_ dddn #1swxYwYtUtj|}|_+jD]} | jdkr| jdkr| j d}t| j,}|D]B}|jdkr5|jdkr*|)|-|Ct]| j,dkr-| t]j}t_|fdtajD] \}} | jdkr(| jdkr|| _12|9| jdkr?| jdkr4| j dJ| j d| _3fd| j4D}t]|dkrt?d|DrJtk|}|d| _6|ddD]}7|d| tqfd}jD]} ts| dr8|:| j6j;| ts| dr8|:| j1j<;| ts| dr8|:| j3j=;| |>D]l}||s_|j<|j=zD]9} | j d}| )|-| :t|j||md}jD]} | jdks | jdkr|dz }tjAtCd |Dtj |}|S)!a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fuserdmkldnn_layout_optimizeNrrdrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configceZdZdZdZdZdS)*optimize_for_inference..MklSupportrrN)r]r^r_NOYESUNKNOWNrjrr MklSupportrs rrr!z)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functionc3,K|]}|jdkVdS)rN)r)roargs r z)optimize_for_inference..7s)II3:3IIIIIIrc2d|fS)Nr) call_method)rrs rrmz(optimize_for_inference..:sh>R>RS^ab`e>f>frrrrrct|dr|jSt|dr|jSdS)Ncolor start_color)hasattrrrr)rufs r get_colorz)optimize_for_inference..get_colorWsT 1g   $7717## # 1m $ $ *771=)) )trcpg|]2}t|tj|'|3Sr1)r%r&r')rorrs rrqz*optimize_for_inference..rsG{{{1 STVXV]H^H^{bkbklmbnbnbz))A,,bzbzbzrc3K|]}|duV dSr1rjrs rrz)optimize_for_inference..vs&9919999999rrc"tSr1)r)rsrrmz(optimize_for_inference..}s H@U@Urrr end_colorzmkldnn conversions: %s)ErupdaterRrdr%rC RuntimeErrortracer>r?r&rArootrDrlistr+rr(rr*mkldnn_supportedrnext parametersdtyper@rdevicemkldnn_supported_unknownranyr#inserting_beforemap_argr r rrinserting_after create_noderHrrrFrIr"r enumeraterrrall_input_nodessortedrrrrrappendrrvaluesrlogging getLoggerr]inforu)r5rrdefault_pass_configr cur_tracerrMrrrsupports_mkldnnrsample_parameter mkldnn_argsdense_xrprv_noderFuser num_nodescur_idx cur_colors other_color mkldnn_graphsrEprvmkldnn_conversionsresultrrrs @@@roptimize_for_inferencer sJ&#."?  {+++>*U +,&u%%34== )*BCT J JJHIII-.FGGGQRRR+,DEkRJ e 4 455H~jox88H$()<)<)>)>$?$?GTX^$$''$- 7m # # -JJ#333",.#' (=(=(?(?#F#F #/+1U[@@@Bm@@@+2el56I6IIIIKnIII W ' '{...",. 888","4 jm + +*"444IItyIIIII**400 h h j4f4f4f4fgg  h h h h h h h h h h h h h h hU27#34kBBDI))$// ' '"..}j4'RR**7333 $w  ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' $D$8$8'BBK&H * * 7m # # z(A(Ay|H$$E . .7m++ {0J0J..x888''---4:!####D)))HN##I 9  B&#8>2244  7m # # {(B(B&D  KK W % %$+*C*C9TYq\**666&Yty|44DNN{{{{0D{{{J:!##99j99999 9 9 9 ++J#ADJ)!""~ 4 4  1 {3333 4-88U8U8U8U,V,VMJJ 4 ! ! B "''$*-- . 4 ; ;D A A A 4 ' ' N "''$"233 4 @ G G M M M 4 % % J "''$.11 2 < C CD I I I %%''==  '' =)EO; * *il**3///##D)))) %+w < < <$$ ;+ % % )B)B ! #  h$$%=?QRRR MMOOO ^E8 , ,F Ms$4JJ" %J" -6L//L3 6L3 )FF)rr)Itorch.fxr& torch.fx.noderrtorch.nn.utils.fusionrtypingrrrr r r r r r@torch.nnr7torch.nn.functional functionalFtorch.fx.passes.shape_proprr> collectionsrtorch.utils.mkldnnutilsmkldnnrzoperatorrrenumrr)rr'r.rr4rRrdrxr:Linearr;ReLU MaxPool2d AvgPool2dAdaptiveAvgPool2drelu transposesigmoid avg_pool2dadaptive_avg_pool2draddmulr MkldnnConv2d MkldnnLinearrrrrrrrrTracerr rjrrr,sb********333333IIIIIIIIIIIIIIIIIIII 000000 ######&&&&&&&&& -#-%S/----HTN"'DQTVYQYN$4bg4S#X4EHO4444 ///58?////@ 0") 0 0 0 0 02")2DM24PRPW=2cghjhocp2222"Iry"."'2<rOc JFAL!/%L(,7Iy%Iy%N== T"']T#ry.5I$ Lbg Lc29n1E LTXY[YbdfdmYmTn L L L L++++++++$$$$L + $    %%%%%%%%8-1 iii 8?i$sCx.)i Oi X_ iiiiiir