Ng+dZddlZddlmZddlmZddlmZddlZddl m Z ej e Z GddZGdd e jZGd d e jZdS) zf Exponential Moving Average (EMA) of model updates Hacked together by / Copyright 2020 Ross Wightman N) OrderedDict)deepcopy)Optionalc&eZdZdZddZdZdZdS) ModelEmaa Model Exponential Moving Average (DEPRECATED) Keep a moving average of everything in the model state_dict (parameters and buffers). This version is deprecated, it does not work with scripted models. Will be removed eventually. This is intended to allow functionality like https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage A smoothed version of the weights is necessary for some training schemes to perform well. E.g. Google's hyper-params for training MNASNet, MobileNet-V3, EfficientNet, etc that use RMSprop with a short 2.4-3 epoch decay period and slow LR decay rate of .96-.99 requires EMA smoothing of weights to match results. Pay attention to the decay constant you are using relative to your update count per epoch. To keep EMA from using GPU resources, set device='cpu'. This will save a bit of memory but disable validation of the EMA weights. Validation will have to be done manually in a separate process, or after the training stops converging. This class is sensitive where it is initialized in the sequence of model init, GPU assignment and distributed training wrappers. H.?czt||_|j||_||_|r|j|t |jd|_|r|||j D]}| ddS)NdevicemoduleF) remaevaldecayr tohasattrema_has_module_load_checkpoint parametersrequires_grad_)selfmodelrr resumeps P/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/utils/model_ema.py__init__zModelEma.__init__&sE??     ' HKKvK & & &%dh99  *  ! !& ) ) )$$&& $ $A  U # # # # $ $ctj|d}t|tsJd|vrt }|dD]0\}}|jr|dsd|zn|}n|}|||<1|j |t ddSt ddS)Ncpu) map_locationstate_dict_emar module.zLoaded state_dict_emazAFailed to find state_dict_ema, starting from loaded model weights) torchload isinstancedictritemsr startswithrload_state_dict_loggerinfowarning)rcheckpoint_path checkpointnew_state_dictkvnames rrzModelEma._load_checkpoint4sZeDDD *d+++++ z ) )(]]N"#34::<< ) )1&01 X0F0FM9q==ADDD'(t$$ H $ $^ 4 4 4 LL0 1 1 1 1 1 OO_ ` ` ` ` `rct|do|j }tj5|}|jD]s\}}|rd|z}||}|jr| |j}| ||j zd|j z |zzt ddddS#1swxYwYdS)Nr r"r ?) rrr#no_grad state_dictrr'detachr rcopy_r)rr needs_modulemsdr0ema_vmodel_vs rupdatezModelEma.updateEsSuh//K8K4K ]__ N N""$$C H//117799 N N5&!A Aa&--//;=%jj j<&>&E&E&G&GIYIYI[I[IbIbIdId"e"e 7 7w;*%jj j<z#ModelEmaV2.update..|s4:>R$*_XYDY3YrrLrMrrs` rr=zModelEmaV2.update{s) U&Y&Y&Y&Y ZZZZZrc6||ddS)Nc|SNrB)rPrQs rrRz ModelEmaV2.set..s1rrSrTrUs rsetzModelEmaV2.set~s  Unn 55555rc|j|i|SrXr rargskwargss rforwardzModelEmaV2.forwardt{D+F+++r)rN) r>r?r@rArrMr=rYr_ __classcell__rHs@rrDrDSs,******777[[[666,,,,,,,rrDc"eZdZdZ dd ed ed ed edededeej dedeffd Z ddeedefdZ ej ddeefdZ d efdZd efdZej dZdZxZS) ModelEmaV3a Model Exponential Moving Average V3 Keep a moving average of everything in the model state_dict (parameters and buffers). V3 of this module leverages for_each and in-place operations for faster performance. Decay warmup based on code by @crowsonkb, her comments: If inv_gamma=1 and power=1, implements a simple average. inv_gamma=1, power=2/3 are good values for models you plan to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps), inv_gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 at 215.4k steps). This is intended to allow functionality like https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage To keep EMA from using GPU resources, set device='cpu'. This will save a bit of memory but disable validation of the EMA weights. Validation will have to be done manually in a separate process, or after the training stops converging. This class is sensitive where it is initialized in the sequence of model init, GPU assignment and distributed training wrappers. rrFr4UUUUUU?NTr min_decayupdate_after_step use_warmup warmup_gamma warmup_powerr foreachexclude_buffersc tt||_|j||_||_||_||_||_ ||_ | |_ ||_ | |_ |j N|t|j kr&d|_ |j|dSdSdS)NFr )rGrrr rrrgrhrirjrkrlr rmnextrr) rrrrgrhrirjrkr rlrmrHs rrzModelEmaV3.__init__s uoo   "!2$((  . ; "ve6F6F6H6H1I1I1P'P'P DL KNN&N ) ) ) ) ) # "'P'Prstepreturnc||jStd||jz dz }|dkrdS|jrBdd||jz z|j zz }tt ||j|j}n|j}|S)zN Compute the decay factor for the exponential moving average. Nrre)rmaxrhrirjrkminrg)rrprs r get_decayzModelEmaV3.get_decays <: 1dT33a788 1993 ? TD$5554;L:LLLEE4:..??EEJE rc||}|jr|||dS|||dSrX)rvrmapply_update_no_buffers_ apply_update_)rrrprs rr=zModelEmaV3.updatesVt$$   -  ) )% 7 7 7 7 7   ue , , , , ,rc|jrg}g}t|j|D]Y\}}|r+||||D||Zttdrtj ||d|z dStj ||tj ||d|z dSt|j|D]{\}}|r4| ||jd|z M|||j|dS)N_foreach_lerp_r4weightscalaralphar )rlrJr r6rKis_floating_pointappendr8rr#r{ _foreach_mul_ _foreach_add_lerp_rr )rrrema_lerp_valuesmodel_lerp_valuesr;r<s rryzModelEmaV3.apply_update_s < @ O " "%dk&<&<&>&>&E&E&G&GIYIYI[I[IbIbIdId"e"e ) )w**,,)#**5111%,,W5555KK((((u.// Z$_6GPRUZPZ[[[[[[#OEBBBB#O5FbSXjYYYYYY"%dk&<&<&>&>&E&E&G&GIYIYI[I[IbIbIdId"e"e @ @w**,,@KK $+ > >rEzKRRRRKK $+ > >????  @ @rct|j}t|}|jrat t drt j||d|z nzt j||t j||d|z nIt||D]8\}}| | |j d|z 9t|j | D]3\}}|| |j 4dS)Nr{r4r|r~rsrr )tupler rrlrr#r{rrrJrrr buffersr8) rrr ema_params model_paramsema_pmodel_pema_bmodel_bs rrxz#ModelEmaV3.apply_update_no_buffers_sW4;113344 U--//00 < Ou.// O$Zb5jQQQQQ#Ju====#J AINNNNN"%j,"?"? O Ow GJJdkJ::2: NNNN!$+"5"5"7"7II 8 8NE7 KK $+ 66 7 7 7 7 8 8rc(t|j|D]3\}}|||j4dSrF)rJr r6rKr8rr )rrr;r<s rrYzModelEmaV3.sets!$+"8"8":":"A"A"C"CUEUEUEWEWE^E^E`E`aa 8 8NE7 KK $+ 66 7 7 7 7 8 8rc|j|i|SrXr[r\s rr_zModelEmaV3.forwardr`r) rrerFr4rfNTFrX)r>r?r@rAfloatintboolrr#r rrvr5r=ryrxrYr_rarbs@rrdrds2""%&$"%"%-1 $)*** * # *  * * *U\***"******<hsmu&U]__--(3----_-@%@@@@08U8888"U]__88_8,,,,,,,rrd)rAlogging collectionsrcopyrtypingrr#torch.nnnn getLoggerr>r*rModulerDrdrBrrrs#######  ' H % %@N@N@N@N@N@N@N@NF/,/,/,/,/,/,/,/,d,,,,,,,,,,r