קgكddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddlm Z m Z ddl mZddlmZmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZddl mZ!dd l"m#Z#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,ej-d kZ.ej-dkZ/ej0e1Z2gdZ3Gdde4Z5de6deeddfdZ7dej8fdZ9dej8fdZ:dee6efde6de;fdZGddeZ?d ee?ee6e?ffd!e6dee6e?ffd"Z@eGd#d$ZAGd%d&e ZBGd'd(eBZCeGd)d*ZDGd+d,ej ZEd-e;fd.ZFd/e6d0ed1ee6efd2ee6ee;e;ffd3ee6e;fd4ee6e;fd5ee6e!jGfd6ejHddfd7ZIGd8d9eEZJGd:d;eEZKdS)<N)ABCabstractmethod) nullcontext) dataclassfield)IntFlag) synchronize) FrameType)AnyCallableDictOptionalSetTupleUnion)ProcessFailurerecord)redirect_stderrredirect_stdout)get_subprocess_handlerSubprocessHandler)TailLogwin32darwin) DefaultLogsSpecsSignalExceptionStdto_mapRunProcsResultPContext get_std_cmMultiprocessContextSubprocessContextLogsDest LogsSpecsc:eZdZdZdedejddffd ZxZS)rz Exception is raised inside the torchelastic agent process by the termination handler if the death signal got received by the process. msgsigvalreturnNcXt|||_dSN)super__init__r()selfr'r( __class__s i/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.pyr-zSignalException.__init__Es&  ) __name__ __module__ __qualname____doc__strsignalSignalsr- __classcell__r/s@r0rr?s^ CDr1rsignumframer)cztj|}tdtjd||)aTermination handler that raises exceptions on the main process. When the process receives death signal(SIGTERM, SIGINT), this termination handler will be invoked. It raises the ``SignalException`` exception that should be processed by the user code. Python does not terminate process after the termination handler is finished, so the exception should not be silently ignored, otherwise the process will never be terminated. zProcess z got signal: )r()r7r8rosgetpid)r;r<r(s r0_terminate_process_handlerr@Js>^F # #F GRY[[GGvGGPV W W WWr1c@tr tjStjS)z@Get the kill signal. SIGKILL for unix, CTRL_C_EVENT for windows.) IS_WINDOWSr7 CTRL_C_EVENTSIGKILLr1r0_get_kill_signalrFW""~r1c@tr tjStjS)zOGet the default termination signal. SIGTERM for unix, CTRL_C_EVENT for windows.)rBr7rCSIGTERMrEr1r0_get_default_signalrJ_rGr1dnprocswhatct|}tt|}||krt|d|d|dS)Nz), local rank mapping mismatch, expected: z , actual: )setkeysrange RuntimeError)rKrLrM actual_keys expected_keyss r0_validate_full_rankrUgszaffhh--Kf &&Mm## A A' A A3> A A   $#r1z^(\d:[0123],)*(\d:[0123])$z^[0123]$c deZdZdZdZdZeezZedede de e dfffdZ dS)rrvmr)cdtdtfd}tjt|r ||Stjt |rOi}|dD]5}|d\}}|||t|<6|St|dtdt d) z Example: :: from_str("0") -> Std.NONE from_str("1") -> Std.OUT from_str("0:3,1:0,2:1,3:2") -> {0: Std.ALL, 1: Std.NONE, 2: Std.OUT, 3: Std.ERR} Any other input raises an exception vr)cTtt|}|tvr|SdSr+)rint)r[ss r0to_stdzStd.from_str..to_stds(CFF ACxxxr1,:z does not match: or <>) r6rrematch _VALUE_REGEX_MAPPING_REGEXsplitr] ValueError)clsrYr_rKmir[s r0from_strz Std.from_str|s c c     8L" % % 6"::  Xnb ) )  "AXXc]] & &wws||1"F1II#a&& HNNNN^NNN r1N) r2r3r4NONEOUTERRALL classmethodr6rr r]rlrEr1r0rrvsk D C C )C#%tCJ/?(?"@[r1r val_or_maplocal_world_sizect|tr(tt ||Si}t |D]%}||tj||<&|S)a Certain APIs take redirect settings either as a single value (e.g. apply to all local ranks) or as an explicit user-provided mapping. This method is a convenience method that converts a value or mapping into a mapping. Example: :: to_map(Std.OUT, local_world_size=2) # returns: {0: Std.OUT, 1: Std.OUT} to_map({1: Std.OUT}, local_world_size=2) # returns: {0: Std.NONE, 1: Std.OUT} to_map({0: Std.OUT, 1: Std.OUT}, local_world_size=2) # returns: {0: Std.OUT, 1: Std.OUT} ) isinstancerdictfromkeysrQgetrm)rrrsmaprks r0rrsp*c""}}U#344jAAA'(( 1 1A^^Asx00CFF r1ceZdZUdZeeZeee fe d<eeZ eee fe d<eeZ eee fe d<eeZ eee fe d<eeZeee fe d<dS) r$zK For each log type, holds mapping of local rank ids to file paths. default_factorystdoutsstderrs tee_stdouts tee_stderrs error_filesN)r2r3r4r5rrvr}r r]r6__annotations__r~rrrrEr1r0r$r$s$eD999GT#s(^999#eD999GT#s(^999"'%"="="=Kc3h==="'%"="="=Kc3h==="'%"="="=Kc3h=====r1r$c eZdZdZdejejdfdeedeee e effdeee e effdee e ddf dZ e d e e e eeffdefd Zee defd ZdS) r%a8 Defines logs processing and redirection for each worker process. Args: log_dir: Base directory where logs will be written. redirects: Streams to redirect to files. Pass a single ``Std`` enum to redirect for all workers, or a mapping keyed by local_rank to selectively redirect. tee: Streams to duplicate to stdout/stderr. Pass a single ``Std`` enum to duplicate streams for all workers, or a mapping keyed by local_rank to selectively duplicate. Nlog_dir redirectsteelocal_ranks_filterr)c>||_||_||_||_dSr+ _root_log_dir _redirects_tee_local_ranks_filter)r.rrrrs r0r-zLogsSpecs.__init__s(%# #5   r1envscdS)aS Given the environment variables, builds destination of log files for each of the local ranks. Envs parameter contains env variables dict for each of the local ranks, where entries are defined in: :func:`~torchelastic.distributed.elastic.agent.server.local_elastic_agent.LocalElasticAgent._start_workers`. NrE)r.rs r0reifyzLogsSpecs.reifysr1cdSr+rEr.s r0 root_log_dirzLogsSpecs.root_log_dirs  r1)r2r3r4r5rrmrr6rr r]rr-rr$rpropertyrrEr1r0r%r%s$"&03*-(15 6 6# 6d38n,- 63S#X& ' 6 %SX. 6  6 6 6 6  3S#X&'       ^   c   ^X   r1r%c >eZdZdZdejejdfdeedeee e effdeee e effdee e ddf fd Z e defd Zdeed efd Zd e e e eeffdefd ZdefdZdedefdZxZS)rz Default LogsSpecs implementation: - `log_dir` will be created if it doesn't exist - Generates nested folders for each attempt and rank. Nrrrrr)c~|tjkr|stjd}nhtj|stj|dn2tj|rtd|dt ||||d|_ dS)N torchelastic_prefixTexist_okz log_dir: z is a file) r>devnulltempfilemkdtemppathexistsmakedirsisfileNotADirectoryErrorr,r- _run_log_dir)r.rrrrr/s r0r-zDefaultLogsSpecs.__init__s bj  N"*/BBBW^^G,, N Gd333337>>'**N,-L-L-L-LMMM )S2DEEE r1c*t|jSr+)r6rrs r0rzDefaultLogsSpecs.root_log_dir s4%&&&r1 rdzv_run_idc|ptjd}tj|dtj|d|}td||S)NrrTr_)rdirzlog directory set to: %s)rrr>rloggerinfo)r.rr base_log_dirrs r0 _make_log_dirzDefaultLogsSpecs._make_log_dirsgJ("2/"J"J"J  L40000&7&7&7\JJJ .444 r1rc6t|}i}|dkr |d}ntd|dd}|dd}d}|jt jkry|js ||j||_t j |jd|}tj |d t j ||jt jkr t j}t|j|}t|j|}|D]\} } || } | | z|| <d} t$t)|| } t$t)|| }i}i}i}t)|D]3} |t jkr9t j|| <t j|| <t j|| <d|| d <Lt j |t+| }t j||| }|t.jzt.jkr#t j |d | | <|t.jzt.jkr#t j |d || <|| }|t.jzt.jkr | | || <|t.jzt.jkr || || <|jrs| |jvrj| |vr|| d| |vr|| d| | | krt j| | <|| | krt j|| <t j |d}||| <td| |||| d <5t;| ||||S)a  Uses following scheme to build log destination paths: - `//attempt_//stdout.log` - `//attempt_//stderr.log` - `//attempt_//error.json` rz;Empty envs map provided when defining logging destinations.TORCHELASTIC_RUN_ID test_run_idTORCHELASTIC_RESTART_COUNT0attempt_T) ignore_errorsTORCHELASTIC_ERROR_FILEz stdout.logz stderr.logNz error.jsonz"Setting worker%s reply file to: %s)lenrwarningrxrr>rrrrjoinshutilrmtreerrrritemsrvrwrQr6mkdirrrnrorpoprr$)r.rrL global_envrun_id restart_countattempt_log_dirredirsts local_ranktee_std redirect_std SYS_STREAMr}r~rrrclogdirrdt error_files r0rzDefaultLogsSpecs.reifysT A::aJJ NNM    5}EE"'CSII !   + +$ S$($6$6t7I6$R$R! gll4+<>X>X>XYYO M/ > > > > K ( ( (   + + jO00 DIv & &$&88:: 8 8 J!*-L!-!7F:   --f z::--f z::&( &(  --+ I+ IJ"*,,*,* J'*,* J'*,* J'>@Z !:;;',,JHH!!!J'LSW,,*,',,w *M*MGJ'LSW,,*,',,w *M*MGJ'zNsw;#')).5j.AK +sw;#')).5j.AK +,9"$*BBB"[00# D999![00# D999z*j88.0j +z*j88.0j +W\\'<@@ *4 J' 8*j?IZ !:;;+{KPPPr1c Hd|jd|jd|jd|jd S)NzDefaultLogsSpecs(root_log_dir=z , redirects=z, tee=z, local_ranks_filter=)rrs r0__repr__zDefaultLogsSpecs.__repr__sW OT-? O OT_ O O9 O O373K O O O r1otherct|tsdS|j|jko/|j|jko|j|jko|j|jkS)NF)rurrrrr)r.rs r0__eq__zDefaultLogsSpecs.__eq__sj%!122 5  %"5 5 F5#33 F UZ' F(E,EE  r1)r2r3r4r5rrmrr6rr r]rr-rrrr$rrobjectboolrr9r:s@r0rrs"&03*-(15 !!#!d38n,-!3S#X& ' ! %SX. !  !!!!!!&'c'''X'Xc]hQ3S#X&'hQ hQhQhQhQT #     F  t         r1rceZdZUdZeeZeee fe d<eeZ eee fe d<eeZ eeefe d<eeZeeefe d<defdZd S) ra Results of a completed run of processes started with ``start_processes()``. Returned by ``PContext``. Note the following: 1. All fields are mapped by local rank 2. ``return_values`` - only populated for functions (not the binaries). 3. ``stdouts`` - path to stdout.log (empty string if no redirect) 4. ``stderrs`` - path to stderr.log (empty string if no redirect) r{ return_valuesfailuresr}r~r)c2t|jdkS)Nr)rrrs r0 is_failedzRunProcsResult.is_faileds4=!!A%%r1N)r2r3r4r5rrvrr r]r rrrr}r6r~rrrEr1r0rrs  %*E$$?$?$?M4S>???*/%*E*E*EHd3&'EEE#eD999GT#s(^999#eD999GT#s(^999&4&&&&&&r1rceZdZdZ ddedeeefdeee fdeeeeeffde de eeeff d Z dd Z ejdd Zejd e efd Zddeded e efdZejd eeeffdZejddejded dfdZ dde ejded dfdZdS)r a The base class that standardizes operations over a set of processes that are launched via different mechanisms. The name ``PContext`` is intentional to disambiguate with ``torch.multiprocessing.ProcessContext``. .. warning:: stdouts and stderrs should ALWAYS be a superset of tee_stdouts and tee_stderrs (respectively) this is b/c tee is implemented as a redirect + tail -f Nname entrypointargsr logs_specslog_line_prefixesc||_t|}||}t|j|dt|j|d||_||_||_|j|_|j|_|j |_ ||_ t||j tj||_t||jtj||_dS)Nr}r~)rrrrUr}r~rrrrrLrrsysstdout _stdout_tailrstderr _stderr_tail) r.rrrrrrrL logs_dests r0r-zPContext.__init__s T$$T** I-vyAAAI-vyAAA$   (  ( $0 # )'5F  $ )'5F  r1r)cBtjtjurtjtjt tjtjt tsHtjtjt tjtj t nt d| |j |jdS)z)@)@ @ @ M&.*D E E E M&-)C D D D J fm-GHHH fn.HIII NN^     !!! !!!!!r1ct)z?Start processes using strategy defined in a particular context.NotImplementedErrorrs r0rzPContext._start "!r1ct)aE Poll the run status of the processes running under this context. This method follows an "all-or-nothing" policy and returns a ``RunProcessResults`` object if either all processes complete successfully or any process fails. Returns ``None`` if all processes are still running. rrs r0_pollzPContext._polls "!r1rWtimeoutperiodc>|dkr|S|dkr tj}tj|z}tj|krC|}|r|Stj|tj|kCdS)a Wait for the specified ``timeout`` seconds, polling every ``period`` seconds for the processes to be done. Returns ``None`` if the processes are still running on timeout expiry. Negative timeout values are interpreted as "wait-forever". A timeout value of zero simply queries the status of the processes (e.g. equivalent to a poll). ..note: Multiprocessing library registers SIGTERM and SIGINT signal handlers that raise ``SignalException`` when the signals received. It is up to the consumer of the code to properly handle the exception. It is important not to swallow the exception otherwise the process would not terminate. Example of the typical workflow can be: .. code-block:: python pc = start_processes(...) try: pc.wait(1) .. do some other work except SignalException as e: pc.shutdown(e.sigval, timeout=30) If SIGTERM or SIGINT occurs, the code above will try to shutdown child processes by propagating received signal. If child processes will not terminate in the timeout time, the process will send the SIGKILL. rN)rrmaxsizetimesleep)r.rrexpiryprs r0waitz PContext.waits2 a<<::<<  Q;;kGw&ikkF""B  Jv    ikkF"" tr1ct)z@Return pids of processes mapped by their respective local_ranks.rrs r0pidsz PContext.pids!rr1 death_sigct)z Terminates all processes managed by this context and cleans up any meta resources (e.g. redirect, error_file files). rr.rrs r0_closezPContext._close&s "!r1c|st}||||jr|j|jr|jdSdS)ar Terminates all processes managed by this context and cleans up any meta resources (e.g. redirect, error_file files). Args: death_sig: Death signal to terminate processes. timeout: Time to wait for processes to finish, if process is still alive after this time, it will be terminated via SIGKILL. )rrN)rJrrstoprrs r0closezPContext.close.s} .+--I i 999   %   " " $ $ $   %   " " $ $ $ $ $ % %r1r+)r)N)rrWr)Nr)r2r3r4r5r6rr r r]rr%rr-rabcrrrrfloatrrr7r8rr rEr1r0r r s"7;! ! ! (C-(! 3: ! 3S#X&' !  ! $DcN3! ! ! ! F""""" """" "x/""""&&E&&h~>V&&&&P "d38n"""" """"d""""JL%%!&.1%CF% %%%%%%r1r std_rdcTts ts|stS||Sr+)rBIS_MACOSr)r  redirect_fns r0r!r!Cs2#X#V#}}{6"""r1rfnrrstdout_redirectsstderr_redirectsret_valsqueue_finished_reading_eventc||}||} ||} ||} ||} t| t} t| t}| D]\}}|tj|<| 5|5t ||}dddn #1swxYwYdddn #1swxYwY| ||dSr+) r!rrrr>environrputr)rrrrrrrrargs_env_ret_val_ stdout_rd stderr_rd stdout_cm stderr_cmkr[rets r0_wrapr"Jsm  E  D #H ,I ,I9o66I9o66I 1 1 !!I!!fRjj% !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! LL %%'''''s6>B7B  B7 B$ $B7'B$ (B77B;>B;ceZdZdZ ddededeeefdeeeeeffdede d e eeefffd Z d Z d e fd Zd e efdZd eeeffdZddejded dfdZxZS)r"z<``PContext`` holding worker processes invoked as a function.Nrrrr start_methodrrc*t|||||||_fdtjD_i_d_tj j _ dS)Nchi|].}|tjj/SrE)mp get_contextr$ SimpleQueue.0rr.s r0 z0MultiprocessContext.__init__..sD    t'899EEGG   r1) r,r-r$rQrL _ret_vals_return_values_pcr'r(Event_worker_finished_event) r.rrrrr$rrr/s ` r0r-zMultiprocessContext.__init__ks           )    #DK00    /104')nT5F&G&G&M&M&O&O###r1c |jrtdtjt|j|j|j|j|j |j |j f|j dd|j |_dS)NzWThe process context already initialized. Most likely the start method got called twice.F)rrrLrdaemonr$)r/rhr'start_processesr"rrrr}r~r-r1rLr$rs r0rzMultiprocessContext._starts 8 B %    +;*   r1r)c<t|j|jkSr+)rr.rLrs r0_is_donezMultiprocessContext._is_dones4&''4;66r1c 4|jJ |jdtd|jD]?}|j|}|s||j|<@|r|j |js3t d|j3t|j|jd|t|j|j|jSdS#t$jt$jf$r}|j}|jj}|jj|}|j|}td|j||j||j|t|t=||j|j|i|j|jcYd}~Sd}~wwxYw) Nrrz>entrypoint fn finished, waiting for all child procs to exit...zreturn_value queue)rr}r~zKfailed (exitcode: %s) local_rank: %s (pid: %s) of fn: %s (start_method: %s)rpidexitcoderrr}r~)r/rrQrLr-emptyrxr.r6r1rOrdebugrUr rr}r~r'ProcessRaisedExceptionProcessExitedException error_indexrr4 processesr exceptionr:r9r$r)r.r return_queueefailed_local_rankfn_name failed_procerror_filepaths r0rzMultiprocessContext._pollsQx###J  HMM"   $At{33 I I #~j9 #))++I6B6F6F6H6HD' 3}} +//111 (--//LLX(--// $'6J %"&"5 L L t)2+DE    ! o2G(,->?K!-.?@N   0$!!    JJLLL!%~#4E!,!5#1 (((           ) sD7EH B,H HHcz|jJtt|jSr+)r/rv enumeraterrs r0rzMultiprocessContext.pidss0x###Idhmmoo..///r1rrrc|jsdS|jjD]h}|rRtd|j|j tj|j|X#t$rYdwxYwitj |z}|jjD]5}|tj z }|dkrn| |6|jjD]}|rftd|j|t tj|jtn#t$rYnwxYw| dS)Nz Closing process %s via signal %sr?Unable to shutdown process %s via %s, forcefully exiting via %s)r/rAis_aliverrr9rr>killProcessLookupErrorr monotonicrrF)r.rrprocend time_to_waits r0rzMultiprocessContext._closesx  FH&  D}} 6).GDHi0000)D n(H& $ $D!1!11Lq   IIl # # # #H&  D}} UH$&&  GDH&6&8&89999)D IIKKKK  s$A.. A;:A;(&E EEr+r )r2r3r4r5r6r r r]rr%rr-rrr6rrrr7r8rr9r:s@r0r"r"hskFF7;PPPP3: P 3S#X&' P  PP$DcN3PPPPPPB   .7$7777Mx/MMMM^0d38n0000""""d""""""""r1r"ceZdZdZ ddededeeefdeeeeeffdede eeeff fd Z d Z d e e fd Z d eeeffd Zddejded dfdZxZS)r#z:``PContext`` holding worker processes invoked as a binary.Nrrrrrrct||||||tt|j|_i|_i|_dSr+)r,r-rOrQrL_running_local_ranks _failuressubprocess_handlers)r.rrrrrrr/s r0r-zSubprocessContext.__init__!sd           /2% 2D2D.E.E!46AC   r1c~jrtdfdtjD_dS)Nz[The subprocess handlers already initialized. Most likely the start method got called twice.c i|]I}|tjj|j|j|j||JS))rrenvrr local_rank_id)rrrrr}r~r*s r0r,z,SubprocessContext._start..=sm $ $ $  .?Yz*Ij)|J/|J/(  $ $ $ r1)rXrhrQrLrs`r0rzSubprocessContext._start8s`  # m  $ $ $ $ $DK00 $ $ $    r1r)ct}|jD]u}|j|}|j}|K|||dkr0t ||jj||j||j |<v|j ||jr|j r| t|j |j |j}|r\t!|jd}t&d|j|j|j|jn1t0t5|j|_|SdS)Nrr8r;c|jSr+) timestamp)fs r0z)SubprocessContext._poll..dsAKr1)keyz.ws2    B     r1)rXrrs r0rzSubprocessContext.pidsvs2  "&":"@"@"B"B    r1rrrcJ|jsdS|jD]\}|jAtd|jj|j||]tj |z}|jD]P}|tj z }|dkrn1 |j |<#tj $rYMwxYw|jD]}|jntd|jj|t|t|j dS)Nz$Sending process %s closing signal %s)rrrL)rXrgrQrcrrr9rr rrPr subprocessTimeoutExpiredrF)r.rrrjrRrSs r0rzSubprocessContext._close|s'  F/6688 3 3G|  ""*:L$N  222n(/6688  G!1!11Lq    !!,////,    /6688 $ $G|  ""*UL$$&&   (8(:(: ;;; !!### $ $sC++C=<C=r+r )r2r3r4r5r6r r]rr%rr-rrrrr7r8rr9r:s@r0r#r#s?DD7;DDDD3: D 3S#X&' D  D$DcN3DDDDDD.   "+x/++++Z d38n    $$$$d$$$$$$$$r1r#)Lr loggingr>rcrr7rqrrrrrr contextlibr dataclassesrrenumrmultiprocessingr typesr typingr r r rrrrtorch.multiprocessingr'0torch.distributed.elastic.multiprocessing.errorsrr3torch.distributed.elastic.multiprocessing.redirectsrrrs^    ########""""""((((((((''''''CCCCCCCCCCCCCCCCCC""""""SSSSSSSSGFFFFF\W $ <8 #  8 $ $   i Xs X8I3F X4 X X X X&.V^ 4S> 3 c    / $$$$$'$$$Nc4S>)*>A #s(^0  > > > > > > >  >, , , , , , , , ^_ _ _ _ _ y_ _ _ D &&&&&&& &,V%V%V%V%V%swV%V%V%r#s####((( sEz ( sDcN" # ( 38n ( 38n (3&'(#."3( ((((