קg[ ddlZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZmZmZmZddlZddlmZddlmZmZddlmZdd lm Z m!Z!m"Z"dd l#m$Z$erdd lm%Z%gd Z&ej'e(Z)Gd de Z*e*j+Z+e*j,Z,e*j-Z-e*j.Z.e*j/Z/e*j0Z0e*j1Z1e*j2Z2e*j3Z3e+Z4e,Z5e-Z6ej7dZ8GddeZ9dee:eee9fde;fdZ< d=dee:eee9fde:de:de=fdZ>GddeZ?d>deej@dee;fdZA d>deej@dee;dee:ej%ffdZBGd d!e?ZCGd"d#eCZDGd$d%eCZEGd&d'eCZF d?d)eee9d*e:dee9fd+ZGd)ee:ee9fd,e e:ge:fde:dee:ee9ffd-ZHGd.d/e?ZIGd0d1eIZJGd2d3eIZK d@d4ZLGd5d6eIZMGd7d8eIZNGd9d:eNZOd;e;fd<ZPdS)AN)ABCabstractmethod) defaultdict)Enum) AnyCallableDictList NamedTupleOptionalSetTuple TYPE_CHECKINGUnion) FSDPModule UnshardHandle)record_function) merge_chunkssplit_args_kwargs_into_chunksTensorChunkSpec)_PipelineStageBase)Work) get_schedule_classPipelineScheduleSinglePipelineScheduleMulti Schedule1F1BScheduleFlexibleInterleaved1F1B ScheduleGPipeScheduleInterleaved1F1BScheduleLoopedBFSScheduleInterleavedZeroBubblecNeZdZdZdZdZdZdZdZdZ dZ d Z d Z e d Zd S) _ComputationTyper ctjdtjdtjdtjdtjdtjdtjdtjdtj d i }||S) NFBWUNSHARDRESHARDSEND_FRECV_FSEND_BRECV_B) r$FORWARDBACKWARDWEIGHTr1r2r3r4r5r6)selfstr_maps b/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/pipelining/schedules.py__str__z_ComputationType.__str__?s]  $c  %s  #S  $i  $i  #X  #X  #X  #X  t}cj|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|dkr tjS|d kr tj Std |) Nr.r/r0r1r2r3r4r5r6zInvalid computation type ) r$r7r8r9r1r2r3r4r5r6 RuntimeError)actions r<from_strz_ComputationType.from_strMs S==#+ + s]]#, , s]]#* * y #+ + y #+ + x  #* * x  #* * x  #* * x  #* *C6CCDD Dr>N)__name__ __module__ __qualname__r7r8r9r1r2r3r4r5r6r= staticmethodrBr>r<r$r$3spGH FGG F F F F   EE\EEEr>r$zD(\d+)([F,B,W]|UNSHARD|RESHARD|SEND_F|RECV_F|SEND_B|RECV_B{0,1})(\d*)cZeZdZUeed<eed<dZeeed<dZe dZ dS)_Action stage_indexcomputation_typeNmicrobatch_indexct|j}|t|jz }|j|t|jz }|SN)strrJrKrL)r:reprs r<__repr__z_Action.__repr__sJ4#$$ D)***  , C-.. .D r>cpt|x}rl|\}}}tt |t |t|rt |ndS|dks|rdStd|d)z Reverse of __repr__ String should be formatted as [stage][action type][(microbatch)] e.g. `2F0`, `1UNSHARD`, `3SEND_F1` NzInvalid action string: zD, should be formatted as [stage][action type][(microbatch)] e.g. 2F0) _action_regexmatchgroupsrIintr$rBlenisspacer@)rOrUrJrKrLs r<rBz_Action.from_strs"'',, ,5 >Cllnn ;K)+;K   ))*:;;),-=)>)>H$%%%D  BYY#++--Y4 oc o o o   r>) rCrDrErW__annotations__r$rLr rQrFrBrGr>r<rIrIzsm&&&&&*hsm***  \   r>rIpipeline_orderreturnc tdD fdt D} fdtD}t t j|ddi}t}dt|D}dt|g|RD dt|d d zzd fd t|Dz} fd t||D}|d zd |zd z}|S)z} Formats the pipeline order in a timestep (row) x rank (column) grid of actions and returns the formatted string c34K|]}t|VdSrNrX).0actionss r< z)_format_pipeline_order..s(HHWCLLHHHHHHr>c g|]D}dt|ttdz zES)zStep r)rOzfillrX)r`i num_stepss r< z*_format_pipeline_order..sQ<=#a&&,,s3y1}#5#566777r>cDg|]}|dgzS)rS)get)r`keyrfr[s r<rgz*_format_pipeline_order..s=693y 011r> fillvaluerSc2g|]}dt|zS)zRank rO)r`res r<rgz*_format_pipeline_order..s#>>>7SVV#>>>r>c@g|]}td|DS)c3VK|]$}|tt|ndV%dS)Nr)rXrO)r`items r<rbz4_format_pipeline_order...s7 F F$d.CD NNNA F F F F F Fr>)max)r`cols r<rgz*_format_pipeline_order..s?  F F# F F FFFr> rr%c38K|]\}}|d|VdS.sJ<<)1E5$;q>$$$<<<<<<r>c ~g|]9\}}|ddfdt|Dz:S): rsc3RK|]!\}}t|d|V"dSrurm)r`rerprxs r<rbz4_format_pipeline_order...s?RR4c$ii3+a.333RRRRRRr>)join enumerate)r`rwrowrxs r<rgz*_format_pipeline_order..se E3  ((RRRR9S>>RRR R R Sr> ) rqvaluesrangesortedlist itertools zip_longestrXzipr|r}) r[ step_labels rank_actionstransposed_actions num_ranks rank_labels header_rowformatted_rowsformatted_tablerxrfs ` @@r<_format_pipeline_orderrs HH0E0E0G0GHHHHHIAFyAQAQK=CN=S=SLi3\PRPPQQN##I>>U9-=-=>>>K{8%7888K KN++a/0388<<<<5>{5K5K<<<44Jk+=>>N !4'$))N*C*CCdJO r>Fnum_microbatches num_stagesenable_zero_bubblec 0i}td|D}t|D]}g}g}tt|D]Y} |t|| kr|| |nd} | ,| j} | t jkr|| Zd|D} t| t|kr|d|D]l} | j} | j} | j }| Jd||kr|d|d||vr6| t j ks| dkr|d || | f||<}||\}}|t j kr-||d z kr|d z }t j }nh|d z}t j }nV|t j kr3|dkr|d |d nx|d z }t j }ntd |d|$|| kr|d |d|d| || kr|d |d|d| ||f||<n|s?t|dkr)td|dd|zdStt|D]} t}t}|| D]} | | j} | j} | j }| t j kr|||| fD| t jkr]|| f|vr|d|d| d|| f|vr|d|d| d|||| ft|t|kr|dt|dkr)td|dd|zdS)a6 pipeline_order[rank] = [(computation_type, microbatch_index, stage_index), ...] Validating that the pipeline order follows the rules: 1. Forward action for a microbatch must be before the Backward action for that microbatch 2. Recv for a microbatch must be before the send for that microbatch 3. Microbatch index is handled in sequential order for each stage 4. A later stage cannot operate on a microbatch before any of the previous stages have operated on it 5. Same microbatch cannot be handled in the same time step across ranks c34K|]}t|VdSrNr_)r` rank_lists r<rbz+_validate_pipeline_order..s(OO)s9~~OOOOOOr>Nch|] }|j SrG)rL)r`rAs r< z+_validate_pipeline_order..s(% % % (.F #% % % r>z>>,33F;;;% % 2J% % % ! ( ) )S1I-J-J J J   N   /6 6 F ,K%6 .H$$T%$$+++  !LX!L!L!LMMM666#'7'???;RSCSCS$$%Qx%Q%QRRR5E{4S'110Gx/P, *#'7'???!Z!^33)3a/?/H,,)3a/?/G,,%)9)BBB!Q!((RRRR)3a/?/H,,$L,<LLL(3+/???!((ccc-AccP`cc"[00$$THTTTTkTT )"5'11 " 9~~"""55558K8KK FF#n--.. L LD3655N14L(. B B>$0 #)#: !2#'7'@@@+&**Hk+BCCC%)9)@@@ +.nDD!((UxUUKUUU!+.,>>!((QxQQKQQQ +$(((K)@AAA>""c,&7&777  !JKKK y>>Q  @H@@@388ICVCVVWW W u{X{Xr>c"eZdZ ddedeedejfdeee dfdee e e fdee e e e fee ff dZd Zd Zd Ze dd eed eedeedeefdZeddddeefdZ dd eed eedeedeefdZdZ ddee dfdee e e ffdZdee de fdZdS)_PipelineScheduleNn_microbatchesloss_fn.args_chunk_speckwargs_chunk_specoutput_merge_specc||_||_||_||_||_ |jdu|_g|_td|j j dS)NzUsing %s) _n_microbatches_loss_fn_args_chunk_spec_kwargs_chunk_spec_output_merge_spec _has_backward_internal_lossesloggerinfo __class__rC)r:rrrrrs r<__init__z_PipelineSchedule.__init__Tsj . /"3"3 "]$657 J 788888r>c|jr?|jr:||||}|j|dSdSdSrN)is_lastr _compute_lossrr)r:stageoutput target_mbsrlosss r<_maybe_compute_lossz%_PipelineSchedule._maybe_compute_lossqsa = /T/ /%%fj.BCCD  ! ( ( . . . . . / / / /r>cd|cxkot|jknc}|jr|jr|r |j|St|jdkr|st d|d|jdS)NrzLoss for microbatch z6 is not available. Available losses for microbatches: )rXrrrr@)r:rr valid_indexs r<_maybe_get_lossz!_PipelineSchedule._maybe_get_lossvs8@@@@c$*?&@&@@@@@ = T/ K (2 2 & ' '1 , ,[ ,NxNN6:6KNN  4r>ct|ts|g}td|D}|ry|wt|j|jkr,t d|jdt|j|||j|jdS)zB Update the losses to those in the internal state c3$K|] }|jV dSrN)rr`rs r<rbz3_PipelineSchedule._update_losses..s$!D!DE%-!D!D!D!D!D!Dr>N Expecting z losses but got ) isinstanceranyrXrrr@clearextend)r:stageslossescontains_last_stages r<_update_lossesz _PipelineSchedule._update_lossess &$'' XF!!D!DV!D!D!DDD  16#54())T-AAA"c!5ccs4K`GaGacc LLNNN MM$/ 0 0 0 ##%%%%%r>arg_mbs kwarg_mbsrrct)z Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the schedule implementation. Args: microbatches: list of microbatch args. NotImplementedError)r:rrrrs r<_step_microbatchesz$_PipelineSchedule._step_microbatchess "!r>targetrct) Run one iteration of the pipeline schedule with *whole-batch* input. Will chunk the input into microbatches automatically, and go through the microbatches according to the schedule implementation. args: positional arguments to the model (as in non-pipeline case). kwargs: keyword arguments to the model (as in non-pipeline case). target: target for the loss function. losses: a list to store the losses for each microbatch. r)r:rrargskwargss r<stepz_PipelineSchedule.steps "!r>cdtffd }| ||dn dgjz}| ||dn igjz}| ||d|4t|tst dt |||fS) z* Pre-process/check inputs namec t|ts!t|dt|t |jkr*t djd|dt |dS)Nz must be a list but got a rrsz but got )rr TypeErrortyperXrr)mbsrr:s r<check_type_and_lenz;_PipelineSchedule._check_inputs..check_type_and_lensc4(( P4 N N499 N NOOO3xx4/// Q!5QQQQs3xxQQ0/r>NrrGrrz losses must be a list but got a )rOrrrrr)r:rrrrrs` r< _check_inputsz_PipelineSchedule._check_inputss #          w 2 2 2 2dT11G   y+ 6 6 6 6t33I  !  z< 8 8 8  fd++ S Q4<< Q QRRR !!r>c.|||SrN)r)r:rrs r<rz_PipelineSchedule._compute_losss}}VV,,,r>rrc|s|r)t|||j|j|j\}}||fSdg|jzig|jzfS)zj Splits a full-batch input into chunks (i.e. microbatches) and returns the chunks rG)rrrr)r:rr args_split kwargs_splits r< _split_inputsz_PipelineSchedule._split_inputsso  L6 L'D$%' (( $J |+ +4$..t7K0KK Kr> output_chunksr\c,t||jS)z Merge output chunks back to a batch state. If output_merge_spec is None, the utility will merge output chunks by dimension 0 (batch dim). )rr)r:rs r<_merge_outputsz _PipelineSchedule._merge_outputss    #   r>NNNNrN)rCrDrErWr rtorchTensorrrr rOrrrrrrrr rrrrrrrGr>r<rrSsd:>AEBFIM 999(3 #4569"%(<"=> 9 $Do)=$>? 9 $E$sCx.%**D$EF 9999:///   &&&.#'$(%)!% ""$"D>"TN "  """^""!% " " "x~ " " "^ "#'$(%)!% $"$"$$"D>$"TN $"  $"$"$"$"L--- ,0LLCHoLc3h(LLLL. DI #      r>rp2p_opsdescct|dkrdS|r|dnd}td||tj|S)zt Simple wrapper over batch_isend_irecv from torch.distributed, which just adds a descriptive logger on top. rNz, rSzbatch_p2p %s%s)rXrdebugdistbatch_isend_irecvpop)rrdesc_strs r< _batch_p2prsb 7||qt"*${{{{H LL!8W555  !' * * . . 0 00r>ctt}i}t|dkr|S|D]"}||j|#t |D]\}}t||||<|S)z Sorts the list of P2P ops by the peer rank, and then calls batch_isend_irecv. Return a dictionary of works by peer rank. This function helps us avoid hangs in case of skip connections. rr)rrrXpeerrritemsr)rr ops_by_peer work_by_peeropropss r<_sorted_batch_p2prs0;4/@/@K)+L 7||q((BG##B''''K--//0088 c'$777 T r>ceZdZdZ ddededeedeee dfdee e e fd ee e e e fee ff fd Zddd d eefd ZxZS)rz Base class for single-stage schedules. Implements the `step` method. Derived classes should implement `_step_microbatches`. Nrrrr.rrct|||||||_|j|_|j|j_|||jr||dSdS)Nrrrrr) superr_stager _num_stagesr has_backward_prepare_forward_infra_prepare_backward_infra)r:rrrrrrrs r<rzPipelineScheduleSingle.__init__1s )+//      +#'#5   $$^444   :  ) ). 9 9 9 9 9 : :r>rrcH|j|||\}}|(tt j||j}nd}||||||jjr| |jj SdSrN) r clear_runtime_statesrrr tensor_splitrrrrr)r:rrrrrr targets_splits r<rzPipelineScheduleSingle.stepNs ((***$(#5#5dF#C#C L   !3FDr)rCrDrE__doc__rrWr rrrr rOrrrr r __classcell__rs@r<rr*s'+AEBFIM::!::(# : "%(<"=> : $Do)=$>? :$E$sCx.%**D$EF:::::::"&x~r>rc deZdZdZ ddeedeedeedeefdZdS) _ScheduleForwardOnlyzo The forward-only schedule. Will go through all the microbatches and perform only the forward pass Nrrrrc ||td|||||\}}g}t|jD]#}t d|5|j|}t|d}|D]} | |j ||||||j |}t|d}| |dddn #1swxYwYtd|jj|%|D]} | dS)z< Run one iteration of the pipeline schedule Nz7Forward-only schedule does not support loss computationForward fwd_recvrfwd_send[%s] Forwarded microbatch %s)r@rrrrr get_fwd_recv_opsrrwaitforward_one_chunkget_fwd_send_opsrrrrJ) r:rrrrfwd_sends_to_waitrerworksworks r<rz'_ScheduleForwardOnly._step_microbatchesvs  !V%7I "//JPVWW.0t+,, U UA A00 9 9k22155)#J???!LLNN  DIIKKKK --aYq\JJJk22155)#J???!((888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 LL79PRS T T T T &  D IIKKKK  sCD77D; >D; rrCrDrErr r rrGr>r<rrps{#'$(%)!% (($(D>(TN (  ((((((r>rc deZdZdZ ddeedeedeedeefdZdS) rz^ The GPipe schedule. Will go through all the microbatches in a fill-drain manner. Nrrrrc,|||||\}}g}t|jD]@}td|5|j|}t |d}|D]} | |j |||||} |j |}t |d}| |dddn #1swxYwYt d|jj|||j| ||B|D]} | |jsdSg} t|jD]2}td|5|j|}t |d}|D]} | ||j|} |j|| |j|}t |d }| |dddn #1swxYwYt d |jj|4||j|| D]} | dS) z Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the GPipe schedule. Args: microbatches: list of microbatch args. rrrrNr z Backward bwd_recvrbwd_sendz[%s] Backwarded microbatch %s)rrrrr r!rrr"r#r$rrrrJrrget_bwd_recv_opsrbackward_one_chunkget_bwd_send_opsr) r:rrrrr%rerr&r'rbwd_sends_to_waitrs r<rz ScheduleGPipe._step_microbatchess"//JPVWW.0t+,, I IA A00 9 9k22155)#J???!LLNN  DIIKKKK66q'!*iPQlSSk22155)#J???!((888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 LL79PRS T T T  $ $T[&*a H H H H &  D IIKKKK!  F.0t+,, V VA Q11 9 9k22155)#J???!LLNN  DIIKKKK++DK;; ..qt.<<<k22155)#J???!((888 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 LL8$+:QST U U U U DK000&  D IIKKKK  s&CD$$D( +D( C J--J1 4J1 rr(rGr>r<rrs#'$(%)!% GG$GD>GTN G  GGGGGGr>rc deZdZdZ ddeedeedeedeefdZdS) rzo The 1F1B schedule. Will perform one forward and one backward on the microbatches in steady state. Nrrrrc:|||||\}}t|j|j|jjz }d}d}d}d} g} t |D]} |j|} t| dx} r| |j |||||}| r| |j |} ||dz krt| d} | |j||||dz } |j |}t| |zdx}r| ||j|}|j|| |j|}|dz }||jkrn|j|} t|| zd x}r| |j |||||}| |j||||j |} |dz }Pt|d } ||jkr|j |}t|d x} r| ||j|}|j|| | r| |j|}t|d } |dz }||jk| r| ||j|dS) z Run one iteration of the pipeline schedule with list of microbatches. Will go through all the microbatches according to the 1F1B schedule. Args: microbatches: list of microbatch args. rNrrrrTfwd_send_bwd_recvr,bwd_send_fwd_recvr-r+)rminrrr rJrr!rr"r#r$rr.rr/r0r)r:rrrr warmup_chunks fwd_mb_index bwd_mb_indexweight_stage_mb_index send_work fwd_sends_ fwd_recvs recv_workr bwd_recvs fuse_workr bwd_sendss r<rzSchedule1F1B._step_microbatchess"//JPVWW   t{6 6    !  }%%  A 44\BBI&yzBBBBy !   [22<AVXabnXoppF  !    44\BBI}q000&yzBBB  $ $T[&*l S S S A LL #  44\BBI'y9'>>I A L#T111(   NN    DK00000r>rr(rGr>r<rrs#'$(%)!% 11$1D>1TN 1  111111r>rr&compute_actionsmax_active_stagesc dtdtttdttfd}t g dtf fd }dtf fd }t |D]\}}|||||d t t fd  }t t fd  }|D] } || |D] } ||  | S) aRGiven a basic schedule involving only compute actions (F,B,W), add UNSHARD/RESHARD actions for FSDP. UNSHARD refers to fetching the full contents of an FSDP-sharded layer, requiring an all-gather operation. RESHARD does the opposite, releasing memory (but doing no commmunication) We abandon the "timestep lock" during lowering max_active_stages controls how many prefetches we allow. It should be measured in mb and tuneable but in practice 3 stages is probably the thing we want? (to account for having one f and one b active, and something else prefetching?) count next_actionsr\ct}g}|D]V}|R|j|vrI||j||jt ||krnW|S)zdRemove duplicates (same stage, different microbatch), find next 'count' stages that will do compute.)rrJrrrX)rFrGseenretas r<next_stage_indicesz0_add_unshard_reshard..next_stage_indicessv  A}d!:!:''' 1=)))s88u$$E r>rJc|t|tddSrN)rrrIr1rJ active_stagesfsdp_aware_actionss r<_unshardz&_add_unshard_reshard.._unshards=+&&&!!'+w"E"EFFFFFr>c|t|tddSrN)removerrIr2rNs r<_reshardz&_add_unshard_reshard.._reshards=[)))!!'+w"E"EFFFFFr>Nc |vSrNrG)srOs r<z&_add_unshard_reshard..s a}&<r>c |vSrNrG)rVnext_ns r<rWz&_add_unshard_reshard..s avor>) rWr r rIrr}rfilterr) rCrDrLrQrTrerAfetchevictrrOrPrYs @@@r<_add_unshard_reshardr]ys  "&x'8"9  c    "eeM(*GcGGGGGGGGcGGGGGGG//** 6 > $#$5qrr7JKKV<<< stage_to_rankc< d|D}dtdtffd dtdtttff fd }dttdttdtffd }|rd}t t |D]}t ||d ksJ||d }||||s>|x||| |rR||\} } ||| ||| j| || d t ||d kr||=d }|s Jd ||S) Nci|]}|gSrGrG)r`rs r< z"_add_send_recv..s-S-S-S4dB-S-S-Sr>rAr\cz|jtkr|jdz kS|jtkr |jdkSdS)NrrF)rKr.rJr/)rArs r< _has_commsz"_add_send_recv.._has_commssD  "a ' '%a7 7  $ ) )%* *ur>c0|s J|d|j}|j}|j}t||tkrt nt |}|tkr|dzn|dz }t||tkrtnt|}||fS)Nz is not a valid comm actionr) rJrKrLrIr.r3r5r4r6)rA stage_idxctypemb_idxsendrecv_stage_idxrecvrcs r< _get_commsz"_add_send_recv.._get_commssz&!!IIf#I#I#IIII& '(yEQJJ&&FFKK*/1**Q)a-~!vvPPTzr> prev_actionscn|dS|jtkrF|jdks;t|j|jtkrtnt |j}||vS|jtkrI|jdz ks;t|j|jtkrtnt |j}||vSdS)aWe don't put our own recv ops in the schedule, we let a sender on another rank put our recv ops in place. This helps ensure a sane (non-hanging) ordering of sends and recvs. But it also means we might not be able to schedule our next compute action yet. NTrr)rKr.rJrIr4r6rLr/)rArl expected_recvrs r<_ready_to_schedulez*_add_send_recv.._ready_to_schedules >4  $ ) )&2D2I2I#" 1Q66F'M !L0 0  $ ) )&2D UV2V2V#" 1Q66F'M !L0 04r>FrTz6Malformed compute schedule, can't schedule sends/recvs) rIboolrr r rrXrrJr) rCr^r comm_actionsrkroprogressrrArhrjrcs ` @r<_add_send_recvrss" .T-S?-S-S-SL7t7uWg-='>!15g 4 R#o..//  Dt,--1111$T*1-F%%fl4.@AA !T"))&111:f%%O!+F!3!3JD$!&--d333 t/?!@!@AHHNNN D ! % %a ( ( (?4())Q..#D)HHQQQQQQ/ R0 r>c|eZdZdZ ddeededeedee e dfd ee e e fd ee e e efe efd ee eefd effd ZdZdZddZddddeefdZ ddeedeedeedeefdZxZS)rzQ Base class for multi-stage schedules. Implements the `step` method. NTrrrr.rrstage_index_to_group_rankuse_full_backwardc Pt|dkrtdt|t||||||_|dj_|dj_|dj _ |jD] } || _ |dj _ jD]} j | _ fd_i_|_jD]3} | |j r| |4dS)Nrz9Multi-stage schedule expects at least two stages but got r rc$|jojduSrN)rr)rr:s r<rWz0PipelineScheduleMulti.__init__..9s%-EDM,Er>)rXrr r_stagesrr group_size pp_group_size group_rankrrurr_should_compute_lossr[rvrr) r:rrrrrrrurvrrs ` r<rzPipelineScheduleMulti.__init__sr v;;!  YCPVKKYY  )+//     !!9/#AY11I( $ 0 L L2K//)/)L&\ 4 4E!%!3E   F E E E ! CE!2\ > >E  ( ( 8 8 8! >--n=== > >r>ct|dd5}tj|}|jD]"}||j|# ddddS#1swxYwYdS)zQDump a CSV representation of the schedule into a file with the provided filename.wrSnewlineN)opencsvwriterr[writerowr:filenamecsvfilerrs r< _dump_csvzPipelineScheduleMulti._dump_csvGs (C , , , ;Z((F+ ; ; 3D 9:::: ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s?A  A$'A$cdttttdzfdtdtffd }t jjks'Jdjdt jtjD]}|jvs Jd||jjj dS)Nrarrc bdt|D}|D]=}||D]0}|t|tsJd|d|j}|j}|j}|t kr'||t |p|tkrM|||t vsJd|d|d||t||tkr] j r Jd|||tvsJd|d|d ||t|2?|D]P}t ttfD]9}t|||} | |ksJd | d |d |d |:QdS)Nc i|]>}|tttttti?SrG)r.rr/r0)r`stage_ids r<raz\PipelineScheduleMulti._validate_schedule.._validate_rank_actions..WsPEEE  suusuusuuEEEr>zGot an invalid action: z, expected instance of _ActionzRunning Backward for stage z , microbatch z without first running ForwardzESchedule contains 'W' actions, but is configured to use full backwardzRunning Weight for stage z without first running BackwardzGot rsz microbatches for stage z , expected ) rrrIrJrKrLr.rr/r0rvrX) rarr stage_actionsrrAs_idrfmb_idstage_mbr:s r<_validate_rank_actionszHPipelineScheduleMulti._validate_schedule.._validate_rank_actionsPsdEE !&j 1 1 EEEM  : :%dm::F~ %XXWWWWXXX"-D"3E"3Ezz%d+A.2259999!!]4%8%;;;;qqqEqqq<;;%d+A.2259999! $ 6ccbccc"]4%8%;;;;ptpp%ppp<;;%d+A.225999/:2& l lAYllE"=#6u#=>>H $4444khkkkkkkYikk5444l l lr>z2Schedule has incorrect number of ranks - expected z , actual z%Schedule is missing actions for rank ) r rWr rIrXr[r{rrr)r:rrs` r<_validate_schedulez(PipelineScheduleMulti._validate_scheduleNs. l#tGdN334. l. l". l. l. l. l. l. lb # $ $(: : : : w@R w w]`aeat]u]u w w ; : :$,-- > >D++++=t==,+++           r> compute_onlyc |dksJt|d5}tj|}t|D]\}}d|D|j|< dddn #1swxYwY|dS)zLoad a CSV representation of the schedule from a file with the provided filename. This API will most likely get renamed/refactored so is marked as internal for now. format must be "compute_only" for PipelineScheduleMulti rrSrcBg|]}t|SrGrIrBr`rVs r<rgz3PipelineScheduleMulti._load_csv..s&,N,N,NQW-=-=a-@-@,N,N,Nr>N)rrreaderr}r[r)r:rformatrrrr~s r< _load_csvzPipelineScheduleMulti._load_csvs '''' (B ' ' ' O7Z((F&v.. O O c,N,N#,N,N,N#D)) O O O O O O O O O O O O O O O O !!!!!s>A%%A),A)rrcV|jD]}||||\}}|(tt j||j}nd}||||||jD]%}|jr| |j cS&dSr) ryrrrrrrrrrr) r:rrrrrrrrs r<rzPipelineScheduleMulti.steps\ ) )E  & & ( ( ( ($(#5#5dF#C#C L   !3FD????? @tr>rrrc H |||||\}}d|jD}t}t}|D]\}|dkr#||j|dz ||jdz kr#||j|dz]t|j|j D]\} } g} | o| j } | j } | j }| Jd| tjkrl||}|| || || }||||| | || n| tjkrd||}||| }|| ||j| || n^| tjkr<|jrt3d|j||}|| nt3d| |D]}|j|}d}| t7|kr|| }||j } |j } |j }| Jd| tjkr;|dz|vr3||dz}| || | tjks| tjkrt3d| |D]}|j|}d}| t7|kr|| }||j } |j } |j }| Jd| tjks| tjkrp| tjkr;|dz |vr3||dz }| || t3d| | r!t=| @#t@$ra}tB"d |j |j#j$| | tB"d tK|j|d}~wwxYw|&|j|dS) Operate on the microbatches for looped schedules (multiple stages on each rank). TODO: Does not use sorted_batch_isend_irecv(). As a result, this schedule does not support models with skip connections. ci|] }|j| SrGrJrs r<raz.*? ? ? ).E u? ? ? r>rrNrr full_backwardzqWe detected a weight update in the pipeline schedule, but self.use_full_backward=zUnknown computation type zy[Rank %s] pipeline schedule %s caught the following exception at time_step %s when running action %sz%s)'rryrkeysrrurr}r[rrKrLrJr$r7r#rrr$r8rr/rvr0r9rbackward_weight_one_chunkrXr!r.rr" ExceptionrerrorrrCrr)r:rrrrstage_index_to_stageall_prev_ranksall_next_ranksrJ time_steprArrKrrrr prev_rank prev_rank_opsprev_rank_action next_rank next_rank_opsnext_rank_actiones r<rz(PipelineScheduleMulti._step_microbatchess"//JPVWW? ? 26,? ? ?  $'55#&55/4466 T TKQ""4#A+PQ/#RSSST-111""4#A+PQ/#RSSS!*4+>ty+I!J!Jr r  Ivq (*%'-'>$%6H"("4K ,,\-,,'+;+CCC 4[ A!&!8!8$gh&789L""00 HUUU 5#9#9(#C#CDDDD)-=-FFF 4[ A#33E8DD00$4t?U1 5#9#9(#C#CDDDD)-=-DDD1",!;!%!7!;!;##!5[ A77AAAA()WEU)W)WXXX"0I$($7 $BM'+$ 3}#5#555+8+C('3+;+L(#3#D&6&B $00`100,/?/GGG*Q2FFF)=[1_(M # 5+A+A(+K+K L L L,0@0III/3C3JJJ!", NB"&3>3>'(3>3>(# 3> "%(<"=> 3> $Do)=$>? 3>$E$sCx.%**D$EF3>$,DcN#;3> 3>3>3>3>3>3>j;;;= = = ~ " " " ""&!!!x~!!!!J#'$(%)!% T2T2$T2D>T2TN T2  T2T2T2T2T2T2T2T2r>rc eZdZdZ ddeeeeefde fdZ dde de ffd Z de fdZ dd eed eed eed eefdZ xZS)_PipelineScheduleRuntimea% Provides a simple runtime that requires a 'schedule IR' including specified communication operations. Can be instantiated directly by creating _PipelineScheduleRuntime and calling load_csv, or can be subclassed and the subclass can be responsible for creating a schedule IR. rrarcj Jdi_|dkr@|D];}gj|<||D]&}|Jj||'.ys(Fq(Ir>)r^rformat= is not implemented)rupipeline_order_with_commsrr]rsrr)r:rarrrAs` r< _load_actionsz&_PipelineScheduleRuntime._load_actions\s2  * 6 6 N 7 6 6CE& _ $ $ H H79.t4%dmHHF!---248??GGGGH H H ~ % %  7KDM88.t44.<.IIII+...D * * * &&E&E&E&EFF Fr>rc|dkr=t|||jdS|dkr}i}t |d5}t j|}t|D]\}}d|D||<|||ddddS#1swxYwYdStd|d ) aLoads a csv in simple format and then lowers it to include comunication actions format must be either "compute_only" or "compute_comms". If compute_only, the lowering passes will automatically be run to generate a compute_comms schedule. rrrSrcBg|]}t|SrGrrs r<rgz6_PipelineScheduleRuntime._load_csv..s&$F$F$FQW%5%5a%8%8$F$F$Fr>)rNrr) r rrr[rrrr}r) r:rrrarrrr~rs r<rz"_PipelineScheduleRuntime._load_csvsC ^ # # GG  h ' ' '   t2 3 3 3 3 3  & &Gh+++ ;wG,,!*6!2!2GGID#$F$F#$F$F$FGDMM""76":::  ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; &&E&E&E&EFF FsAB;;B?B?c|j Jdt|dd5}tj|}|jD]"}||j|# ddddS#1swxYwYdS)zaDump a CSV representation of the compute + comms schedule into a file with the provided filename.Nz6Must initialize compute_comms schedule before dump_csvrrSr)rrrrrrs r<rz"_PipelineScheduleRuntime._dump_csvs  * 6 6 C 7 6 6 (C , , , FZ((F6 F F >t DEEEE F F F F F F F F F F F F F F F F F F Fs?A11A58A5Nrrrrc |||||\}}d|jD}|j Jdi}i}g}itdtffd } t |j|jD]\} } | j} | j| jnd} | dks| ttfvsJd| d | j }||}t|j t}td | | | t"kr7|t'|| n| t*kr7|t'|| n| t.kr9|| f|vs Jd t'|| ||| f<nZ| t2kr9|| f|vs Jd t'|| ||| f<n| tkr8|r4|vr|vsJd |d|j d|<n| tkrA|r=|vsJd|d|vsJd|d|j n| t:kr|r | ||js=|| f|vsJd| d||| f |!| || || }|"|||| n| tFkr|r | ||j$s=|| f|vsJd| d||| f |%|| }|&| ||j'n_| tPkrA|r | ||j'rtSd|j'|*| ntSd| d#tV$rD}t,d| | t[t]|j|d}~wwxYwt_|r5| t_|5t_dks Jd|0|j|dS)rci|] }|j| SrGrrs r<raz?_PipelineScheduleRuntime._step_microbatches..rr>Nz=Must call _load_actions() before calling _step_microbatches()rec|vr2||=||vs Jd|dS)zQIf an unshard is active for `stage_idx`, wait() it and mark `stage_idx` unshared.z*Attempted to compute on sharded stage_idx=N)r"r)re unshard_opsunsharded_stagess r<_assert_unshardedzF_PipelineScheduleRuntime._step_microbatches.._assert_unshardedslK''I&++--- * $$Y///---->)>>.----r>rzaction=z missing mb_indexz8_PipelineScheduleRuntime running time_step %d, action %szARecv twice for {stage_idx=} {mb_index=} without executing forwardzBRecv twice for {stage_idx=} {mb_index=} without executing backwardzUnsharding the same stage_idx=z twiceT)async_opzResharding stage_idx=z without unshardingz before finishing unshardzComputing action=z before receiving inputz Attempted to run compute action=rzmWe detected a weight update in the pipeline schedule, but self.use_full_backward=z is unknown or unsupportedz\_PipelineScheduleRuntime caught exception at step %s when running action %s. Full Schedule:zUnused unshard operations)1rryrrrWr}rrKrLr1r2rJrsubmodrrrr3rrr$r5r0r4r!r6r.unshardreshardr7is_firstrr"r#rr8rrr/rvr9rrrrprintrrXr)r:rrrrr bwd_recv_ops fwd_recv_opssend_opsrrrA comp_typerrerstage_uses_fsdprrrrrs @@r<rz+_PipelineScheduleRuntime._step_microbatchess"//JPVWW? ? 26,? ? ?   * 6 6 J 7 6 657 46  "13 55 ? ? ? ? ? ? ? ?"+4+I$)+T!U!Ur r  Ivq "3 .:++  1}} 6)))1f000)))#. ,Y7",U\:"F"F N&&OOJu/E/Eh/O/O$P$PQQQQ&((OOJu/E/Eh/O/O$P$PQQQQ&((! ****,o***;E..x88;;L)X!677&((! ****,p***;E..x88;;L)X!677'))&U%-=== ) < < <D)DDD!= < <161E1Et1E1T1T I.'))&/%)9999HHHH:99&[888NNNN988 ,,...'))&5)))444 >G%$ * * * *,Y+X+X+X * * *%(()X)>??DDFFF"44 '("3Yx5HF,,UFJQQQQ(**&5)))444 =G%$ * * * *YXXX  * * * %(()X)>??DDFFF//x@@D,, t4;Q-&((&5)))444-(7!37733H====$%K%K%K%KLLL    r,T-KLLMMM (mm " LLNN   ! ! !(mm ";1$$$&A$$$ DL&11111s N"P.. Q<8?Q77Q<rr)rCrDrErr rWr r rIrOrrrrrrs@r<rrTs9%!G!Gc4 1223!G!G!G!G!GFGG#GsGGGGGG* F# F F F F#'$(%)!% i2i2$i2D>i2TN i2  i2i2i2i2i2i2i2i2r>rc eZdZdZ d deededeedee e e e fe e fffd ZdZxZS) r!ai Breadth-First Pipeline Parallelism. See https://arxiv.org/abs/2211.05953 for details. Simliar to Interleaved 1F1B, Looped BFS supports multiple stages per rank. What is different is that when microbatches are ready for multiple local stages, Loops BFS will prioritizes the earlier stage, running all available microbatches at once. Nrrrrct||||i|_t|jD]!}||}||j|<"dS)N)rrrr)r rr[rr{!_calculate_single_rank_operations)r:rrrrrrank_opsrs r<rzScheduleLoopedBFS.__init__Vs )/    CE$,-- 1 1D==dCCH(0D  % % 1 1r>c jt|j}t||j|z|j}g}t|D]}|d|D]G}t|jD]0}|t |tj|1Hd|jdz |z z}| dg|zt|D]T}tt|jD]0}|t |tj |1U|S)Nr%r) rXryrr{rrrIr$r7rreversedr8) r:rn_local_stages stage_indicesrr=rJrpost_warmup_opss r<rz3ScheduleLoopedBFS._calculate_single_rank_operationsmsdT\** $$~5t7I  -/t " "A OOD ! ! ! !(  K!$"677  K)9)A8LL t1A5<=0111#M22  K$U4+?%@%@AA  K)9)BHMM r>)NN)rCrDrErr rrWr rrr rOrrrrrrs@r<r!r!Ls'+IM 11'(11(# 1 $E$sCx.%**D$EF 111111.r>r!c tt} tt} tt} g} t|D]}| d||zd|dz |z zz||zz }| r||z dz }||z|z}g}d}t|D]f}||krl||}| |x}dz| |<| t |t j|||dz kr| dg|zu||cxkr ||zkr nn||}| |x}dz| |<| t |t j|||}| |x}dz| |<| t |t j|||| r]||z |krT|||}| |x}dz| |<| t |t j ||dz }| s| d||}| |x}dz| |<| t |t j|||| r]||z |krT|||}| |x}dz| |<| t |t j ||dz }h| r||t|kri|||}| |x}dz| |<| t |t j ||dz }| r|t|ki| S)Nr%rr) rrWrrrIr$r7rr8r9rX)rr{ warmup_ops fwd_bwd_ops cooldown_opsrforward_stage_indexbackward_stage_indexnum_1f1b_microbatchesrfwd_stage_mb_indexbwd_stage_mb_indexr:rr=r total_opsbackward_op_idsweight_op_countrfwd_stage_indexrr8bwd_stage_indexr9weight_stage_indexweight_mb_indexs r<_get_1f1b_rank_opsrs*5S)9)9)4S)9)9,7,<,<)+H 4[[ &ma.?$.F)GG d O3'$.2[(<7IOOII%I% ??11"55O/??3  / OO)9)A8LL   Z!^## 8999 2 8 8 8 8 [ 8 8 8 8 8 811"55O 2? CC 3  / OO)9)A<PP   32266O 2? CC 3  / OO)9)BLQQ     " "2 & & &! %b:o9N&N&N%9%9#O4&&"(==O'PPO=%&89*,<,C_  1$ & &%%%22266O 2? CC 3  / OO)9)BLQQ     " "2 & & &! %b:o9N&N&N%9%9#O4&&"(==O'PPO=%&89*,<,C_  1$ 33G3G!G!G11//2RSS45GH HO 501  &(8(? Q Q    1 33G3G!G!G Or>ceZdZdZ d deededeedee e dfdee e e fd ee e e efe eff fd Zd eeefd ZxZS)r ak The Interleaved 1F1B schedule. See https://arxiv.org/pdf/2104.04473 for details. Will perform one forward and one backward on the microbatches in steady state and supports multiple stages per rank. When microbatches are ready for multiple local stages, Interleaved 1F1B prioritizes the earlier microbatch (also called "depth first"). Nrrrr.rrc|dj|_||jzdkrtd|d|jdt||||||t ||_|dj|_|dj |_ i|_ t|jD]!}| |}||j |<"dS)Nrz?Interleaved 1F1B schedule requires the number of microbatches (zD) to be a multiple of the number of pipeline ranks (z).)rrrrrr) rzr{rr rrXrr|rgroupr[rr) r:rrrrrrrrrs r<rz ScheduleInterleaved1F1B.__init__s$AY1 D. .! 3 3ZR`ZZCGCUZZZ  )+//    "&kk1I( AY_ CE$,-- 1 1D==dCCH(0D  % % 1 1r>r\c  fd}| jjz}| z }||z } |z|z}td |||fd} fd}t jj ||||S)Ncjdz jz}|djdz |z zz}t|jjzSNrr%)rr{r6r)rwarmups_ops_last_stagerr:s r<get_rank_warmup_opszVScheduleInterleaved1F1B._calculate_single_rank_operations..get_rank_warmup_ops?sQ&*&9A&=AS%S "/!8JQ8NRV7V2WWJz4#7$:M#MNN Nr>=rank %s, warmup_ops %s, 1f1b %s, cooldown_ops %s total_ops %scB|jzjz}|jzzSrN)r{rr local_indexrr:s r<rzVScheduleInterleaved1F1B._calculate_single_rank_operations..forward_stage_index[s+4#559LLK$"44< c^jdz |z jzjzz }|jzzSNr)rr{rrrr:rs r<rzWScheduleInterleaved1F1B._calculate_single_rank_operations..backward_stage_index`sI#:%$*<<@SST   $"44< )rrrrrr{) r:rrmicrobatch_opsrrrrrrs `` @r<rz9ScheduleInterleaved1F1B._calculate_single_rank_operations>s O O O O O)(.. ,t/CC$z1 % 3 ,|;   K          = = = = = =  = = = = = = ="             r>r)rCrDrErr rrWr rrrr rOrrrrIrrrs@r<r r s'+AEBFIM%1%1'(%1%1(# %1 "%(<"=> %1 $Do)=$>? %1$E$sCx.%**D$EF%1%1%1%1%1%1N3 hw>O9P3 3 3 3 3 3 3 3 r>r ceZdZdZ ddeededeedee e dfd ee e e fd ee e e efe efd effd Zd eeefdZdZxZS)ra The Flexible Interleaved 1F1B schedule. This schedule is mostly similar to the interleaved 1F1B schedule. It differs by being relaxing the requirement of num_microbatch % pp_size == 0. Using the flex_pp schedule, we will have num_rounds = max(1, n_microbatches // pp_group_size) and it works as long as n_microbatches % num_rounds is 0. As a few examples, support 1. pp_group_size = 4, n_microbatches = 10. We will have num_rounds = 2 and n_microbatches % 2 is 0. 2. pp_group_size = 4, n_microbatches = 3. We will have num_rounds = 1 and n_microbatches % 1 is 0. When enable_zero_bubble is True, we will use the ZB1P schedule in https://openreview.net/pdf?id=tuzTN0eIO5 NFrrrr.rrrc J|dj|_t||||||| t ||_|dj|_td||jz|_ ||j z|_ ||_ ||j zdkrtd|j d|di|_ t|jD]!}||} | |j |<"||j|jz|_ dS)Nr)rrrrrrrvrzhFlexible Interleaved 1F1B requires the number of microbatches to be a multiple of the number of rounds (z ), but got .)rzr{r rrXrr|rrqnumber_of_roundsmicrobatches_per_roundrrr[rr_add_bubbles_to_actions) r:rrrrrrrrrrs r<rz(ScheduleFlexibleInterleaved1F1B.__init__s`$AY1 )+//"44    "&kk1I( #A~9K'K L L&48M&M#"4 D1 1Q 6 6-595J--)--- CE$,-- 1 1D==dCCH(0D  % % #::  $"4 4  r>r\c Z fd}| jjz}| z }||z } |z|z}td |||fd} fd}jr%} t jj ||||| d St jj ||||S)Ncjdz jz}jrdnd}||jdz |z zz}t |jjzSr)rrrr{r6r)rrmultiply_factorrr:s r<rz^ScheduleFlexibleInterleaved1F1B._calculate_single_rank_operations..get_rank_warmup_opsso#a'+&, "$(#:AaaO//#a'4/3J z4#7$:M#MNN Nr>rcB|jzjz}|jzzSrN)rrr{rs r<rz^ScheduleFlexibleInterleaved1F1B._calculate_single_rank_operations..forward_stage_indexs+4#>>$BUUK$"44< c^jdz |z jzjzz }|jzzSr)rrr{rs r<rz_ScheduleFlexibleInterleaved1F1B._calculate_single_rank_operations..backward_stage_indexsK#:%$*EE%&&   $"44< T)r)rrrrrrr{) r:rrrrrrrrrrs `` @r<rzAScheduleFlexibleInterleaved1F1B._calculate_single_rank_operationssQ O O O O O)(.. ,t/CC$z1 % 3 ,|;  K          = = = = = =  = = = = = = =  " $( !%#"#$%#'    "             r>ct|j}|js|Sd}t}i}i}i}d}t|jD]} g|| <d|| <d|| < d} t} t|jD]} || } | t || kr%d} || | || | } | J| \}}}||||||sR|| || | || |||f|| xxdz cc<|| d|| xxdz cc<|| xxdz cc<|| d|| | rnE|dkrt d|||S)Nc|tjkr|dkr |dz ||f|vrdSn4|tjkr$||dz kr|tj|f|vS|dz||f|vSdS)NrrTF)r$r7r8)rr microbatchnum_stages_globalseen_opss r< need_bubblezLScheduleFlexibleInterleaved1F1B._add_bubbles_to_actions..need_bubbles%---A::519b*"=X"M"M4'000-111!#3#;ZHPXXX 2z2(BB5r>rTFrz?Non zero bubbles added: total_bubbles_added=%s bubbles_added=%s) r[rrrr{rXrrupdaterwarning)r:r rar r result next_pointer bubbles_addedtotal_bubbles_addedr should_stop temp_seen_ops timestamp temp_actionrJrr s r<rz7ScheduleFlexibleInterleaved1F1B._add_bubbles_to_actionssV%& N   )NNNNF)rCrDrErr rrWr rrrr rOrrrprrIrrrrs@r<rrts'  $'+AEBFIM#(, , '(, , (# , "%(<"=> , $Do)=$>? , $E$sCx.%**D$EF, !, , , , , , \J hw>O9PJ J J J XBBBBBBBr>rceZdZdZ d deededeedee e dfdee e e fd ee e e efe eff fd ZxZS) r"a/ The Interleaved Zero Bubble schedule. See https://arxiv.org/pdf/2401.10241 for details. Will perform one forward and one backward on inputs for the microbatches in steady state and supports multiple stages per rank. Uses the backward for weights to fill in the pipeline bubble. Nrrrr.rrc Xt||||||ddS)NT)rrrrrrr)r r)r:rrrrrrrs r<rz&ScheduleInterleavedZeroBubble.__init__KsE )+//#      r>r)rCrDrErr rrWr rrrr rOrrrrrs@r<r"r"Bs'+AEBFIM  '(  (#  "%(<"=>  $Do)=$>?  $E$sCx.%**D$EF          r>r" schedule_namec tttttt t td}||vrtd|||S)z Maps a schedule name to its corresponding class object. Args: schedule_name (str): The name of the schedule. )1F1BInterleaved1F1BGPipeFlexibleInterleaved1F1B LoopedBFSInterleavedZeroBubblerrzUnknown schedule name: ) rr rrr!r"rrr)r schedule_maps r<rr_sW2#B&!>"8!6  LL((B=BBCCC  &&r>)FrN)r&)rF)Qrrloggingreabcrr collectionsrenumrtypingrrr r r r r rrrrtorch.distributed distributedr.torch.distributed._composable.fsdp.fully_shardrrtorch.profilerrr rrrrrr__all__ getLoggerrCrr$r7r8r9r1r2r3r4r5r6r.r/r0compilerTrIrWrOrrprrP2POprrrrrrr]rsrrr!rr rr"rrGr>r<r0s  ##############                         TTTTTTTT******TTTTTTTTTT%%%%%%'&&&&&&     8 $ $/E/E/E/E/Et/E/E/Ed  "  $    "  "            K      j   D$4T(7:K5L0L+M$RU$$$$V % MXMXd8G#4556MXMXMX MXMXMXMX`n n n n n n n n b11TZ(1 11116: $* %-c] #ty.6CCCCC.CCCL.....1...bMMMMM*MMM`E1E1E1E1E1)E1E1E1TCC(7+,CC ']CCCCLJ#tG},-JSE3J'JJ #tG}  JJJJZE2E2E2E2E2-E2E2E2P u2u2u2u2u24u2u2u2p=====-===R~~~~Bd d d d d 3d d d NKKKKK&;KKK\     $C   :'c''''''r>