קg?ddlZddlmZmZmZmZmZddlZddlm Z ddl m Z m Z gdZ ejeZ dZGddZGd d eZeejd d ZdZGd dZGddZdZ ddeedfdeeeefdedeeedfdeeeefdeeeeeff dZdeefdZdS)N)AnyDictListOptionalTuple map_aggregate) tree_flattentree_unflatten)TensorChunkSpecsplit_args_kwargs_into_chunks merge_chunksFceZdZdZdZdS)_CustomReducera$ Custom reducer class that can be used to specify a custom operation that reduces losses of multiple microbatches into one value. Example: >>> # xdoctest: +SKIP >>> sum_reducer = _CustomReducer( >>> torch.tensor(0.0), >>> lambda a, b: a + b >>> ) c"||_||_dSN) init_value reduce_fn)selfrrs c/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/distributed/pipelining/microbatch.py__init__z_CustomReducer.__init__(s$"N)__name__ __module__ __qualname____doc__rrrrrs-  #####rrceZdZdS) _LossReducerNrrrrrrrr-Drrgc ||zSrr)abs rr%1s 1q5rceZdZUdZdZeed<dZdZe de edffdZ e de e effd Zd S) r z2 Class used to specify chunking of inputs c||_dSr split_dim)rr)s rrzTensorChunkSpec.__init__=s "rr)cJ|jjd|jjd|jdS)N.()) __class__rrr)rs r__repr__zTensorChunkSpec.__repr__Bs/~( V V4>+B V VT^ V V V rcd|jdS)NzTensorChunkSpec(r-r(r/s r__str__zTensorChunkSpec.__str__Gs3$.3333r chunk_dims.c(t|d}|S)a A helper for creating a tuple of `TensorChunkSpec` from a tuple of chunk dimensions (int's). Example: >>> # xdoctest: +SKIP >>> # There are three positional arguments to the model, and >>> # we are chunking them along dimension 0, 0 and 1, respectively >>> args_chunk_spec = TensorChunkSpec.from_tuple((0, 0, 1)) c t|Srr dims rr%z,TensorChunkSpec.from_tuple..Y,,rr)r3args_chunk_specs r from_tuplezTensorChunkSpec.from_tupleJs$(  , ,  rc(t|d}|S)a\ A helper for creating a dictionary of `TensorChunkSpec` from a dictionary of chunk dimensions (int's). Example: >>> # xdoctest: +SKIP >>> # Chunk dimension 0 for the "id" argument, 1 for the "mask" argument >>> kwargs_chunk_spec = TensorChunkSpec.from_dict({"id": 0, "mask": 1}) c t|Srr6r7s rr%z+TensorChunkSpec.from_dict..kr9rr)r3kwargs_chunk_specs r from_dictzTensorChunkSpec.from_dict]s%*  , ,  ! rN)rrrrrint__annotations__r0r2 staticmethodrr;rstrr?rrrr r 8s###NNN   444#s(O\$!cN!!!\!!!rr ceZdZdS) _ReplicateNr rrrrErEqr!rrEc i}g}|}d}t|t|ksNJdt|dt||D]\}}t |\} } || ||} | Jt | \} } t| t| krt d|d| g}t| | D]\}}|tust|tj s||g|zCt|tr}t|tj s J|d| |j}||krB|r'td|d |d |d |}nt#d |d |d|dtj|||j}t&rg}d}|D]}tj|}|| |jz}t+dddg|jz}t+||||j<|||<|||| |jz }||n||d}t/d||||<g}t1|D]Z}i}|D],\}}g}|D]}||||||<-||[g}|D]y} i}!t|t| ksJt| |D]\\}}}"t3||"|!|<||!z|S)aW Given a dictionary of args, and a dictionary of chunking specs, shard the args according to the chunking specs. Args: args_dict: Dictionary of args args_chunk_spec: Dictionary of chunking specs num_chunks: Number of chunks to shard the args into Returns: args_split: List of sharded args Tzargs_dict.keys() = z args_chunk_spec.keys() = NzArgument value z9 did not have the same number of values as as chunk spec z is not a tensorz%Tensor size on chunking dimension is z', downsizing the number of chunks from z to r+zArg z% on chunking dimension has a size of z$, smaller than the number of chunks z. PiPPy cannot reduce the number of chunks because other arguments have bigger chunk-dimension sizes. Please adjust your num_chunks setting.rFzUnrecognized chunk spec: )lenlistkeysitemsr append ValueErrorziprE isinstancetorchTensorr sizer)loggerwarning RuntimeError tensor_split_debug_mask_minibatches zeros_likeslicendim TypeErrorranger )# args_dictr: num_chunksargs_sharded_replicated arg_specsreal_num_chunks first_tensorarg_keyargflatspec chunk_specchunk_spec_flat_sharded_arg_flatvchunk_vv_split_dim_size chunk_tensorsexpanded_chunks split_dim_idx chunk_tensornew_val upper_idx slice_indices chunks_flat chunk_idx chunk_argskeyarg_single_chunkv_flat args_splitchunkper_chunk_argsarg_specs# r_shard_dict_of_argsr~us(!I OL y>>S   mT).."2"233mmtTcThThTjTjOkOkmm   "))I<I< !#&& d$W- %%%)*55 t99O,, , ,8#88+588  dO448 G8 GJAw*$$Jq%,,G,G$ ''o(=>>>>G_555 G"!U\22JJq4J4J4JJJJ#$66'*;#<#< #o55#hDThhDNhhTdhhh+;*E7EEQaEEAKEEE!& 2(9!! +;&(O$%M(5 N N "'"21"5"5$1L4E4EgFW4X4X$X ).tT4)@)@(AGL(P ;@)9<< g&782> .'..w777%):):7;L)M)MM $++O<<<<$++M:::$  EG E EFFF+;((K?++''  /5577 / /HC!  ; ; ''y(9::::.JsOO:&&&&J**9~~U++++$' y$A$A @ @ JS#"0h"?"?N3  .)))) rargs.kwargschunksr:r>returnc  |i}|%ttft|z}|-t|tt}t tt |tt ||}t|}t |||}t||krTt|}t tt |tt ||}t|t|kr/tdt|dt|g}|D]J |t fdtt DK||fS)a Given a sequence of args and kwargs, split them into a number of chunks according to their respective chunking specs. Args: args: Tuple of args kwargs: Dict of kwargs chunks: Number of chunks to split the args and kwargs into args_chunk_spec: chunking specs for args, in same shape as args kwargs_chunk_spec: chunking specs for kwargs, in same shape as kwargs Returns: args_split: List of sharded args kwargs_split: List of sharded kwargs Nz;args and kwargs are split into different number of chunks: z, c3(K|] }|V dSrr).0irvs r z0split_args_kwargs_into_chunks..Ys'NN! 1 NNNNNNr) r DEFAULT_CHUNK_DIMrGdictfromkeysr~ enumeraterTrKtupler[) rrrr:r>args_split_dictr` kwargs_splitrzrvs @rr r sp~*+<==?#d))K  MM&/BS2T2TUU) Yt__ Y ' '((O /**O&L  <?**l++- 4 ! ! ?++ , ,     ?s<0000 ;?## ; ;'*<'8'8 ; ;   J%PP %NNNNuS__7M7MNNNNNOOOO | ##rc^|t|\}}n=t|d\}}ttgt|z}g|D]^}t|\}}t|t|krt d|d||_g}t |D]T\} t| trmfdttD} tr | dj } | ddD]} | j | ksJtj tj | ddit| | j } g}d}t| t| ksJt| | D]s\}}||| jz}t#dddg|jz}t#|||| j<||}|||}tn| }|tj|| j t| t(r_| j}ttD]$}| ||}%||d}tdtD]}||ksJ||Vt/||S) z Given a list of chunks, merge them into a single value according to the chunk spec. Args: chunks: list of chunks chunk_spec: Chunking spec for the chunks Returns: value: Merged value NrzChunk z did not match chunk spec c,g|]}|Srr)rruarg_idxchunks_flatteneds r z merge_chunks..s3!+G4rdevicemeta)sectionsr8r7)r r rrGrLrKrrNr[rVshaperOrUemptyr)rMrQrXrYcatrrrr )rrfspec_flattened flatten_spec chunk0_flatr{chunk_flattenedrhargs_flattenedrcpartial_values overall_shapeval meta_chunks values_to_catchunk_start_idx partial_value meta_chunk chunk_end_idxrssliced reduced_valruvaluerrs @@rrr^sZ'3J'?'?$ %1$;$;! \)*;<<=K@P@PP11)%00   3~#6#6 6 6SeSSzSSTT T0000 N!.110)0)  c? + +/ )!&s+;'<'v>> 00  !# "#>**c+.>.>>>>>14^[1Q1Q44-M:$3joocm6T6T$TM%*4t%<%<$= @R$RM38-3X3XM#-0*=9F!((000&3OO4!/  ! !%)Ms}"M"M"M N N N N ^ , , ).K"3'7#8#899   !mm!1)!rs\33333333333333 ''''''<<<<<<<<     8 $ $  ########$     >   l<5<,,.@.@AA 5!5!5!5!5!5!5!5!r        ~~~J>B>B e$e$ S/e$ T#s(^ $e$ e$eOS$89: e$  S/%9 :; e$  4;T "# e$e$e$e$Pw8 Iw8w8w8w8w8w8r