
    ΧgC                        d dl Z d dlmZmZmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ ddgZde	e
j        ee         eed	f         eeef         f         d
ee
j                 fdZ	 	 ddee         dee         deeeeef                           deeee	ee
j        f                                    d
ee         f
dZdS )    N)AnycastDictListOptionalSequenceTupleUnion)ExceptionWrapper_get_device_index)Module	get_a_varparallel_applyobj.returnc                    t          | t          j                  r| S t          | t          t          f          r6t          t          |           D ] }t          |t          j                  r|c S !t          | t                    rHt          t          |                                           D ] }t          |t          j                  r|c S !d S )N)	
isinstancetorchTensorlisttuplemapr   dictitems)r   results     \/var/www/html/ai-engine/env/lib/python3.11/site-packages/torch/nn/parallel/parallel_apply.pyr   r      s     #u|$$ 
#e}%% )S)) 	 	F&%,// #t )SYY[[11 	 	F&%,// 4    modulesinputs
kwargs_tupdevicesc                   
 t          |           t          |          k    s*J dt          |            dt          |                       |#t          |           t          |          k    sJ n8t          t          t          t          f         i           ft          |           z  }|#t          |           t          |          k    sJ ndgt          |           z  }d |D             }d |D             }t          j                    i t          j                    t          j	                    c	 	 ddt          dt          dt          d	t          t          t          f         d
t          t          t          t          j        f                  dt          t          j        j                 ddffd
t          |           dk    r_
fdt#          t%          | ||||                    D             }|D ]}|                                 |D ]}|                                 n. 
d| d         |d         |d         |d         |d                    g }t+          t          |                    D ]H}|         }	t-          |	t.                    r|	                                 |                    |	           I|S )a  Apply each `module` in :attr:`modules` in parallel on each of :attr:`devices`.

    Args:
        modules (Module): modules to be parallelized
        inputs (tensor): inputs to the modules
        devices (list of int or torch.device): CUDA devices

    :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
    :attr:`devices` (if given) should all have same length. Moreover, each
    element of :attr:`inputs` can either be a single object as the only argument
    to a module, or a collection of positional arguments.
    zThe number of modules z& is not equal to the number of inputs Nc                 .    g | ]}t          |d           S )Tr   .0xs     r   
<listcomp>z"parallel_apply.<locals>.<listcomp>;   s#    ;;;a D));;;r   c                 L    g | ]!}t           j                            |          "S  )r   cudacurrent_streamr%   s     r   r(   z"parallel_apply.<locals>.<listcomp><   s(    ===uz((++===r   imoduleinputkwargsdevicestreamr   c                    t          j        	           |Xt          |          }|3
5  t          d|  d          | <   d d d            n# 1 swxY w Y   d S |                                }|t           j                            |          }	 t           j                            |          5  t           j                            |          5  t           j	        
                    d          5  t          |t          t          f          s|f} ||i |}d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   
5  || <   d d d            d S # 1 swxY w Y   d S # t          $ r9 
5  t          d|  d|           | <   d d d            Y d S # 1 swxY w Y   Y d S w xY w)Nzin replica zQ, no device was provided and no tensor input was found; device cannot be resolved)wherer+   )enabledz on device )r   set_grad_enabledr   r   
get_devicer+   r,   r1   r2   ampautocastr   r   r   	Exception)r-   r.   r/   r0   r1   r2   toutputautocast_enabledgrad_enabledlockresultss           r   _workerzparallel_apply.<locals>._workerD   s    	|,,,>%  Ay  !14A 4 4 4" " "GAJ              
 \\^^F>Z..v66F	""6** 2 2EJ,=,=- - 2 2y!!&2B!CC2 2 "%$77 %"HE1&112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2  $ $#
$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 	 	 	  ->>>f>>  
                   	s   AAAF 1 E"D>3(D'D>'D++D>.D+/D>2E>E	EE	E	F EF EF "E5(F 5E99F <E9=F GF6(G6F:	:G=F:	>GG   c                 Z    g | ]'\  }\  }}}}}t          j        ||||||f           (S ))targetargs)	threadingThread)r&   r-   r.   r/   r0   r1   r2   rA   s          r   r(   z"parallel_apply.<locals>.<listcomp>j   s^     
 
 
 ;:FE666 a%O  
 
 
r   r   NN)lenr   r   strr   rF   Lockr   is_grad_enabledis_autocast_enabledintr   r   r
   r1   r+   Stream	enumeratezipstartjoinranger   r   reraiseappend)r   r    r!   r"   streamsthreadsthreadoutputsr-   r<   rA   r=   r>   r?   r@   s             @@@@@r   r   r      s   $ w<<3    aGaaTWX^T_T_aa   7||s:.....4S>2..03w<<?
7||s7||+++++&3w<<';;7;;;G==W===G>DG!## #L" 6:.2# ### # S#X	#
 sEL012# *+# 
# # # # # # # # #J 7||a
 
 
 
 ?HGVZ'BB? ?	
 
 
  	 	FLLNNNN 	 	FKKMMMM	 	71:vay*Q-WQZPPPG3v;;  f.// 	NNvNr   rH   )rF   typingr   r   r   r   r   r   r	   r
   r   torch._utilsr   torch.cuda._utilsr   torch.nn.modulesr   __all__r   r   rJ   rN   r1   r   r*   r   r   <module>r`      s       J J J J J J J J J J J J J J J J J J J J  ) ) ) ) ) ) / / / / / / # # # # # # (
)	u|T#Yc3hc3hG	Hel   ( 6:FJ	b bfbSMb $sCx.12b hxc5<.?(@ABC	b
 
#Yb b b b b br   