
    Ng                         d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	  ej
        e          Zd Zdd	Zd
 Zd ZddZd Zd Zd Z	 	 	 ddedee         dee         fdZdS )zZ Distributed training/validation utils

Hacked together by / Copyright 2020 Ross Wightman
    N)Optional)distributed   )unwrap_modelc                     |                                  }t          j        |t          j        j                   ||z  }|S )Nop)clonedist
all_reduceReduceOpSUM)tensornrts      R/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/utils/distributed.pyreduce_tensorr      s7    	BOB4=,----!GBI    Fc                 2   t          |                               d          D ]r\  }}d|v sd|v re|rCt          j                            |t
          j        j                   |t          |          z  }Rt          j        	                    |d           sd S )NT)recurserunning_meanrunning_varr   r   )
r   named_bufferstorchr   r   r   r   r   float	broadcast)model
world_sizereducebn_namebn_bufs        r   distribute_bnr"      s    '..<<T<JJ 7 7g%%=G+C+C 7!,,V8I,JJJ%
+++ !++FA6667 7r   c                     | j         dk    S Nr   )rankargss    r   is_global_primaryr(   %   s    9>r   c                     | j         dk    S r$   )
local_rankr&   s    r   is_local_primaryr+   )   s    ?ar   c                 B    |rt          |           nt          |           S )N)r+   r(   )r'   locals     r   
is_primaryr.   -   s$    %*GD!!!0A$0G0GGr   c                      dt           j        v r#t          t           j        d                   dk    S dt           j        v r#t          t           j        d                   dk    S dS )N
WORLD_SIZEr   SLURM_NTASKSFosenvironint r   r   is_distributed_envr7   1   sU    rz!!2:l+,,q00##2:n-..225r   c                  P   d} dD ]1}|t           j        v r!t          t           j        |                   }  n2d}dD ]1}|t           j        v r!t          t           j        |                   } n2d}dD ]1}|t           j        v r!t          t           j        |                   } n2| ||fS )Nr   )
LOCAL_RANKMPI_LOCALRANKIDSLURM_LOCALIDOMPI_COMM_WORLD_LOCAL_RANK)RANKPMI_RANKSLURM_PROCIDOMPI_COMM_WORLD_RANKr   )r0   PMI_SIZEr1   OMPI_COMM_WORLD_SIZEr2   )r*   vglobal_rankr   s       r   world_info_from_envrE   9   s    J]  
??RZ]++JE  KI  
??bjm,,KE  JO  
??RZ]++JE  {J..r   c           	      p   d| _         d| _        d| _        d| _        t	          t          | dd          t          | dd           t          | dd                     }|d         | _        |d	         | _        |d
         | _        |d         | _        |d         | _         t          j        | j                  }|S )NFr   r   devicecudadist_backenddist_url)rG   rI   rJ   r   rD   r*   r   )r   r   r%   r*   init_distributed_device_sogetattrrG   r   )r'   resultrG   s      r   init_distributed_devicerN   O   s     DDODIDO'tXv..T>488z400  F
 "DK\*DO}%DI\*DOm,D\$+&&FMr   rH   rG   rI   rJ   c                    d}d}d}d}|                      dd          ^}}|dddd	}	|	                    |d
          }|pd}t                      r	dt          j        v rt                      \  }}}t          |          t          j        d<   t          |          t          j        d<   t          |          t          j        d<   t          j        	                    ||||           not                      \  }}
}
t          j        	                    ||           t          j        
                                }t          j                                        }d}|dk    r,t          j                                        sJ d|  d            |r6| dk    r0|r't                              d|d          d|  d           | d| } |                     d          rt          j                            |            t%          | ||||          S )NFr   r   :)maxsplitcclhcclnccl)xpuhpurH   gloozenv://r?   r9   r=   r0   )backendinit_methodr   r%   )rX   rY   TrH   zCUDA is not available but z was specified.cpuzdevice index z removed from specified (z).zcuda:)rG   rD   r*   r   r   )splitgetr7   r3   r4   rE   strr   r   init_process_groupget_world_sizeget_rankrH   is_available_loggerwarning
startswith
set_devicedict)rG   rI   rJ   r   r   rD   r*   device_type
device_idxdist_backends_s              r   rK   rK   d   sU    KJKJ%||C!|<<K* 
 

 %((f==#8H  RZ''2E2G2G/JZ'*:BJ|$!$[!1!1BJv'*:BJ|$00$$% 	 1      344J100$$ 1    *99;;J+4466Kfz&&((^^*^v*^*^*^^^^ /v  	`OO^JqM^^TZ^^^___..*..!! &
f%%%   r   )F)rH   NN)__doc__loggingr3   typingr   r   r   r   r   r   	getLogger__name__rb   r   r"   r(   r+   r.   r7   rE   rN   r]   rK   r6   r   r   <module>rp      sT     				        % % % % % %      
'
H
%
%  
7 
7 
7 
7       H H H H  / / /,  , &*"&O OOsmO 3-O O O O O Or   