
    NgZ                      6   d Z ddlZddlZddlmZmZ ddlmZ ddlZddl	m
Z ddlmZ 	 ddlZddlmZ ddlmZ n# e$ rZ ed           edZ[ww xY wd	d
lmZ d	dlmZ d	dlmZ  eej                            dd                    Z G d de          Z dS )z' Dataset reader for HF IterableDataset
    N)repeatchain)Optional)Image)split_dataset_by_node)	SplitInfozDPlease install Hugging Face datasets package `pip install datasets`.   )load_class_map)Reader)SharedCountHFIDS_SHUFFLE_SIZEi   c                         e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 d$dedee         dededededededee         dededededee         dee         f fdZd Z		 d%dee         fdZ
d Zd Zd  Zd! Zd&d"Zd&d#Z xZS )'ReaderHfidsNtrainFr	   r   *   imageRGBlabel namerootsplitis_training
batch_sizedownloadrepeatsseed	class_map	input_keyinput_img_mode
target_keytarget_img_modeshuffle_sizenum_samplesc                 ^   t                                                       || _        || _        || _        || _        || _        || _        || _        |pt          | _
        |
| _        || _        || _        || _        t          j        ||          | _        |r| j                                         d }| j        j        j        rT|| j        j        j        v rAt+          | j        j        j        |         t,                    r| j        j        j        |         }|r|| _        n%|r|j        r|j        | _        nt3          d          d| _        |	rt7          |	          | _        d| _        ni | _        d| _        d| _        t?          j                     rZt?          j!                    rGt?          j"                    dk    r0t?          j#                    | _        t?          j"                    | _        d | _$        d| _%        d| _&        d| _'        d| _(        d | _)        tU                      | _+        d S )N)	cache_dirzDataset length is unknown, please pass `num_samples` explicitely. The number of steps needs to be known in advance for the learning rate scheduler.FTr   r	   ),super__init__r   r   r   r   r   r   common_seedSHUFFLE_SIZEr#   r   r    r!   r"   datasetsload_dataset_builderbuilderdownload_and_prepareinfosplits
isinstancer   r$   num_examples
ValueErrorremap_classr
   class_to_idx	dist_rankdist_num_replicasdistis_availableis_initializedget_world_sizeget_rankworker_info	worker_idnum_workersglobal_worker_idglobal_num_workersdsr   epoch)selfr   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   
split_info	__class__s                    Z/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/data/readers/reader_hfids.pyr(   zReaderHfids.__init__   s+   $ 		
&$ (8L",$.4TTJJJ 	0L--///*.
<# 	R1B1I(I(I$,+2599EE R26,2C2J52Q
 	*D 	J3 	)6Dd  
 ! 	# .y 9 9D#D "D !" 	;4#6#8#8 	;T=P=R=RUV=V=V!]__DN%)%8%:%:D"   !"# 7; ]]


    c                     || j         _        d S N)rC   value)rD   counts     rG   	set_epochzReaderHfids.set_epochk   s     
rH   r?   c                 V    | j         d S ||| _        | j        | j        z  | _        d S d S rJ   )rB   r?   r7   rA   )rD   r?   s     rG   set_loader_cfgzReaderHfids.set_loader_cfgo   s@     7F"*D&*&<t?O&OD### #"rH   c                 T   | j         tt          j        j                                        }||| _         |j        | _        |j        | _        | j        | j        z  | _	        | j
        | j        z  | j        z   | _        | j        r<| j                            | j                  }|                    | j	                  }n | j                            | j                  }| j        r!|                    | j        | j                  }t-          || j
        | j                  | _        dS )z8 Lazily initialize worker (in worker processes)
        N)r   )
num_shards)r   buffer_size)rank
world_size)r=   torchutilsdataget_worker_infoidr>   r?   r7   rA   r6   r@   r   r-   
as_datasetr   to_iterable_datasetas_streaming_datasetr   shuffler)   r#   r   rB   )rD   r=   datasetrB   s       rG   
_lazy_initzReaderHfids._lazy_inity   s    #+*::<<K&#. !,#.#: &*&<t?O&OD#$(NT5E$E$VD!= 	El--DJ-??G,,8O,PPBB 222DDB 	R!1t?PQQB (DLbcccrH   c                 @   t          d| j                  | j        z  t          | j        | j                  z  }| j        s| j        dk    rt          j        |          }| j        r+| j        $t          j        || j        z            | j        z  }t          |          S )Nr	   )
maxr   r$   rA   r7   r   mathceilr   int)rD   num_worker_sampless     rG   _num_samples_per_workerz#ReaderHfids._num_samples_per_worker   s    4<  4#33c$:QSWSi6j6jj 	 	?t599!%+=!>!> 	c ;!%+=+O!P!PSWSb!b%&&&rH   c              #     K   | j         |                                  | j                             | j        j                   |                                 }d}| j        r't          j        t          | j                             }nt          | j                   }|D ]}|| j                 }| j        r*|j        | j        k    r|                    | j                  }|| j                 }| j        rOt#          |t$          j                  s
J d            |j        | j        k    r|                    | j                  }n| j        r| j        |         }||fV  |dz  }| j        r	||k    r d S d S )Nr   z7target_img_mode is specified but target is not an imager	   )rB   r_   rM   rC   rK   rf   r   r   from_iterabler   iterr   r    modeconvertr!   r"   r1   r   r4   r5   )rD   target_sample_countsample_countds_itersample
input_datatarget_datas          rG   __iter__zReaderHfids.__iter__   s     7?OO$**+++"::<< 	$)&//::GG47mmG 	 	F&,T^&<J" Ez$:M'M'M'//0CDD
 1K# =!+u{;;vv=vvvv#t';;;"-"5"5d6J"K"KK! ="/<k))))AL L4G$G$G	 	rH   c                 >    |                                  | j        z  }|S rJ   )rf   r?   )rD   r$   s     rG   __len__zReaderHfids.__len__   s!    2244t7GGrH   c                     J d            )NFzNot supported )rD   indexbasenameabsolutes       rG   	_filenamezReaderHfids._filename   s    %o%%%rH   c                     | j         |                                  g }| j         D ]U}d|v r	|d         }n1d|v r	|d         }n$d|v r	|d         }nd|v r	|d         }n
J d            |                    |           V|S )z0 Return all filenames in dataset, overrides baseN	file_namefilenamerY   image_idFzNo supported name field present)rB   r_   append)rD   rx   ry   namesro   r   s         rG   	filenameszReaderHfids.filenames   s    7?OOg 	 	Ff$$k*v%%j)d|v%%j)?????LLrH   )Nr   Fr	   Fr   r   Nr   r   r   r   NNrJ   )FF)__name__
__module____qualname__strr   boolrd   dictr(   rM   rO   r_   rf   rr   rt   rz   r   __classcell__)rF   s   @rG   r   r      s        #'  %"(,$"'%#%*.)-!K# K#K# 3-K# 	K#
 K# K# K# K# K#  ~K# K#  K# K# !K# #3-K#  "#!K# K# K# K# K# K#Z! ! ! *.P P!#P P P Pd d dB' ' '  8  & & & &       rH   r   )!__doc__rb   os	itertoolsr   r   typingr   rU   torch.distributeddistributedr8   PILr   r+   datasets.distributedr   datasets.splitsr   ImportErroreprintr   r
   readerr   shared_countr   rd   environgetr*   r   rv   rH   rG   <module>r      sz     				 # # # # # # # #                         OOO::::::)))))))   	E
PQQQ
G
 & % % % % %       % % % % % % s2:>>"6==>>z z z z z& z z z z zs   ; A AA