
    Ng3B                        d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZmZ ddlZddlmZ ddlZddlmZ ddlmZmZmZ 	 ddl Z!dd	l"m#Z#m$Z$ dd
l%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ n# e,$ r dZ!dZ&Y nw xY wddl-m.Z. ddl/m0Z0 ddl1m2Z2  ej3        e4          Z5 e6ej7        8                    dd                    Z9 e6ej7        8                    dd                    Z:d(dZ;e	 G d d                      Z<de=defdZ>d Z?	 	 	 	 d)d Z@d! ZAe! G d" d#e!jB                  ZCndZC G d$ d%e          ZD G d& d'e0          ZEdS )*zR Dataset reader for webdataset

Hacked together by / Copyright 2022 Ross Wightman
    N)	dataclass)partial)islice)AnyCallableDictListOptionalTuple)Image)DatasetIterableDatasetget_worker_info)_shufflegetfirst)expand_urls)base_plus_ext
url_openertar_file_expandervalid_sample   )load_class_map)Reader)SharedCountWDS_SHUFFLE_SIZEi    WDS_INITIAL_SIZEi   z
_info.jsonz	info.jsonc                    t          |t                    r|f}g }d}|D ]}t          j                            | |          }	 |                    |           t          j        |          5 }|                    d          rt          j
        |          }nt          j        |          }d d d            n# 1 swxY w Y   |c S # t          $ r}t          |          }Y d }~d }~ww xY wt                              d| d| d           i S )N z.jsonzDataset info file not found at z	. Error: z.. Falling back to provided split and size arg.)
isinstancestrospathjoinappendwdsgopenendswithjsonloadyaml	safe_load	Exception_loggerwarning)	rootnamestriederr_strn	full_pathf	info_dictes	            X/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/data/readers/reader_wds.py
_load_infor:   *   sw   % EG  GLLq))			LL###9%% 2::g&& 2 $	!II $q 1 1I	2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
  	 	 	!ffGGGGGG	 OO	7% 	7 	7' 	7 	7 	78 8 8 Is<   )C+?B6*C6B:	:C=B:	>C
C(C##C(c                   f    e Zd ZU eed<   ee         ed<   dZee         ed<   dZeed<   dZ	eed<   dS )		SplitInfonum_samples	filenames shard_lengthsr   	alt_labelnameN)
__name__
__module____qualname__int__annotations__r   r!   r@   rA   rB   r?       r9   r<   r<   B   s]         Sz "M5:"""IsD#NNNNNrH   r<   splitinfoc           	        	 d }d| v sd| v ra|                      d          } d}d}t          |           dk    rt          | d                   }| d         } d| vrF|                      d	d
          }t          |          dz
  }|rd|v r||         |d         v r||         }t          |           }|r|d         |         }|sd t	          |d         |d                   D             	t          	fd|D                       }t          	                                          |d<   t          	                                          |d<   ||d<    ||          }nxt          |||          }ned|vs
| |d         vr<t          d|  d|                    di                                            d          | } |d         |          } ||          }|S )Nc           	          t          | d         t          | d                   t          | d                   |                     dd          | d                   S )Nr=   r>   r@   rA   r   rB   )r=   r>   r@   rA   rB   )r<   tupleget)	dict_infos    r9   _info_convertz(_parse_split_info.<locals>._info_convertL   sZ    !-0Ik233	/ :;;mmK446"
 
 
 	
rH   tarz..|r   r   r   z::-   splitsc                     i | ]\  }}||	S r?   r?   ).0r6   cs      r9   
<dictcomp>z%_parse_split_info.<locals>.<dictcomp>h   s    bbb1q!bbbrH   r>   r@   c              3   (   K   | ]}|         V  d S Nr?   )rW   r6   _fcs     r9   	<genexpr>z$_parse_split_info.<locals>.<genexpr>i   s'      !B!BQ#a&!B!B!B!B!B!BrH   r=   )rB   r=   r>   zsplit z not found in info ())rI   lenrF   r   zipsumrM   keysvaluesr<   RuntimeErrorrN   )
rI   rJ   rP   r=   
split_namesplit_parts	split_idxsplit_filenames
split_infor\   s
            @r9   _parse_split_inforj   K   s,   
 
 
 ~~ C  
u::>>eAh--Kau++c1--KK((1,I 4X--+i2HDQYN2Z2Z(3
%e,, 	h
3J 8bbJ{,CZP_E`(a(abbb!!B!B!B!B/!B!B!BBB*/

*;*;
;'.3CJJLL.A.A
?+,7
=)&z22JJ"')  JJ 45X#>#>ccc488HVXCYCYC^C^C`C`cccddd(^E*
"]:..
rH   c                     t                               dt          |            d           t          | t                    r| dS )zRCall in an exception handler to ignore exceptions, isssue a warning, and continue.zHandling webdataset error (z). Ignoring.T)r.   r/   reprr    	TypeError)exns    r9   log_and_continuero   ~   sB    OOI$s))IIIJJJ#y!! 	4rH   jpgRGBclsr   c                    |r8t          j        | d                   }t          ||                   }|dk     rdS nt          | |                   }t          | |          }t	          j        |          5 }t          j        |          }|                                 ddd           n# 1 swxY w Y   |r|	                    |          }t          |||                     dd                    }	|	S )z Custom sample decode
    * decode and convert PIL Image
    * cls byte string label to int
    * pass through JSON byte string (if it exists) without parse
    r)   r   N)rp   rr   r)   )r)   loadsrF   r   ioBytesIOr   openr*   convertdictrN   )
sample	image_key
image_mode
target_keyrA   metaclass_labelimgbdecodeds
             r9   _decoder      s&     .z&.))$y/**??4  &,-- 69
%
%C	C Ajmm


                &kk*%% s&**VT2J2JKKKGNs   4)B))B-0B-c                  l    t                      } | | j        S t          j                                        S )z'get dataloader worker seed from pytorch)r   seedr&   utilspytorch_worker_seed)worker_infos    r9   r   r      s0    !##K9((***rH   c                   $    e Zd Z	 	 	 	 ddZd ZdS )	detshuffle2  d   r   c                 >    || _         || _        || _        || _        d S r[   )bufsizeinitialr   epoch)selfr   r   r   r   s        r9   __init__zdetshuffle2.__init__   s$     #DL"DLDIDJJJrH   c                 ,   t          | j        t                    r| j        j        }n| xj        dz  c_        | j        }| j        dk     rt                      |z   }n
| j        |z   }t          j        |          }t          || j	        | j
        |          S )Nr   r   )r    r   r   valuer   r   randomRandomr   r   r   )r   srcr   r   rngs        r9   runzdetshuffle2.run   s    $*k22 #
( 

a


y1}}*,,u4y5(-%%CCt|SAAArH   N)r   r   r   r   )rC   rD   rE   r   r   r?   rH   r9   r   r      sJ         
	 
	 
	 
		B 	B 	B 	B 	BrH   r   c                   <     e Zd ZdZej        dddf fd	Zd Z xZS )ResampledShards2z,An iterable dataset yielding a list of urls.NTr   c                 R   t                                                       t          j                            |          }|| _        t          | j        d         t                    sJ || _        t          j
                    | _        |t          n|| _        || _        || _        dS )zSample shards from the shard list with replacement.

        :param urls: a list of URLs as a Python list or brace notation string
        r   N)superr   r&   
shardlistsr   urlsr    r!   nshardsr   r   r   r   worker_seeddeterministicr   )r   r   r   r   r   r   	__class__s         r9   r   zResampledShards2.__init__   s     	~))$//	$)A,,,,,,=??2=2E..;*


rH   c              #     K   t          | j        t                    r| j        j        }n| xj        dz  c_        | j        }| j        r.t          j        |                                 |z             | _        t          | j
                  D ]O}| j                            dt          | j                  dz
            }t          | j        |                   V  PdS )z#Return an iterator over the shards.r   r   )urlN)r    r   r   r   r   r   r   r   r   ranger   randintr_   r   ry   )r   r   _indexs       r9   __iter__zResampledShards2.__iter__   s      dj+.. 	J$EE JJ!OJJJE 	A}T%5%5%7%7%%?@@DHt|$$ 	- 	-AH$$QDI(:;;E49U+,,,,,,,	- 	-rH   )	rC   rD   rE   __doc__sysmaxsizer   r   __classcell__r   s   @r9   r   r      sb        66
      ,- - - - - - -rH   r   c            "           e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d'dedee         dededee         dedededee         dedededededee         dee         f  fdZd Z		 d(dee         fdZ
d  Zd! Zd" Zd# Zd$ Zd)d%Zd)d&Z xZS )*	ReaderWdsNtrainFr   r   *   jpg;png;webprq   rr   r   filenamer0   rB   rI   is_trainingr=   
batch_sizerepeatsr   	class_map	input_keyinput_img_moder}   target_img_modefilename_keysample_shuffle_sizesmaple_initial_sizec                    t                                                       t          t          d          || _        || _        || _        || _        || _        d| _	        |pt          | _        |pt          | _        |
| _        || _        || _        || _        d| _        t'          | j                  | _        t+          || j                  | _        ||| _        n| j        j        | _        | j        st          d          d| _        |	rt3          |	          | _        d| _        ni | _        d| _        d| _        t;          j                    rZt;          j                    rGt;          j                     dk    r0t;          j!                    | _        t;          j                     | _        d | _"        d| _#        || _$        d| _%        d| _&        d| _'        d| _(        tS                      | _*        d | _+        d S )	NzcPlease install webdataset 0.2.x package `pip install git+https://github.com/webdataset/webdataset`.i  z.JPEGz4Invalid split definition, num_samples not specified.FTr   r   ),r   r   r&   rd   r0   r   r   r   common_seedshard_shuffle_sizeSAMPLE_SHUFFLE_SIZEr   SAMPLE_INITIAL_SIZEsample_initial_sizer   r   r}   r   key_extr:   rJ   rj   ri   r=   remap_classr   class_to_idx	dist_rankdist_num_replicasdistis_availableis_initializedget_world_sizeget_rankr   	worker_idr   num_workersglobal_worker_idglobal_num_workers
init_countr   epoch_countds)r   r0   rB   rI   r   r=   r   r   r   r   r   r   r}   r   r   r   r   r   s                    r9   r   zReaderWds.__init__  s   & 	;uw w w	&$"%#6#M:M #6#M:M ",$(ty))	+E49=="*D#:D 	XVWWW  	# .y 9 9D#D "D !" 	;4#6#8#8 	;T=P=R=RUV=V=V!]__DN%)%8%:%:D"   !"#&== rH   c                     || j         _        d S r[   )r   r   )r   counts     r9   	set_epochzReaderWds.set_epochQ  s    !&rH   r   c                 V    | j         d S ||| _        | j        | j        z  | _        d S d S r[   )r   r   r   r   )r   r   s     r9   set_loader_cfgzReaderWds.set_loader_cfgT  s@     7F"*D&*&<t?O&OD### #"rH   c                      j         t          j        j                                        }|+| _         |j         _        |j         _        |j	         _	         j
         j	        z   _         j         j	        z   j        z    _         fd j        j        D             }t!          j        |          g} j        r|                    t)           j         j         j                   j        t!          j        t4                    t!          j         j         j        t=          j         j                            g           n4|                     j        t!          j        t4                    g           |                    t!          j         tC          tD           j#         j$         j        j%                  t4                    t!          j&         j#         j'                  g           t!          j(        |  _)        dS )z8 Lazily initialize worker (in worker processes)
        Nc                 Z    g | ]'}t           j                            j        |          (S r?   )r"   r#   r$   r0   )rW   r6   r   s     r9   
<listcomp>z(ReaderWds._lazy_init.<locals>.<listcomp>l  s+    ]]]arw||DIq99]]]rH   )r   r   )handler)r   r   r   )r{   r|   rA   )imagetarget)*r   torchr   datar   idr   r   r   r   r   r   r   r   ri   r>   r&   SimpleShardListr   extendr   r   r   r   _split_by_node_and_workertarfile_to_samplesro   shuffler   r   r   r   mapr   r   r   r   rA   renamer}   DataPipeliner   )r   r   abs_shard_filenamespipelines   `   r9   
_lazy_initzReaderWds._lazy_init^  s    #+*::<<K&#. !,#.#3 #.#: &*&<t?O&OD#$(NT5E$E$VD! ^]]]4?C\]]]'(;<<= 	OO+)*  
 .&/?@@@ 4 4d&677        OO.&/?@@@   
 	G"n#2"o7	   )   JT^DODDD
 	 	 	 "H-rH   c              #   |   K   | j         dk    r%t          || j        d | j                   D ]}|V  d S |D ]}|V  d S Nr   )r   r   r   )r   r   ss      r9   r   z#ReaderWds._split_by_node_and_worker  sr      "Q&&C!6d>UVV       rH   c                    | j         t          | j        | j                  z  }| j        s| j        dk    rt          j        |          }| j        r$t          j        || j        z            | j        z  }t          |          S r   )	r=   maxr   r   r   mathceilr   rF   )r   num_worker_sampless     r9   _num_samples_per_workerz!ReaderWds._num_samples_per_worker  s    !-D4KTMc0d0dd 	?t599!%+=!>!> 	c!%+=+O!P!PSWSb!b%&&&rH   c              #   8  K   | j         |                                  |                                 }| j        s| j        dk    r| j                             |          }n| j         }d}|D ]/}|d         }| j        r| j        |         }|d         |fV  |dz  }0d S )Nr   r   r   r   )r   r   r   r   r   
with_epochr   r   )r   r   r   irz   r   s         r9   r   zReaderWds.__iter__  s      7?OO!99;; 	t599
 ##$677BBB 	 	FH%F 3*62/6))))FAA	 	rH   c                 >    |                                  | j        z  }|S r[   )r   r   )r   r=   s     r9   __len__zReaderWds.__len__  s!    2244t7GGrH   c                     J d            )NFzNot supportedr?   )r   r   basenameabsolutes       r9   	_filenamezReaderWds._filename  s    %o%%%rH   c                    | j         |                                  g }| j         D ]g}| j        |v r|| j                 }nd|v r|d         | j        z   }n
J d            |                    |           t          |          | j        k    r nh|S )z0 Return all filenames in dataset, overrides baseN__key__FzNo supported name field present)r   r   r   r   r%   r_   r=   )r   r   r  r1   rz   rB   s         r9   r>   zReaderWds.filenames  s    7?OOg 		 		F F**d/0f$$i(4<7?????LL5zzT--- .rH   )Nr   FNr   r   r   Nr   rq   rr   r   r   NNr[   )FF)rC   rD   rE   r!   r
   boolrF   ry   r   r   r   r   r   r   r   r   r  r>   r   r   s   @r9   r   r     s        #'  %)-(,+"'##% *1515#H HH 3-H 	H
 H "#H H H H  ~H H  H H !H H  "*#!H" "*##H H H H H HT' ' '
 *.P P!#P P P P3. 3. 3.j  ' ' '  0  & & & &       rH   r   )r   )rp   rq   rr   r   )Fr   ru   r)   loggingr   r"   r   r   dataclassesr   	functoolsr   	itertoolsr   typingr   r   r   r	   r
   r   r   torch.distributeddistributedr   r+   PILr   torch.utils.datar   r   r   
webdatasetr&   webdataset.filtersr   r   webdataset.shardlistsr   webdataset.tariteratorsr   r   r   r   ImportErrorr   r   readerr   shared_countr   	getLoggerrC   r.   rF   environrN   r   r   r:   r<   r!   rj   ro   r   r   PipelineStager   r   r   r?   rH   r9   <module>r     s    
			    				  



 ! ! ! ! ! !             = = = = = = = = = = = = = = = =                     F F F F F F F F F F55555555111111bbbbbbbbbbbbb   
CKKK & % % % % %       % % % % % %
'
H
%
%c"*..);TBBCC c"*..);TBBCC    0        0S 0 0 0 0 0f   ! ! ! !H+ + + ?B B B B Bc' B B B B> K)- )- )- )- )- )- )- )-XL L L L L L L L L Ls    A? ?	B
B