Ngo>ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZddl Z ddl!m"Z"ddl#m$Z$m%Z%dd l&m'Z'dd l(m)Z)gd Z*d#d ee+d e+fdZ,de+fdZ-Gddej.Z/Gdde)Z0Gdde$Z1Gdde1Z2Gdde1Z3Gdde3Z4GddZ5 Gdd Z6 e67d!"a8dS)$N) OrderedDict)IOBase) TracebackType) AnyCallableDictIOIterableIteratorListMutableMappingOptionalSetTypeTypeVarUnion)urlparse)download) EventLoggerVTYPE)NonBlockingIOManager)Protocol)LazyPath PathManager get_cache_dir file_lock cache_dirreturnc|2tjtjdd} t|tj|tjsJn~#ttf$rjtj tj d}tjt}||d|d|}YnwxYw|S)aS Returns a default directory to cache static files (usually downloaded from Internet), if None is provided. Args: cache_dir (None or str): if not None, will be returned as is. If None, returns the default cache directory as: 1) $FVCORE_CACHE, if set 2) otherwise ~/.torch/iopath_cache N FVCORE_CACHEz~/.torch/iopath_cache iopath_cachez is not accessible! Using z instead!)ospath expandusergetenv g_pathmgrmkdirsaccessW_OKOSErrorAssertionErrorjointempfile gettempdirlogging getLogger__name__warning)rtmp_dirloggers Q/var/www/html/ai-engine/env/lib/python3.11/site-packages/iopath/common/file_io.pyrr*sG&& In&= > >  ###yBG,,,,,, ^ $',,x244nEE"8,,)QQwQQQRRR  s;A22A8C-,C-r#ctj|} tj|dn#t$rYnwxYwt j|dzdS)a! A file lock. Once entered, it is guaranteed that no one else holds the same lock. Others trying to enter the lock will block for 30 minutes and raise an exception. This is useful to make sure workers don't cache files to the same location. Args: path (str): a path to be locked. This function will create a lock named `path + ".lock"` Examples: filename = "/path/to/file" with file_lock(filename): if not os.path.isfile(filename): do_create_file() Texist_okz.locki)timeout)r"r#dirnamemakedirsr* portalockerLock)r#r:s r5rrEsr&good##G  Gd+++++        D7ND 9 9 99s8 AAc~eZdZdZddgZdegefddfdZdefdZdefd Z d effd Z d Z deffd Z xZ S)ra= A path that's lazily evaluated when it's used. Users should be careful to not use it like a str, because it behaves differently from a str. Path manipulation functions in Python such as `os.path.*` all accept PathLike objects already. It can be materialized to a str using `os.fspath`. __getstate__ __setstate__funcrNc"||_d|_dS)z Args: func: a function that takes no arguments and returns the actual path as a str. It will be called at most once. N)_func_value)selfrAs r5__init__zLazyPath.__init__ts  %) cP|j||_|jSN)rDrCrEs r5 _get_valuezLazyPath._get_value}s! ; **,,DK{rGc*|SrI)rKrJs r5 __fspath__zLazyPath.__fspath__s   rGnamec|tjvr!t|S|jt d|dt |j|S)Nz)Uninitialized LazyPath has no attribute: .)rKEPT_ATTRIBUTESsuper __getattr__rDAttributeErrorgetattr)rErN __class__s r5rSzLazyPath.__getattr__s] 8+ + +77&&t,, , ;  !TT!T!T!TUU Ut{D)))rGcH|jtd|j|S)Nz,Uninitialized LazyPath is not subscriptable.)rD TypeError)rEkeys r5 __getitem__zLazyPath.__getitem__s& ; JKK K{3rGc`|j|jStSrI)rDrR__str__rErVs r5r\zLazyPath.__str__s' ; "; 77??$$ $rG)r1 __module__ __qualname____doc__rQrstrrFrKrMrSrZr\ __classcell__rVs@r5rrfs  &~6O*Xb#g.*4****C !C!!!!*******   %%%%%%%%%%%rGrczeZdZdZd dZdeeedeedeeddfdZ de e fd Z dS) TabularIOzF Context Manager interface to be used by PathHandler methods. rcdSrIrJs r5 __enter__zTabularIO.__enter__ rGexc_typeexc_valexc_tbNcdSrIrg)rErjrkrls r5__exit__zTabularIO.__exit__s rGcdSrIrgrJs r5__iter__zTabularIO.__iter__rirG)rre) r1r^r_r`rhrr BaseExceptionrrnr rrprgrGr5reres     4 ./ -( '        (3-      rGreceZdZdZdZ d)deejjddffd Z de e e fddfdZ dee fd Zd*d e d ede de fd Z d*de de dede ddf dZ d+d e de dede def dZ d,d e de dede deee eeff dZ d-d e de deedgdfdede def dZd)d ee de defdZde defdZ d*de de dede def dZde de de defdZ d e de defd Z!d e de defd!Z"d e de defd"Z#d e de dee fd#Z$d e de ddfd$Z%d e de ddfd%Z&de de de defd&Z'd ee dfde defd'Z(d e de fd(Z)xZ*S). PathHandlerz PathHandler is a base class that defines common I/O functionality for a URI protocol. It routes I/O for a generic URI which may look like "protocol://*" or a canonical filepath "/foo/bar/baz". TNasync_executorrctd|_||_ ddlm}||dS#t $rYdSwxYw)a. When registering a `PathHandler`, the user can optionally pass in a `Executor` to run the asynchronous file operations. NOTE: For regular non-async operations of `PathManager`, there is no need to pass `async_executor`. Args: async_executor (optional `Executor`): Used for async file operations. Usage: ``` path_handler = NativePathHandler(async_executor=exe) path_manager.register_handler(path_handler) ``` Nr)setup_handler_defaults)rRrF_non_blocking_io_manager_non_blocking_io_executoriopath.common.setup_defaultsrv ImportError)rErtrvrVs r5rFzPathHandler.__init__s{$ (,%)7&  K K K K K K " "4 ( ( ( ( (    DD sA AAkwargsc:|jr7t|dkr"td|dSt jt }|D].\}}|d||/dS)a Checks if the given arguments are empty. Throws a ValueError if strict kwargs checking is enabled and args are non-empty. If strict kwargs checking is disabled, only a warning is logged. Args: kwargs (Dict[str, Any]) rzUnused arguments: {}z$[PathManager] {}={} argument ignoredN) _strict_kwargs_checklen ValueErrorformatr/r0r1itemsr2)rEr{r4kvs r5 _check_kwargszPathHandler._check_kwargss  $ T6{{Q !7!>!>v!F!FGGG&x00F  T T1ELLQPQRRSSSS T TrGct)zg Returns: List[str]: the list of URI prefixes this PathHandler can support NotImplementedErrorrJs r5_get_supported_prefixesz#PathHandler._get_supported_prefixess "###rGFr#forcec t)a Get a filepath which is compatible with native Python I/O such as `open` and `os.path`. If URI points to a remote resource, this function may download and cache the resource to local disk. In this case, the cache stays on filesystem (under `file_io.get_cache_dir()`) and will be used by a different run. Therefore this function is meant to be used with read-only resources. Args: path (str): A URI supported by this PathHandler force(bool): Forces a download from backend if set to True. Returns: local_path (str): a file path which exists on the local file system rrEr#rr{s r5_get_local_pathzPathHandler._get_local_path""###rG local_pathdst_path overwritec t) Copies a local file to the specified URI. If the URI is another local path, this should be functionally identical to copy. Args: local_path (str): a file path which exists on the local file system dst_path (str): A URI supported by this PathHandler overwrite (bool): Bool flag for forcing overwrite of existing URI Returns: status (bool): True on success rrErrrr{s r5_copy_from_localzPathHandler._copy_from_localrrGr mode bufferingc trIrrEr#rrr{s r5_opentzPathHandler._opents"###rGc t) Open a stream to a URI, similar to the built-in `open`. Args: path (str): A URI supported by this PathHandler mode (str): Specifies the mode in which the file is opened. It defaults to 'r'. buffering (int): An optional integer used to set the buffering policy. Pass 0 to switch buffering off and an integer >= 1 to indicate the size in bytes of a fixed-size chunk buffer. When no buffering argument is given, the default buffering policy depends on the underlying I/O implementation. Returns: file: a file-like object. rrs r5_openzPathHandler._opens&"###rGcallback_after_file_closec hdtfd|Dstd||jstd|j|_ |j|||j||fi|||S#t$rFtj t}| d| YdSwxYw) a Open a file with asynchronous methods. `f.write()` calls will be dispatched asynchronously such that the main program can continue running. `f.read()` is an async method that has to run in an asyncio event loop. NOTE: Writes to the same path are serialized so they are written in the same order as they were called but writes to distinct paths can happen concurrently. Usage (write, default / without callback function): for n in range(50): results = run_a_large_task(n) # `f` is a file-like object with asynchronous methods with path_manager.opena(uri, "w") as f: f.write(results) # Runs in separate thread # Main process returns immediately and continues to next iteration path_manager.async_close() Usage (write, advanced / with callback function): # To asynchronously write to storage: def cb(): path_manager.copy_from_local( "checkpoint.pt", uri ) f = pm.opena("checkpoint.pt", "wb", callback_after_file_close=cb) torch.save({...}, f) f.close() Usage (read): async def my_function(): return await path_manager.opena(uri, "r").read() Args: ...same args as `_open`... callback_after_file_close (Callable): An optional argument that can be passed to perform operations that depend on the asynchronous writes being completed. The file is first written to the local disk and then the callback is executed. buffering (int): An optional argument to set the buffer size for buffered asynchronous writing. Returns: file: a file-like object with asynchronous methods. >abwc3 K|]}|vV dSrIrg).0m valid_modess r5 z%PathHandler._opena..cs(221 #222222rGz.`opena` mode must be write or append for path F)bufferedexecutor)r#io_objrrzAn exception occurred in `NonBlockingIOManager`. This is most likely due to invalid `opena` args. Make sure they match the `open` args for the `PathHandler`.N) allrrwrrxget_non_blocking_io_get_path_with_cwdrr/r0r1 exception _async_close)rEr#rrrr{r4rs @r5_openazPathHandler._opena-s9j&oo 2222T22222 VTdTTUU U , ,@7---D )  0DD,,T22!tz$7777*C# E   &x00F   D             s=BA C$#C$c |js.tjt}|d|||j|r||ndS)a Ensures that desired async write threads are properly joined. Args: path (str): Pass in a file path to wait until all asynchronous activity for that path is complete. If no path is passed in, then this will wait until all asynchronous jobs are complete. Returns: status (bool): True on success zJThis is an async feature. No threads to join because `opena` was not used.N)rwr/r0r1r2r_joinr)rEr#r{r4s r5 _async_joinzPathHandler._async_joins, &x00F NN(    6""",22-1 ;D # #D ) ) )t   rGc |js.tjt}|d|||jS)z Closes the thread pool used for the asynchronous operations. Returns: status (bool): True on success zNThis is an async feature. No threadpool to close because `opena` was not used.)rwr/r0r1r2r_close_thread_pool)rEr{r4s r5rzPathHandler._async_closese, &x00F NN(    6""",??AAArGsrc_pathc t)X Copies a source path to a destination path. Args: src_path (str): A URI supported by this PathHandler dst_path (str): A URI supported by this PathHandler overwrite (bool): Bool flag for forcing overwrite of existing file Returns: status (bool): True on success r)rErrrr{s r5_copyzPathHandler._copys"###rGc t) Moves (renames) a source path to a destination path. Args: src_path (str): A URI supported by this PathHandler dst_path (str): A URI supported by this PathHandler Returns: status (bool): True on success rrErrr{s r5_mvzPathHandler._mvs"###rGc t) Checks if there is a resource at the given URI. Args: path (str): A URI supported by this PathHandler Returns: bool: true if the path exists rrEr#r{s r5_existszPathHandler._exists"###rGc t)z Checks if the resource at the given URI is a file. Args: path (str): A URI supported by this PathHandler Returns: bool: true if the path is a file rrs r5_isfilezPathHandler._isfilerrGc t) Checks if the resource at the given URI is a directory. Args: path (str): A URI supported by this PathHandler Returns: bool: true if the path is a directory rrs r5_isdirzPathHandler._isdirrrGc tz List the contents of the directory at the provided URI. Args: path (str): A URI supported by this PathHandler Returns: List[str]: list of contents in given path rrs r5_lszPathHandler._lsrrGc t) Recursive directory creation function. Like mkdir(), but makes all intermediate-level directories needed to contain the leaf directory. Similar to the native `os.makedirs`. Args: path (str): A URI supported by this PathHandler rrs r5_mkdirszPathHandler._mkdirss"###rGc t) Remove the file (not directory) at the provided URI. Args: path (str): A URI supported by this PathHandler rrs r5_rmzPathHandler._rms"###rGc t)z Symlink the src_path to the dst_path Args: src_path (str): A URI supported by this PathHandler to symlink from dst_path (str): A URI supported by this PathHandler to symlink to rrs r5_symlinkzPathHandler._symlink s"###rGc t)al Set the current working directory. PathHandler classes prepend the cwd to all URI paths that are handled. Args: path (str) or None: A URI supported by this PathHandler. Must be a valid absolute path or None to set the cwd to None. Returns: bool: true if cwd was set without errors rrs r5_set_cwdzPathHandler._set_cwds"###rGc|S)a5 Default implementation. PathHandler classes that provide a `_set_cwd` feature should also override this `_get_path_with_cwd` method. Args: path (str): A URI supported by this PathHandler. Returns: path (str): Full path with the cwd attached. rgrEr#s r5rzPathHandler._get_path_with_cwd%s  rGrIFrrrr)rNr)+r1r^r_r`r}r concurrentfuturesExecutorrFrrarrr rboolrrintrerrr bytesrrrrrrrrrrrrrrrrrrbrcs@r5rsrss  AE !3!<= :TDcNTtTTTT"$c$$$$$$C$$$PS$$$$(AF$$$),$9=$QT$ $$$$(<>$$$"$58$IL$ $$$$ <>$$$"$58$IL$ r#w5 ! "$$$$0FJ V V V V $,HdVT\,B#C V  V  V  V V V V p         .BSBTBBBB"?D$$$'*$7;$OR$ $$$$ $C $3 $# $$ $ $ $ $ $C $3 $4 $ $ $ $ $C $3 $4 $ $ $ $ $3 $# $$ $ $ $ $ $ $s $tCy $ $ $ $ $C $3 $4 $ $ $ $$$s$t$$$$$$$s$t$$$$ $U39- $ $ $ $ $ $ s s        rGrsc.eZdZdZdZ d&deejjddffd Z d'de de d e de fd Z d'd e d e d e d e ddf dZ d(de de dedee dee dee de deed e deee eeffdZ d'de d e d e d e de f dZde d e d e de fdZde d e d e de fdZde d e de fdZde d e de fdZde d e de fd Zde d e dee fd!Zde d e ddfd"Zde d e ddfd#Zdee dfd e de fd$Zde de fd%Z xZ!S))NativePathHandlerz Handles paths that can be accessed using Python native system calls. This handler uses `open()` and `os.*` calls on the given path. NrtrcJt|dSrI)rRrF)rErtrVs r5rFzNativePathHandler.__init__<s# (((((rGFr#rr{c T||tj|SrI)rr"fspathrs r5rz!NativePathHandler._get_local_pathBs$ 6"""yrGrrrc ||||}||}|jd|||d|sJdS)N)rrrrg)rrrrs r5rz"NativePathHandler._copy_from_localFs 6""",,Z88 **844tz (i  KQ       rGrrTrrencodingerrorsnewlineclosefdopenerc || t|||||||||S)u Open a path. Args: path (str): A URI supported by this PathHandler mode (str): Specifies the mode in which the file is opened. It defaults to 'r'. buffering (int): An optional integer used to set the buffering policy. Pass 0 to switch buffering off and an integer >= 1 to indicate the size in bytes of a fixed-size chunk buffer. When no buffering argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer is chosen using a heuristic trying to determine the underlying device’s “block size” and falling back on io.DEFAULT_BUFFER_SIZE. On many systems, the buffer will typically be 4096 or 8192 bytes long. encoding (Optional[str]): the name of the encoding used to decode or encode the file. This should only be used in text mode. errors (Optional[str]): an optional string that specifies how encoding and decoding errors are to be handled. This cannot be used in binary mode. newline (Optional[str]): controls how universal newlines mode works (it only applies to text mode). It can be None, '', ' ', ' ', and ' '. closefd (bool): If closefd is False and a file descriptor rather than a filename was given, the underlying file descriptor will be kept open when the file is closed. If a filename is given closefd must be True (the default) otherwise an error will be raised. opener (Optional[Callable]): A custom opener can be used by passing a callable as opener. The underlying file descriptor for the file object is then obtained by calling opener with (file, flags). opener must return an open file descriptor (passing os.open as opener results in functionality similar to passing None). See https://docs.python.org/3/library/functions.html#open for details. Returns: file: a file-like object. )rrrrrr)ropenr) rEr#rrrrrrrr{s r5rzNativePathHandler._openPsVh 6"""  # #D ) )     rGrc D||||}||}tj|rE|sCt jt}|d |dS tj ||dS#t$rY}t jt}|d t|Yd}~dSd}~wwxYw)r#Destination file {} already exists.FTzError in file copy - {}N)rrr"r#existsr/r0r1errorrshutilcopyfile Exceptionra)rErrrr{r4es r5rzNativePathHandler._copys 6"""**844**844 7>>( # # I &x00F LL>EEhOO P P P5  OHh / / /4   &x00F LL299#a&&AA B B B55555 s%B<< DADDc @||||}||}tj|rCt jt}|d |dS tj ||dS#t$rY}t jt}|d t|Yd}~dSd}~wwxYw)rrFTzError in move operation - {}N)rrr"r#rr/r0r1rrrmoverrarErrr{r4rs r5rzNativePathHandler._mvs  6"""**844**844 7>>( # # &x00F LL>EEhOO P P P5  K( + + +4   &x00F LL7>>s1vvFF G G G55555 s#B:: DADDc ||||}||}tjt}t j|s*|d |dSt j|r*|d |dS t j ||dS#t$r@}|d t|Yd}~dSd}~wwxYw)a Creates a symlink to the src_path at the dst_path Args: src_path (str): A URI supported by this PathHandler dst_path (str): A URI supported by this PathHandler Returns: status (bool): True on success zSource path {} does not existFz#Destination path {} already exists.TzError in symlink - {}N) rrr/r0r1r"r#rrrsymlinkrrars r5rzNativePathHandler._symlinks5 6"""**844**844"8,,w~~h''  LL8??II J J J5 7>>( # #  LL>EEhOO P P P5  Jx * * *4    LL077A?? @ @ @55555 s,D E  5EE c ||tj||SrI)rr"r#rrrs r5rzNativePathHandler._exists8 6"""w~~d55d;;<<>>?? ? trGc|s|Stj|js|n$tj|j|SrI)r"r#normpathr r,rs r5rz$NativePathHandler._get_path_with_cwdsI Kw DDDrw||DIt'D'D   rGrIr)rrNNNTN)"r1r^r_r`r rrrrrFrarrrrrrrr rrrrrrrrr rrrrrrbrcs@r5rr3s DAE)) !3!<=) )))))) CPS AF   ), 9= QT     "& $!%%)> > > >  > 3- >  > #> > "> >  r#w5 ! "> > > > B?D'*7;OR <C3#$6st8=C=3=4=====C=3=4====<3<#<$<<<<99s9tCy9999C34stU39-  s s        rGrceZdZdZdZdfd ZdeefdZ dded e d e ed e def d Z ddedede d e deeeeeff dZxZS)HTTPURLHandlerz/ Download URLs and cache them to disk. rNcVti|_dSrI)rRrF cache_mapr]s r5rFzHTTPURLHandler.__init__!s$ )+rGc gdS)N)zhttp://zhttps://zftp://rgrJs r5rz&HTTPURLHandler._get_supported_prefixes%s0000rGFr#rrr{c |||s4||jvs+tj|j|st jt}t|}tj t|tj |j d}| dd}t||jkr&|dddzt!jjz}tj ||} t'| 5tj| s:|d|t/|||} dddn #1swxYwY|d|| | |j|<|j|S) This implementation downloads the remote resource and caches it locally. The resource will only be downloaded if not previously requested. /rNd_zDownloading {} ...)filenamezURL {} cached in {})rrr"r#rr/r0r1rr,rr:lstripsplitr~MAX_FILENAME_LENuuiduuid4hexrrinforr) rEr#rrr{r4 parsed_urlr:rcacheds r5rzHTTPURLHandler._get_local_path(s 6"""  *4>))7>>$."677*&x00F!$Jglli(("'//*/:P:PQT:U:U*V*VGzz#r*H8}}t444#DSD>C/$*,,2BBW\\'844F6"" H Hw~~f--HKK 4 ; ;D A ABBB%dGhGGGF H H H H H H H H H H H H H H H KK-44T6BB C C C#)DN4 ~d##sAGG Grrrrc |||dvs(Jd|jj||dksJ|jjd||d}t ||S)a Open a remote HTTP path. The resource is first downloaded and cached locally. Args: path (str): A URI supported by this PathHandler mode (str): Specifies the mode in which the file is opened. It defaults to 'r'. buffering (int): Not used for this PathHandler. Returns: file: a file-like object. )rrbz%{} does not support open with {} moderz* does not support the `buffering` argumentF)r)rrrVr1rr)rEr#rrr{rs r5rzHTTPURLHandler._openKs 6"""{"""$K$R$R N #T% % """ OOOn% Q Q Q OO))$e)<< J%%%rGrN)FNr)r1r^r_r`rrFr rarrrrrrrr rrrbrcs@r5rrs,,,,,,1c1111 #' !$!$!$!$C= !$  !$ !$!$!$!$H<>&&&"&58&IL& r#w5 ! "&&&&&&&&rGrc beZdZdZdZdedefdZdeefdZd ded e d e deffd Z xZ S) OneDrivePathHandlerz< Map OneDrive (short) URLs to direct download links zhttps://1drv.ms/u/s! one_drive_urlrctjt|d}|dddddd}d|d}|S) a Converts a short OneDrive URI into a download link that can be used with wget Args: one_drive_url (str): A OneDrive URI supported by this PathHandler Returns: result_url (str): A direct download URI for the file zutf-8rr+-=z'https://api.onedrive.com/v1.0/shares/u!z /root/content)base64 b64encoderdecodereplacerstrip)rEr(data_b64data_b64_string result_urls r5 create_one_drive_direct_downloadz4OneDrivePathHandler.create_one_drive_direct_downloadms#E-$A$ABB OOG $ $ , ,S# 6 6 > >sC H H O OPS T T  Uo T T T rGc|jgSrI)ONE_DRIVE_PREFIXrJs r5rz+OneDrivePathHandler._get_supported_prefixess%&&rGFr#rr{c tjt}||}|d|d|t jtj|fd|i|S)rzURL z mapped to direct download link r) r/r0r1r5r rRrr"r)rEr#rr{r4 direct_urlrVs r5rz#OneDrivePathHandler._get_local_pathsy "8,,::4@@  M4MMMMNNN&uww&ry'<'<TTETVTTTrGr) r1r^r_r`r7rar5r rrrrrbrcs@r5r'r'fs.cc&'c'''' U UC U U UPS U U U U U U U U U UrGr'ceZdZdZd1dZdeeejfde fdZ de de ee fddfd Z ded ed ede ee ffd Z d2ded ed ededef dZ d3ded ed ededeeeeeff dZ d4ded ed edeedgdfdedef dZdededefdZdedefdZ d5dededededef dZdedededefdZd5dedededefdZ d5d ededededef d!Z dededefd"Z!dededefd#Z"dededefd$Z#dedede$efd%Z%dededdfd&Z&dededdfd'Z'dedededefd(Z(deedfdedefd)Z) d6de d+eddfd,Z*d-eddfd.Z+d6d/Z,dededededef d0Z-dS)7ra A class for users to open generic paths or translate generic paths to file names. path_manager.method(path) will do the following: 1. Find a handler by checking the prefixes in `self._path_handlers`. 2. Call handler.method(path) on the handler that's found rNct|_ t|_ d|_ t |_ d|_dS)NT)r_path_handlersr_native_path_handlerr set_async_handlers_enable_loggingrJs r5rFzPathManager.__init__s^@K  2C1D1D! $(  25 $  rGr#ctj|}|jD]&}||r|j|cS'|jS)a  Finds a PathHandler that supports the given path. Falls back to the native PathHandler if no other handler is found. Args: path (str or os.PathLike): URI path to resource Returns: handler (PathHandler) )r"rr<keys startswithr=)rEr#ps r5__get_path_handlerzPathManager.__get_path_handlersey$))++ . .Aq!! .*1---- .((rGhandlerkvscH|jr|||dt|j |dS#t $r9tjt}| dd|_YdSwxYwdS)z Logs a dictionary of key-value pairs to a given path handler. Args: handler (PathHandler): The path handler to send the key-value pairs to. kvs (Dict): Dict of Key-value pairs to be logged. rFz]An exception occurred in telemetry logging.Disabling telemetry to prevent further exceptions.FN) r@add_keysadd_keytyper1 log_eventrr/r0r)rErFrGr4s r5__log_tmetry_keyszPathManager.__log_tmetry_keyss   -   S ! ! ! OOItG}}'= > > > -!!##### - - - *844  I(-$$$$  - - -sA?BBrrci}d|d<d|vrd|d<nd|vrd|d<n d|vrd |d<||d <d |vrd |d <nd|d <||d<|S)z Helper function to return common set of key-value pairs applicable to open apis. Args: path (str): mode (str): buffering (int): roprreadrrwriterappendrrbinaryrtextr#rg)rEr#rrrGs r5__get_open_keyszPathManager.__get_open_keyssD $;; CKK D[[!CKK D[["CK$K $;;$CMM"CMF  rGrrr{c H||j|||fi|S)a Open a tabular data source. Only reading is supported. The opent() returns a Python iterable collection object, compared to bytes/text data with open() Args: path (str): A URI supported by this PathHandler mode (str): Specifies the mode in which the file is opened. It defaults to 'r' buffering (int): number of rows fetched and cached Returns: A TabularIO context manager object )_PathManager__get_path_handlerrrs r5opentzPathManager.opents1 4t&&t,,3D$ TTVTTTrGrc ||}||j|j||fd|i|}||||}||||S)rr)rW set_loggingr@r_PathManager__get_open_keys_PathManager__log_tmetry_keys)rEr#rrr{rFbretrGs r5rzPathManager.opens&))$// D0111w}T4GG9GGG""4y99 w,,, rGrc d|vr ||d<||d<||j||fi|}d|vr-|j|||S)a Open a file with asynchronous methods. `f.write()` calls will be dispatched asynchronously such that the main program can continue running. `f.read()` is an async method that has to run in an asyncio event loop. NOTE: Writes to the same path are serialized so they are written in the same order as they were called but writes to distinct paths can happen concurrently. Usage (write, default / without callback function): for n in range(50): results = run_a_large_task(n) # `f` is a file-like object with asynchronous methods with path_manager.opena(uri, "w") as f: f.write(results) # Runs in separate thread # Main process returns immediately and continues to next iteration path_manager.async_close() Usage (write, advanced / with callback function): # To asynchronously write to storage: def cb(): path_manager.copy_from_local("checkpoint.pt", uri) f = pm.opena("checkpoint.pt", "wb", callback_after_file_close=cb) torch.save({...}, f) f.close() Usage (read): async def my_function(): return await path_manager.opena(uri, "r").read() Args: ... callback_after_file_close (Callable): Only used in write mode. An optional argument that can be passed to perform operations that depend on the asynchronous writes being completed. The file is first written to the local disk and then the callback is executed. Returns: file: a file-like object with asynchronous methods. rrr)rWrr?add)rEr#rrrr{non_blocking_ios r5openazPathManager.opena,sb $;;2KF. /"+F; >$11$77>       $;;  $ $T%<%Src and dest paths must be supported by the same path handler.mvrk)rWrr\rErrr{rFr]rGs r5rnzPathManager.mvs"&&    $ $       L    ))(33w{8X88888BB w,,, rGrc tj|}||} |j|fd|i|}n#t$r|j|fi|}YnwxYwd||d}||||S)a Get a filepath which is compatible with native Python I/O such as `open` and `os.path`. If URI points to a remote resource, this function may download and cache the resource to local disk. Args: path (str): A URI supported by this PathHandler force(bool): Forces a download from backend if set to True. Returns: local_path (str): a file path which exists on the local file system rget_local_path)rOr#r)r"rrWrrXr\)rEr#rr{rFr]rGs r5rqzPathManager.get_local_pathsy))$// ;*7*4GGuGGGDD ; ; ;*7*4::6::DDD ;%teDD w,,, s<AArc tj|s Jd|||}d|||d}|jd|||d|}||||S)rz local_path = copy_from_local)rOr#rr)rrrrg)r"r#rrWrr\)rErrrr{rFrGr]s r5rszPathManager.copy_from_locals"w~~j))GG+G:+G+GGGG))(33$ "    (w' !H   MS   w,,, rGc ||}|j|fi|}d|d}||||S)rrrOr#)rWrr\rEr#r{rFr]rGs r5rzPathManager.existsX))$//wt..v..t,, w,,, rGc ||}|j|fi|}d|d}||||S)z Checks if there the resource at the given URI is a file. Args: path (str): A URI supported by this PathHandler Returns: bool: true if the path is a file rru)rWrr\rvs r5rzPathManager.isfilerwrGc ||}|j|fi|}d|d}||||S)rrru)rWrr\rvs r5rzPathManager.isdir/sX))$//w~d--f--d++ w,,, rGc D||j|fi|Sr)rWrrs r5lszPathManager.ls?s-1t&&t,,0@@@@@rGc ||}|j|fi|}d|d}||||S)rr'ru)rWrr\rvs r5r'zPathManager.mkdirsKsX))$//wt..v..t,, w,,, rGc ||}|j|fi|}d|d}||||S)rrmru)rWrr\rvs r5r~zPathManager.rmZsX))$//w{4**6**4(( w,,, rGc ||||ksJ||}|j||fi|}d||d}||||S)zSymlink the src_path to the dst_path Args: src_path (str): A URI supported by this PathHandler to symlink from dst_path (str): A URI supported by this PathHandler to symlink to rrk)rWrr\ros r5rzPathManager.symlinkgs&&    $ $X . .////))(33w(==f==hGG w,,, rGc | |jdS||p|j}||p|jj|fi|r ||_d}nd}d|d}||||S)aq Set the current working directory. PathHandler classes prepend the cwd to all URI paths that are handled. Args: path (str) or None: A URI supported by this PathHandler. Must be a valid absolute Unix path or None to set the cwd to None. Returns: bool: true if cwd was set without errors NTFset_cwdru)r rWrr\rvs r5rzPathManager.set_cwdxs 4 " "4#449 5 5 >t N Nv N N DIDDD-- w,,, rGTallow_overridec tjt}t|ts J|t|t r|r||_ntddS|D]}||j vr ||j |<t|j |}|rt|tkr^| d|dd tjdz| d|d |d ||j |<t!d|d |d t#t%|j d d|_ dS)z Register a path handler associated with `handler._get_supported_prefixes` URI prefixes. Args: handler (PathHandler) allow_override (bool): allow overriding existing handler for prefix za`NativePathHandler` is registered by default. Use the `allow_override=True` kwarg to override it.Nz-[PathManager] Attempting to register prefix 'z!' from the following call stack: )limitz[PathManager] Prefix 'z' is already registered by zk. We will override the old handler. To avoid such conflicts, create a project-specific PathManager instead.z' already registered by !c|dS)Nrrg)ts r5z.PathManager.register_handler..s adrGT)rYreverse)r/r0r1 isinstancersrr=rrr<rKr&r2r, traceback format_stackKeyErrorrsortedr)rErFrr4prefixold_handler_types r5register_handlerzPathManager.register_handlers"8,,';//88888 g0 1 1  ,3)) B F5577  FT000.5#F+#D$7$?@@  9$$NN6666'')"8q"A"A"ABBC NN###.### /6#F++`V``M]``` * 4&,,..NND Q Q Q  rGenablecd||j_|jD] }||_ dS)a Toggles strict kwargs checking. If enabled, a ValueError is thrown if any unused parameters are passed to a PathHandler function. If disabled, only a warning is given. With a centralized file API, there's a tradeoff of convenience and correctness delegating arguments to the proper I/O layers. An underlying `PathHandler` may support custom arguments which should not be statically exposed on the `PathManager` function. For example, a custom `HTTPURLHandler` may want to expose a `cache_timeout` argument for `open()` which specifies how old a locally cached resource can be before it's refetched from the remote server. This argument would not make sense for a `NativePathHandler`. If strict kwargs checking is disabled, `cache_timeout` can be passed to `PathManager.open` which will forward the arguments to the underlying handler. By default, checking is enabled since it is innately unsafe: multiple `PathHandler`s could reuse arguments with different semantic meanings or types. Args: enable (bool) N)r=r}r<values)rErrFs r5set_strict_kwargs_checkingz&PathManager.set_strict_kwargs_checkingsC,:@!6*1133 2 2G+1G ( ( 2 2rGc||_dSrI)r@)rEenable_loggings r5rZzPathManager.set_loggings-rGc ||}|jJ||}|jJ|j|fi|}|j||fd|i|S)Nr)rWrr)rErrrr{ src_handler dst_handler local_files r5rlz!PathManager._copy_across_handlerss--h77 *666--h77 +7770[0DDVDD +{+   ,5 9?   rGr%rr)rrNr)T).r1r^r_r`rFrrar"PathLikersrWrrr\rr[rrerXr rrrrrrarrerhrjrnrqrsrrrr r{r'r~rrrrrZrlrgrGr5rrs3    <)uS"+-='>);))))"--4U ;K-PT----0CsstCQVJGW8<>UUU"U58UILU UUUU&<>"58IL r#w5 ! "@FJ ==== = $,HdVT\,B#C =  = ====~B C D     ?D'*7;OR 43#83tss8AF!!!),!9=!QT! !!!!F3#$ 3#$ # As Ac Ad3i A A A A 3 # $     s c d    scd"E#t),:<@: : ": 48: : : : : x22$22228....    '*  7;  GJ         rGrcReZdZdZdZiZeedfdefdZedZ dS)PathManagerFactorya PathManagerFactory is the class responsible for creating new PathManager instances and removing them when no longer needed. PathManager can be instantiated directly too, but it is recommended that you use PathManagerFactory to create them. global_path_managerFrc|tjvrPttj|<|r3 ddlm}|tj|n#t $rYnwxYwtj|S)a$ Get the path manager instance associated with a key. A new instance will be created if there is no existing instance associated with the key passed in. Args: key (str): defaults_setup (bool): If True, setup_defaults is called. r)setup_defaults)rpm_listrryrrz)rYdefaults_setuprs r5getzPathManagerFactory.get s (0 0 0.9mm  &s + KKKKKK"N#5#=c#BCCCC"D")#..s!A AAcf|tjvr"tj|}~dSdS)zn Remove the path manager instance associated with a key. Args: key (str): N)rrpop)rY_pms r5rzPathManagerFactory.remove7s8 $, , ,$,0055C - ,rGN) r1r^r_r`GLOBAL_PATH_MANAGERr staticmethodrrrrgrGr5rrsu0G#E//k///\/*\rGrT)rrI)9r-concurrent.futuresrrr/r"rr-rr collectionsriortypesrtypingrrrr r r r r rrrrr urllib.parserr<iopath.common.downloadriopath.common.event_loggerrriopath.common.non_blocking_iortyping_extensionsr__all__rarrrrrersrrr'rrrr&rgrGr5rs    ######"!!!!!++++++99999999>>>>>>&&&&&& D C CXc]c6:C::::B1%1%1%1%1%r{1%1%1%h        *AAAAA+AAAH d d d d d d d d NI&I&I&I&I&[I&I&I&X)U)U)U)U)U.)U)U)UXi  i  i  i  i  i  i  i  X,........b  " "$ " 7 7 rG