çgp{ UddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZe rddlZddlmZedZn edZedZejdZeeegefZGd d Z Gd d e Z!Gd de Z"Gdde Z#Gdde Z$Gdde Z%Gdde Z&Gdde Z'GddeeefZ(Gdde Z)de iZ*de+d<d'd(d&Z,e e!e%e"e$e#e&e'e)f D] Z-e,e-dS))) annotationsN)FutureThreadPoolExecutor) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTupleOptional OrderedDictTypeVar) ParamSpecPTfsspeccJeZdZUdZdZded<dd ZddZddZddZ ddZ dS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]name blocksizeintfetcherFetchersizereturnNonech||_d|_||_||_d|_d|_d|_dSNr)rnblocksrr hit_count miss_counttotal_requested_bytes)selfrrrs J/var/www/html/ai-engine/env/lib/python3.11/site-packages/fsspec/caching.py__init__zBaseCache.__init__:s:"   %&"""start int | Nonestopbytescn|d}||j}||jks||krdS|||S)Nrr()rrr%r)r+s r&_fetchzBaseCache._fetchDsF =E <9D DI  $3||E4(((r(c0d|_d|_d|_dS)zAReset hit and miss counts for a more ganular report e.g. by file.rN)r"r#r$r%s r& _reset_statszBaseCache._reset_statsMs%&"""r(strcl|jdkr |jdkrdSd|j|j|j|jfzS)z2Return a formatted string of the cache statistics.rz3 , %s: %d hits, %d misses, %d total requested bytes)r"r#rr$r1s r& _log_statszBaseCache._log_statsSsH >Q  4?a#7#72D I N O  & H   r(cd|jjd|jd|jd|jd|jd|jd|jdS) Nz ) __class____name__rr!rr"r#r$r1s r&__repr__zBaseCache.__repr___s . !  !^  "\  "Y   "^   "_   %)$>    r(Nrrrrrrrrr)r*r+r*rr,rr)rr3) r9 __module__ __qualname____doc__r__annotations__r'r/r2r6r:r(r&rr(s  !D    ''''))))''''            r(rcNeZdZdZdZ ddfd ZddZddZddZd dZ xZ S)! MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapNrrrrrlocation str | Noneblocksset[int] | Nonerrct||||tn||_||_||_dSN)superr'setrHrF _makefilecache)r%rrrrFrHr8s r&r'zMMapCache.__init__wsP GT222%~ceee6   ^^%% r(mmap.mmap | bytearrayc*ddl}ddl}|jdkrtS|j$t j|js|j(|}t|_ nt|jd}| |jdz | d|nt|jd}|j||jS)Nrzwb+1zr+b)rEtempfiler bytearrayrFospathexists TemporaryFilerMrHopenseekwriteflushfileno)r%rErTfds r&rNzMMapCache._makefiles  9>>;;  = t}(E(E }$++--!ee $-// GGDIM " " " HHTNNN HHJJJJdmU++Btydi000r(r)r*endr,c td|d||d}|j}|jks||krdS|jz}|jz}fdt ||dzD}fdt ||dzD}xjt |z c_xjt |z c_|r|d}|jz}t|jzj} xj | |z z c_ td|d |d| d  || j || <j ||j ||S) NzMMap cache fetching -rr(c&g|] }|jv |SrBrH.0ir%s r& z$MMapCache._fetch..s%UUUa@T@T@T@T@Tr(rRc&g|] }|jv |SrBrdres r&rhz$MMapCache._fetch..s%QQQaT[@P@P@P@P@Pr(zMMap get block #z ())loggerdebugrrranger#lenr"popminr$rrOrHadd) r%r)r` start_block end_blockneedhitsrgsstartsends ` r&r/zMMapCache._fetchs 9E99C99::: =E ;)C DI  #3t~- 4>) UUUU5i!m<<UUUQQQQ5i!m<<QQQ 3t99$ #d))#  A'Fv. ::D  & &$- 7 & & LLAAAAAA$AAA B B B&*ll64&@&@DJvd{ # KOOA    z%)$$r(dict[str, Any]c>|j}|d=|S)NrO)__dict__copyr%states r& __getstate__zMMapCache.__getstate__s! ""$$ 'N r(r}cl|j|||_dSrK)rzupdaterNrOr|s r& __setstate__zMMapCache.__setstate__s- U###^^%% r()NN) rrrrrrrFrGrHrIrr)rrPr)r*r`r*rr,rrxr}rxrr) r9r>r?r@rr'rNr/r~r __classcell__r8s@r&rDrDls D $"& & & & & & & &1111,%%%%8 &&&&&&&&r(rDc0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadrrrrrrrcxt|||d|_d|_d|_dS)Nr(r)rLr'rOr)r`r%rrrr8s r&r'zReadAheadCache.__init__s8 GT222  r(r)r*r`r,c|d}| ||jkr|j}||jks||krdS||z }||jkr:||jkr/|xjdz c_|j||jz ||jz S|j|cxkr |jkrDnnA|xjdz c_|j||jz d}|t |z}|j}n|xjdz c_d}t|j||jz}|xj ||z z c_ | |||_||_|jt |jz|_||jd|zSNrr(rR) rr)r`r"rOr#rnrprr$r)r%r)r`lparts r&r/zReadAheadCache._fetchs =E ;# //)C DI  #3 %K DJ  3$(?? NNa NN:edj033CCD D Z5 + + + +48 + + + + + OOq OO:edj0223D TNAHEE OOq OOD$)S4>122 ""cEk1""\\%--  :DJ/dj!n$$r(r;rr9r>r?r@rr'r/rrs@r&rrsb D %%%%%%%%r(rc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstrrrrrrrcl||kr|}t|||d|_dSrK)rLr'rOrs r&r'zFirstChunkCache.__init__s: t  I GT222#' r(r)r*r`r,c|pd}||jkrtddSt||j}||jkr|j|xjdz c_||jkrD|xj|z c_|d|}|d|j|_||dS|d|j|_|xj|jz c_|j||}||jkr6|xj||jz z c_|||j|z }|xj dz c_ |S|xjdz c_|xj||z z c_|||S)Nrz,FirstChunkCache: requested start > file sizer(rR) rrkrlrprrOr#r$rr")r%r)r`datars r&r/zFirstChunkCache._fetchs  49   LLG H H H3#ty!! 4> ! !z!1$''..#5..<<3//D!%&6&6!7DJ<'!\\!T^<< **dn<**:eCi(DT^##**cDN.BB** T^S999 NNa NNK OOq OO  & &#+ 5 & &<<s++ +r(r;rrrs@r&rrsb D((((((,,,,,,,,r(rc^eZdZdZdZ ddfd Zd ZddZddZd dZ d!fd Z d"dZ xZ S)# BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcache rrrrr maxblocksrrct|||tj||z |_||_t j||j|_ dSrK) rLr'mathceilr!r functools lru_cache _fetch_block_fetch_block_cachedr%rrrrr8s r&r'zBlockCache.__init__7sd GT222y !122 "#A9#6y#A#A$BS#T#T   r(c4|jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor1s r&rzBlockCache.cache_info?'22444r(rxc|j}|d=|S)Nrrzr|s r&r~zBlockCache.__getstate__Js  ' ( r(r}c|j|tj|d|j|_dS)Nr)rzrrrrrr|s r&rzBlockCache.__setstate__OsH U####J9#6u[7I#J#J  $ $    r(r)r*r`r,c|d}||j}||jks||krdS||jz}||jz}t||dzD]}|||||||S)Nrr(rRstart_block_numberend_block_number)rrrmr _read_cache)r%r)r`rr block_numbers r&r/zBlockCache._fetchUs =E ;)C DI  #3#dn4$.0""46F6JKK 3 3L  $ $\ 2 2 2 2  1-     r(rc<||jkrtd|d|jd||jz}||jz}|xj||z z c_|xjdz c_t d|t||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (rjrRzBlockCache fetching block %d) r! ValueErrorrr$r#rkinforLr/)r%rr)r`block_contentsr8s r&rzBlockCache._fetch_blockls $, & &999)-999  t~-dn$ ""cEk1"" 1 2LAAAs33r(rrc ||jz}||jz}|xjdz c_||kr||}|||S|||dg}|t |jt |dz||||d|d|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rRNr(rr"rextendmaprmappendjoin r%r)r`rr start_posend_posblockouts r&rzBlockCache._read_cache~sDN* & ! !1 1 1334FGGE7*+ +++,>?? KLC JJ,,q02BCC    JJt//0@AA(7(K L L L88C== r(r rrrrrrrrrrrrr)rrrr, r)rr`rrrrrrr,) r9r>r?r@rr'rr~rr/rrrrs@r&rrs, DMOUUUUUUU 5 5 5         .$&!&!&!&!&!&!&!&!r(rcHeZdZUdZdZded< ddfd ZddZddZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r,rrTrrrrrtrimboolrrct|||d|_d|_d|_||_dS)Nr()rLr'rOr)r`r)r%rrrrr8s r&r'zBytesCache.__init__sA GT222 !% # r(r)r*r`cV|d}||j}||jks||krdS|jL||jkrA|j:||jkr/||jz }|xjdz c_|j|||z|z S|jrt |j||jz}n|}||ks ||jkrdS|j ||jkrY|j ||jkrG|xj||z z c_|xjdz c_| |||_||_nu|jJ|jJ|xjdz c_||jkr|j|j|z |jkr6|xj||z z c_| |||_||_n|xj|j|z z c_| ||j}||_||jz|_n|j||jkr|j|jkrn||jz |jkr6|xj||z z c_| |||_||_nB|xj||jz z c_| |j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|j rT|j|jz |jdzz}|dkr4|xj|j|zz c_|j|j|zd|_|Sr) rr)r`r"rOrrpr$r#rrnr)r%r)r`offsetbendnewrnums r&r/zBytesCache._fetchsa =E ;)C DI  #3 J "##$dhTZ'F NNa NN:fv|e';;< < > ty#"677DDD 5==EDI--3 J %$*"4"4 H dh  & &$, 6 & & OOq OOeT22DJDJJ:)))8''' OOq OOtz!!8#tx#~'F'F..$,>..!%eT!:!:DJ!&DJJ..$*u2DD..,,udj99C!&DJ!$tz!1DJJ%$//8di''48^dn44..$,>..!%eT!:!:DJ!&DJJ..$/A..,,tx66C!%c!1DJ:DJ/#j&3,"667 9 @8dj(dnq.@ACQww dns22 !Z(<(>(>?  r(c*t|jSrK)rnrOr1s r&__len__zBytesCache.__len__s4:r()T) rrrrrrrrrrr)rr) r9r>r?r@rrAr'r/rrrs@r&rrs  "D!!!!IMGGGGRr(rcFeZdZUdZdZded< ddfd ZddZxZS)AllBytesz!Cache entire contents of the fileallrrNrr*rFetcher | Nonerr bytes | Nonerrct||||@|xjdz c_|xj|jz c_|d|j}||_dS)NrRr)rLr'r#r$rrr)r%rrrrr8s r&r'zAllBytes.__init__sl GT222 < OOq OO  & &$) 3 & &<<49--D r(r)r+r,c@|xjdz c_|j||S)NrR)r"rr.s r&r/zAllBytes._fetchs# !yt$$r()NNNN) rr*rrrr*rrrrr< r9r>r?r@rrAr'r/rrs@r&rr s}++D!%"&!       %%%%%%%%r(rcFeZdZUdZdZded< ddfd Zdfd ZxZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrNTrrrrrr&Optional[dict[tuple[int, int], bytes]]strictr_rc `t|||||_|rt|}|dg}||dg} |ddD]\} } |d\} } | | kr-| | f|d<| dxx|| | fz cc<C|| | f| || | ftt|| |_ dSi|_ dS)NrrR) rLr'rsortedkeysrordictzipr)r%rrrrrr old_offsetsoffsetsrHr)r+start0stop0r8s r&r'zKnownPartsOfAFile.__init__=s4 GT222    --K"1~&Ghh{1~../F*122 ; ; t ' E>>#)4.GBK2JJJ$((E4="9"99JJJJNNE4=111MM$((E4="9"9::::S&1122DIIIDIIIr(r)r*r+rr,c|d}||j}d}|jD]w\\}}}||cxkr|krbn||z }||||z|z }|jr||cxkr|kr2nn/|d||z t |z zz }|xjdz c_|cS|}nx|jtd||fdtj d||fdt d|d ||xj ||z z c_ |xj dz c_ |t||zS) Nrr(rRz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching rb)rritemsrrnr"rrwarningswarnrkrlr$r#rLr/) r%r)r+rloc0loc1roffr8s r&r/zKnownPartsOfAFile._fetch[s =E <9D"&)//"3"3   LT4$u####t#####dl3te!334{ dd&:&:&:&:d&:&:&:&:&: 7dUlSXX&=>>CNNa'NNJJJ !EE!$* < WudmWWWXX X   3eT] 3 3 3     GGGGGHHH ""dUl2"" 1UWW^^E40000r()NT) rrrrrrrrrrrrr<rrs@r&rr$s,"D!!!!8< <+1+1+1+1+1+1+1+1+1+1r(rcXeZdZdZGddeZddd ZddZddZddZ ddZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c8eZdZUded<ded<ded<ded<dS)UpdatableLRU.CacheInforrumissesmaxsizecurrsizeN)r9r>r?rArBr(r& CacheInfors4    r(rfuncCallable[P, T]max_sizerrrctj|_||_||_d|_d|_tj|_ dSr ) collectionsr _cache_func _max_size_hits_misses threadingLock_lock)r%rrs r&r'zUpdatableLRU.__init__sA+6+B+D+D  !  ^%% r(argsP.argskwargsP.kwargsrc&|r$td||j5||jvrC|j||xjdz c_|j|cdddS dddn #1swxYwY|j|i|}|j5||j|<|xjdz c_t|j|j kr|j ddddn #1swxYwY|S)Nz Got unexpected keyword argument rRFlast) TypeErrorrr r move_to_endrrrrnrpopitem)r%r r results r&__call__zUpdatableLRU.__call__s  PNv{{}}NNOO O Z ) )t{"" ''--- a {4(  ) ) ) ) ) ) ) )" ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) T,V,, Z 0 0 &DK  LLA LL4;$.00 ###///  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0  s%ABB B 'ADD  D rrcV|j5||jvcdddS#1swxYwYdSrK)r r)r%r s r& is_key_cachedzUpdatableLRU.is_key_cachedsx Z ' '4;& ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's ""rc|j5||j|<t|j|jkr|jdddddS#1swxYwYdS)NFr)r rrnrr)r%rr s r&add_keyzUpdatableLRU.add_keys Z 0 0 &DK 4;$.00 ###/// 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0sAAAArc|j5||jt|j|j|jcdddS#1swxYwYdS)N)rrrur)r rrrnrrrr1s r&rzUpdatableLRU.cache_infos Z  >>T[))Z| "                  s:AAAN)r)rrrrrr)r r r r rr)r rrr)rrr rrrrr) r9r>r?r@r rr'rrrrrBr(r&rrs J &&&&&&''''0000 r(rcneZdZUdZdZded< d!d"fd Zd#dZd$dZd%dZ d&dZ d'd(fd Z d)d Z xZ S)*BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrrrrrrrrrrc>t|||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_dS)NrR max_workers)rLr'rrr!rrrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrs r&r'zBackgroundBlockCache.__init__s GT222y !122 "#/0A9#M#M 2q A A A6:'37"+."2"2r(rc4|jSrrr1s r&rzBackgroundBlockCache.cache_inforr(rxc2|j}|d=|d=|d=|d=|d=|S)Nrr"r#r$r%rr|s r&r~z!BackgroundBlockCache.__getstate__s:  ' ( $ % . / / " & ' r(c|j|t|j|d|_t d|_d|_d|_tj |_ dS)NrrRr ) rzrrrrrr"r#r$rrr%r|s r&rz!BackgroundBlockCache.__setstate__sk U####/0A5CU#V#V 2q A A A*.'!"+."2"2r(r)r*r`r,cV|d}||j}||jks||krdS||jz}||jz}d}d}|j5|j|jJ|jr`t d|j |j |jd|_d|_n@t||jcxko|knc}|r|j}|j}d|_d|_dddn #1swxYwY|Gt d|j | |t||dzD]}|||dz} |j5|jR| |j krG|j| s-| |_|j|j| d|_dddn #1swxYwY|||||S)Nrr(z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rRasyncr)rrr%r$r#donerkrrrrrrmr!rr"submitrr) r%r)r`rrfetch_future_block_number fetch_future must_joinrend_block_plus_1s r&r/zBackgroundBlockCache._fetch s, =E ;)C DI  #3#dn4$.0$(!  $ 2 2!-6BBB%**,,2KK UVVV,44*1133T5T7;D3)-D&&!%*:,,,,+,,,,!!I !2594S1'+'9 ;?7-1*7 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2<  # KKB C C C  $ , ,##%%'@    ""46F6JKK 3 3L  $ $\ 2 2 2 2,a/  $  "*$ 440>>?OPP53C/%)%:%A%A%'7&&"                 1-     s&C DDD!AHH H syncrlog_infor3c>||jkrtd|d|jd||jz}||jz}td|||xj||z z c_|xjdz c_t||}|S)rrrrjz!BlockCache fetching block (%s) %drR) r!rrrkrr$r#rLr/)r%rr2r)r`rr8s r&rz!BackgroundBlockCache._fetch_blockVs $, & &999)-999  t~-dn$ 7<PPP ""cEk1"" 1s33r(rrc ||jz}||jz}|xjdz c_||kr||}|||S|||dg}|t |jt |dz||||d|d|Srrrs r&rz BackgroundBlockCache._read_cachehsDN* & ! !1 1 1,,-?@@E7*+ +++,>?? KLC JJ,,q02BCC    JJt//0@AA(7(K L L L88C== r(rrrrr=r)r1)rrr2r3rr,r)r9r>r?r@rrAr'rr~rr/rrrrs@r&rrs2'D&&&&MO 3 3 3 3 3 3 3 5 5 5 53333J J J J X$(!(!(!(!(!(!(!(!r(rz!dict[str | None, type[BaseCache]]cachesFclstype[BaseCache]clobberrrrc~|j}|s)|tvr td|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rr5r)r6r8rs r&register_cacher:sL 8D Wtv~~UDUUvd|UUVVVF4LLLr()F)r6r7r8rrr). __future__rrrloggingrrVrrconcurrent.futuresrrtypingrrrr r r r r rrEtyping_extensionsrrr getLoggerrkrr,rrrDrrrrrrrrr5rAr:crBr(r&rBs""""""" 99999999                      KKK++++++ #AA A GCLL  8 $ $ C:u$ %A A A A A A A A HS&S&S&S&S& S&S&S&l+%+%+%+%+%Y+%+%+%\+,+,+,+,+,i+,+,+,\F!F!F!F!F!F!F!F!RbbbbbbbbJ%%%%%y%%%0b1b1b1b1b1 b1b1b1J9999971a4=999xK!K!K!K!K!9K!K!K!` )- (     AN1  r(