ghX!UddlmZddlZddlZddlZddlmZddlmZ ddl Z n#e $r esddl Z YnwxYwer)ddl m Z ddlmZmZddlmZdd lmZeeefZd ed <Gd d ZdS)) annotationsN) TYPE_CHECKING) atomic_write)Iterator)AnyLiteral) TypeAlias)CachingFileSystemr DetailcneZdZdZd%dZd&d Zd'd Z d(d)dZd*dZd+dZ d,dZ d-dZ d.d Z d,d!Z d/d#Zd$S)0 CacheMetadataa\Cache metadata. All reading and writing of cache metadata is performed by this class, accessing the cached files and blocks is not. Metadata is stored in a single file per storage directory in JSON format. For backward compatibility, also reads metadata stored in pickle format which is converted to JSON when next saved. storage list[str]cT|std||_ig|_d|_dS)z Parameters ---------- storage: list[str] Directories containing cached files, must be at least one. Metadata is stored in the last of these directories by convention. z3CacheMetadata expects at least one storage locationFN) ValueError_storage cached_files_force_save_pickle)selfrs m/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/fsspec/implementations/cache_metadata.py__init__zCacheMetadata.__init__&s= TRSS S +-$#(fnstrreturnr c t|d5}tj|}dddn #1swxYwYnL#t$r?t|d5}t j|}dddn #1swxYwYYnwxYw|D]B}t |dtrt|d|d<C|S)z6Low-level function to load metadata from specific filerNrbblocks) openjsonloadrpicklevalues isinstancegetlistset)rrfloadedcs r_loadzCacheMetadata._load9sO (b# &!1 & & & & & & & & & & & & & & & ( ( (b$ (1Q ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( / /A!%%//400 /!!H+..(  sK?3 ?7?7?BA:. B:A> >BA> BBmetadata_to_saveNonec|jr?t|5}tj||ddddS#1swxYwYdSt|d5}t j||ddddS#1swxYwYdS)z4Low-level function to save metadata to specific fileNw)mode)rrr$dumpr")rr.rr*s r_savezCacheMetadata._saveFs  " /b!! 1Q ,a000 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1bs+++ /q *A... / / / / / / / / / / / / / / / / / /s:>>A;;A?A?F writable_onlyboolIterator[tuple[str, str, bool]]c#Kt|j}t|jD]8\}}||dz k}|r|stj|d||fV9dS)aYield locations (filenames) where metadata is stored, and whether writable or not. Parameters ---------- writable: bool Set to True to only yield writable locations. Returns ------- Yields (str, str, bool) r cacheN)lenr enumerateospathjoin)rr5nirwritables r_scan_locationszCacheMetadata._scan_locationsOs   #DM22 D DJAwAEzH X ',,w00'8C C C C C  D Drr=cfsCachingFileSystem | None#Literal[False] | tuple[Detail, str]ct||jD]\\}}}}||vr||}|Y|jr%|d|j|krV|jr&tj|dz |jkrtj ||d}tj |r||fcSdS)zIf path is in cache return its details, otherwise return ``False``. If the optional CachingFileSystem is specified then it is used to perform extra checks to reject possible matches, such as if they are too old. NuidtimerF) ziprBrcopy check_filesfsukeyexpiryrHr<r=r>exists)rr=rCrbase_r9details r check_filezCacheMetadata.check_filees%((<(<(>(>@Q$R$R " " MRq55  4[%%''F?ve} D8I8I'I'I:$)++v">"K"KdF4L11Bw~~b!! "rz!!! "ur expiry_timeinttuple[list[str], bool]cg}|jdD]\}}tj|dz |kr|dd}|st d|t j|j d|}| ||jd ||jdrLt j|j dd}| |jd||jd }||fS)zRemove expired metadata from the cache. Returns names of files corresponding to expired metadata and a boolean flag indicating whether the writable cache is empty. Caller is responsible for deleting the expired files. rHrz)Cache metadata does not contain 'fn' for r9) rrJitemsrHr' RuntimeErrorr<r=r>rappendpopr4)rrT expired_filesr=rRr cache_pathwritable_cache_emptys r clear_expiredzCacheMetadata.clear_expiredsB  -b16688>>@@ 0 0LD&y{{VF^+k99ZZb))&JDJJW\\$-"3R88$$R(((!"%))$///  R  :dmB&7AAJ JJt(,j 9 9 9#'#4R#88222rcg}|D]c\}}}tj|r)|||N|id|pig|_dS)z>Load all metadata from disk and store in ``self.cached_files``N)rBr<r=rOr\r-r)rrrrQs rr#zCacheMetadata.loads ,,.. ( (HB1w~~b!! (##DJJrNN3333##B''''(0RDrr*rc|jd|}|ddur-t|d|jz|jkr d|d<dSdSdS)zPerform side-effect actions on closing a cached file. The actual closing of the file is the responsibility of the caller. rXr TN)rr: blocksizesize)rr*r=r,s ron_close_cached_filez"CacheMetadata.on_close_cached_files_  b !$ ' X;d " "s1X;'7'7!+'E'O'OAhKKK # "'O'Or str | Nonec ||d}|sdS|\}}||jdr5|jd||nt d|S)zRemove metadata of cached file. If path is in the cache, return the filename of the cached file, otherwise return ``None``. Caller is responsible for deleting the cached file. NrXzi|]\}}||S)rJ).0kvs r z&CacheMetadata.save..s&BBBTQQBBBrrXN)rIrBrr<r=rOr-rZupdatemaxr%r&r)r(r4) rrrQrAr9rrrr,r s rrjzCacheMetadata.saves(+D,@,@,B,BDDU(V(V 1 1 $ RHu w~~b!! %#zz"~~ (..00 3 3DAqEzzX;$..%(82D2L2L*.AhKK &+1Xh%7F"MM!H+666*0AhK$'& 58F3C$D$D& #(8E?%"KKMM,,DAq ,,*+ Q, % BB\-?-?-A-ABBBE\\^^ 4 4ak3//4"&q{"3"3AhK JJub ! ! !$0D b ! !A 1 1rrRc&||jd|<dS)z8Update metadata for specific file in memory, do not saverXN)r)rr=rRs r update_filezCacheMetadata.update_files&,"d###rN)rr)rrrr )r.r rrrr/)F)r5r6rr7)r=rrCrDrrE)rTrUrrV)rr/)r*rr=rrr/)r=rrrg)r=rrRr rr/)__name__ __module__ __qualname____doc__rr-r4rBrSrar#rfrmrjrxrprrrrs ((((&    ////%*DDDDD,633334 1 1 1 1("1"1"1"1H------rr) __future__rr<r$rHtypingr fsspec.utilsrujsonr" ImportErrorcollections.abcrrrtyping_extensionsr cachedr dictrr __annotations__rrprrrsH""""""" %%%%%%  '((((((########++++++))))))S#XF&&&&N-N-N-N-N-N-N-N-N-N-s & 44