gh@ddlZddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ddl ZddlmZ ddlZn#e$r e sddlZYnwxYwddlmZddlmZddl mZmZmZdd lmZdd lmZmZm Z ej!d Z"Gd d e#Z$dZ%dZ&dZ'Gddej(j)Z*Gddej(j+Z,dZ-Gddej(j.Z/GddeZ0GddeZ1dS)N) lru_cache)chain) TYPE_CHECKINGLiteral)AbstractBufferedFile)AsyncFileSystem)DEFAULT_CALLBACK) filesystemopensplit_protocol)AsyncFileSystemWrapper) isfilelikemerge_offset_ranges other_pathszfsspec.referencec$eZdZfdZdZxZS)ReferenceNotReachablecPtj|||_||_dSN)super__init__ referencetarget)selfrrargs __class__s h/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/fsspec/implementations/reference.pyrzReferenceNotReachable.__init__s)$" c&d|jd|jS)Nz Reference "z" failed to fetch target )rrrs r__str__zReferenceNotReachable.__str__$sST^SSdkSSSr)__name__ __module__ __qualname__rr __classcell__rs@rrrsN TTTTTTTrrc^tt|Sr)nextitervalues)ds r_firstr+(s QXXZZ  ! !!rc||}t|ttfrFt|dtr-|drt |ddn|dSdSdSNr)get isinstancelisttuplestrr )path referencesrefs r_prot_in_referencesr6,s{ ..  C#e}%%?*SVS*A*A?,/F>~c!f%%a((A>????rct|trt|||giSi}|D];}t||}||g|<|Sr)r/r2r6 setdefaultappend)pathsr4outr3protocols r_protocol_groupsr=2sy%A#E:66@@ C22&tZ88 x$$++D1111 JrceZdZdZdS)RefsValuesViewc#K|jjD]*}tj|V+|jjEd{V|jD]b}|j|}t|dkr|j|dzVB|j |Ed{VcdS)Nr/0) _mapping zmetadatar)jsondumpsencode_itemslistdir_get_chunk_sizeslen_generate_all_records)rvalfield chunk_sizess r__iter__zRefsValuesView.__iter__=s =*1133 + +C*S//((** * * * *='..000000000]**,, B BE-88??K;1$$mEDL1111}::5AA A A A A A A A A  B BrNr!r"r#rOrrr?r?<s( B B B B Brr?ceZdZdZdS) RefsItemsViewc~t|j|jSr)ziprBkeysr)rs rrOzRefsItemsView.__iter__Js.4=%%'')=)=)?)?@@@rNrPrQrrrSrSIs(AAAAArrScxd}d}t|ddd|dddD]\}}|||zz }||z}|S)Nr)rU)idxsizesrLmultiss rravel_multi_indexr_NsZ C DC"IuTTrT{++1 q4x   JrcVeZdZdZedZedZ d&ded fd Zd Z d Z e d'dZ e dZd(dZdZe ddZdZdZdZdZdZdZdZdZe dZdZd)d Zd)d!Zd"Zd#Zd$Z d%Z!dS)*LazyReferenceMapperaoThis interface can be used to read/write references from Parquet stores. It is not intended for other types of references. It can be used with Kerchunk's MultiZarrToZarr method to combine references into a parquet store. Examples of this use-case can be found here: https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storagecddl}|Sr-)numpy)rnps rrdzLazyReferenceMapper.np`s rcddl}|Sr-)pandas)rpds rrgzLazyReferenceMapper.pdfs rN fastparquetengine)rjpyarrowcJ||_i|_||_||_||_|jdz|_|t jdn||_|j |p|j|_ ddl m }|jdkr|dtddSdS)a This instance will be writable, storing changes in memory until full partitions are accumulated or .flush() is called. To create an empty lazy store, use .create() Parameters ---------- root : str Root of parquet store fs : fsspec.AbstractFileSystem fsspec filesystem object, default is local filesystem. cache_size : int, default=128 Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. categorical_threshold : int Encode urls as pandas.Categorical to reduce memory footprint if the ratio of the number of unique urls to total number of refs for each variable is greater than or equal to this number. (default 10) engine: Literal["fastparquet","pyarrow"] Engine choice for reading parquet files. (default is "fastparquet") z/{field}/refs.{record}.parqNfiler) find_specrlz)engine choice `pyarrow` is not installed.)rootrN cat_threshrk cache_sizeurlfsspecr fsunstrip_protocolout_rootimportlib.utilro ImportError)rrprurwrrcategorical_thresholdrkros rrzLazyReferenceMapper.__init__lsB / $9<</1z&#F+++r001FTYGG ,,,,,, ;) # # )(<(<(DIJJ J $ #(D(Drcj|dvr!||j|St|)N)rG record_sizerC)setup__dict__AttributeErrorritems r __getattr__zLazyReferenceMapper.__getattr__s6 9 9 9 JJLLL=& &T"""rcNi_jdjdgjd<t jjd}|d_|d_tj fd}|_ dS)N/ .zmetadatar|metadata)maxsizec*j||}tjj|} j|jfdj D}n#t$rd}YnwxYw|S)zcached parquet file loader)rMrecord)rkcFi|]}||SrQ)to_numpy).0cdfs r z@LazyReferenceMapper.setup..open_refs..s)@@@2a5>>++@@@rN) rsformatioBytesIOrucat_filerg read_parquetrkcolumnsOSError)rMrr3datarefsrrs @r open_refsz,LazyReferenceMapper.setup..open_refss8??v?>>D:dg..t4455D W))$t{)CC@@@@RZ@@@    Ks 4B BB) rGrurjoinrprDloadsr|rCrrrr)rmetrs` rr}zLazyReferenceMapper.setups $(G$4$4 HHdi. / /% %  L!j\233}-Z 4? + + +     , + #r'c i|d}|tjj|fi|pi\}}||r||d||d|d|dgtj | t||fi|S)aJMake empty parquet reference set First deletes the contents of the given directory, if it exists. Parameters ---------- root: str Directory to contain the output; will be created storage_options: dict | None For making the filesystem to use for writing is fs is None fs: FileSystem | None Filesystem for writing record_size: int Number of references per parquet file kwargs: passed to __init__ Returns ------- LazyReferenceMapper instance rr|NT recursiveexist_okrr) rtcore url_to_fsexistsrmmakedirspiperrDrErFra)rpstorage_optionsrur|kwargsrs rcreatezLazyReferenceMapper.creates,k:: :{,TMMo6KMMHB 99T?? ( EE$$E ' ' ' D4 ((( $ -.. 30F0F0H0HIII"466v666rcBd|jD}t|S)zList top-level directoriesc3xK|]5}|d|dddV6dS).zrrXrN) startswithrsplitrps r z.LazyReferenceMapper.listdir..sGVV1<z*LazyReferenceMapper.ls..sP   ||H%% +.QXXh5G5G*G*G *G*G*Grclh|]0}|rd|v.|1Srr)rfrs r z)LazyReferenceMapper.ls..sU <<))/2(9K9K.K.K  /L.K.Krrc3K|]}|VdSrrQrnames rrz)LazyReferenceMapper.ls..s"55dT555555rc3K|]}|VdSrrQrs rrz)LazyReferenceMapper.ls..s"22dT222222rFcg|]}|ddd S) directoryrrtypesizerQrs rrz*LazyReferenceMapper.ls..s"WWW$k1EEWWWrc g|]H}|dt|jvrtjj|n j|dISrnrrJrCrDrErGrrrs rrz*LazyReferenceMapper.ls..su   !"4>11 4>$#7888![.   rc|dS)NrrQ)r^s rz(LazyReferenceMapper.ls..s AfIrkeycDg|]}|d|Srrrrr3s rrz*LazyReferenceMapper.ls..s0 L L Ld4 0K0K LT L L LrcDg|]}|d|Srrrs rrz*LazyReferenceMapper.ls..s0KKKt$zzz/J/JKtKKKrc g|]H}|dt|jvrtjj|n j|dISrrrs rrz*LazyReferenceMapper.ls.. su    t~--Jt~d3444T*      rc<g|]\}}|d |d|ddS)rrnrYrrQ)rrrecs rrz*LazyReferenceMapper.ls..sD   c1v 63r7 ; ;   r) rstriprHrrCrGupdatesortedr_keys_in_fieldr0rKrU) rr3detaildirnamesrothersdirinfofileinforMrVrecsrecinfors `` @rlszLazyReferenceMapper.lss;{{3!%-4#::2<<>>         G!N55dn55522dk222F d###f~~%WWRVWWWG    #   H'H,2E2EFFF F L L L Ldn L L LKKKK KKK L           ""5)) U??<<$t**, ,))%00   t__    '!!rc( ||jvr |j|S||jvr1tj|j|Sd|vs||rt ||dd\}}||\} }|j ||fi d}|t |r|S|dkrdS | || n/#tttf$r}t ||d}~wwxYwgd} fd|D} | d } | | S| dt d | dd ddgkr | ddS| dd S) zfGet the reference for one key Returns bytes, one-element list or three-element list. rrXFNrrr3offsetrrawc8g|]}|vr|ndSrrQ)rrrris rrz5LazyReferenceMapper._load_one_key..>s-IIIAAIIT!WR[[4IIIrrYz1This reference does not exist or has been deleted)rGrCrDrErF_is_metaKeyErrorr_key_to_recordr.r ValueError TypeErrorFileNotFoundError) rrrM_r chunk_sizemaybeexcr selectionrrrs @@r _load_one_keyz!LazyReferenceMapper._load_one_key"s $+  ;s# # DN " ":dnS12299;; ; ^^t}}S11^3-- ::c1%%q!%!4!4S!9!9J 4488UCC =N  L 1__3 )>>%00DDI'89 ) ) )3--S ( )333IIIIIIII m ?J Q< NOO O QqS>aV # #RaR= !}s4D D7"D22D7ic2|dd\}}||}t|dkrdSd|dD}t ||}||jz}||jz}||t|fS)z3Details needed to construct a reference for one keyrrXr)rrrc,g|]}t|SrQ)intrrs rrz6LazyReferenceMapper._key_to_record..Qs666SVV666r.)rrIrJsplitr_r|) rrrMchunkrN chunk_idx chunk_numberrrs rrz"LazyReferenceMapper._key_to_recordJszz#q)) u++E22 {  q 766U[[%5%5666 (K@@ !11 D, ,r3{++++rc||jvrC|j|d}dt|d|dD}|pdg|j|<|j|S)z6The number of chunks along each axis for a given fieldz/.zarraycBg|]\}}tj||z SrQ)mathceil)rr^rs rrz8LazyReferenceMapper._get_chunk_sizes..[s7%)Q !a%  rshapechunksrX)rNrCrU)rrMzarray size_ratios rrIz$LazyReferenceMapper._get_chunk_sizesWsz ( ( (^u$6$6$67F-0&BR-S-SJ'1&7QCD U #&&rc |||}tt|}t |dkr d|DSt |dkr|dSd|DS)z8The references for a given parquet file of a given fieldrc34K|]}t|VdSrr0rts rrz7LazyReferenceMapper._generate_record..gs(((DGG((((((rrXrc3dK|]+}|dst|ddn|dV,dS)rNr r s rrz7LazyReferenceMapper._generate_record..msABB!qt5D2A2KKK1BBBBBBr)rr(rUr)rJ)rrMrrits r_generate_recordz$LazyReferenceMapper._generate_recordas~~eV,, #t{{}}% & & t99>>((R((( ( YY!^^; CBrBBB Brc#Kd}||D]}||z}tj||jz }t |D]}|||Ed{VdS)zJLoad all the references within a field by iterating over the parquet filesrXN)rIrrr|ranger)rrMnrecchrs rrKz)LazyReferenceMapper._generate_all_recordsos''..  B BJDDy 0011Dkk < >D, , , >>%88      @V$Ex(GV$EE?DDGGD,bfCG@@Ehht/wh??GHHT-WH==E774+RV37??D (( 6 6GAt$%% 6 $$ 22Qu{'===!00a99E7at99q==!%aGAJ#AwE!Ha aQ#+//55Q \\!       7==??bgoo//41 5 G GF**:66BvJ")6::UO )DM F FE F F F F F (3dm==e==MMM ;) # #!3U ;   [M ) )#2&!!   &&D&D&DEE E  ;+9tw 1488         (((((sA AAct|jD]5}t|tr|\}}|||||6t|jD]@}|dkr8d|vr4t j|j||j|<A|j|j d}|j t j | |jd<|j d|p|jdg|jd|jdS)zOutput any modified or deleted keys Parameters ---------- base_url: str Location of the output )rKrrrrrN)r0rGr/r1r rDrrJrCr|rIrErFrurrrwr cache_clear)rrKrthingrMrkrs rflushzLazyReferenceMapper.flushsT$+&&  E%''  % v %$3 dk"" C CAL  TQYY$(Jt{q/A/A$B$Bq!>$:JKK $(JsOO$:$:$<$< L!  HHh/$-> ? ? K %   ""$$$$$rc2d}|D]G}|dr|dz }|tj||z }H|t |jz }|td|jDz }|S)NrrrXc3DK|]}t|tdVdS)rXN)r/r1)rrs rrz.LazyReferenceMapper.__len__../s1HH1:a3G3GHQHHHHHHr) rHrrprodrIrJrCsumrG)rrBrMs r__len__zLazyReferenceMapper.__len__%s\\^^ A AE$$ A 4#8#8#?#?@@@ T^$$$ HH HHHHHH rc#Kt|j}||j|D]}t |t r|V|D]"}||D] }||vr|V #dSr)rrCrrGr/r2rHr)rmetasbitrMrWs rrOzLazyReferenceMapper.__iter__2sDN## T[!!!  C#s##  \\^^  E((//  99GGG   rcT ||dS#t$rYdSwxYw)NTF)rrrs r __contains__z LazyReferenceMapper.__contains__?sB    t $ $ $4   55 s  ''c#K||}t|dkr |dzVdStjd|D}|D])}|dzdd|DzV*dS)z|List key names in given field Produces strings like "field/x.y" appropriate from the chunking of the array rrANc34K|]}t|VdSr)r)rr]s rrz5LazyReferenceMapper._keys_in_field..Os("A"A588"A"A"A"A"A"Arrrc,g|]}t|SrQ)r2rs rrz6LazyReferenceMapper._keys_in_field..Qs)>)>)>Q#a&&)>)>)>r)rIrJ itertoolsproductr)rrMrNindsinds rrz"LazyReferenceMapper._keys_in_fieldFs ++E22 {  q $,    F "A"A["A"A"AB @ @C#+)>)>#)>)>)> ? ?? ? ? ? ? @ @r)NNrhrirj)NNr)rTNN)"r!r"r#__doc__propertyrdrgrrrr} staticmethodrrrHrrrrIrrKr)rrrr'rr*r rXr]rOrbrrQrrraraWsYYYX X  4A.K.K01.K.K.K.K`######0777\7<Y[[[ @"@"@"@"D&&&PYt__ , ,_ ,''' C C C<<<$$$###'''OOO(44\4((("S)S)S)S)j!%!%!%!%F       @ @ @ @ @rraceZdZdZdZdZ d(fd Zd)d Zd)d Zd)d Z d Z dZ e fdZ d*dZd+dZd,dZdZd,dZdZdZdZd-dZd.dZdZdZdZd.dZd/fd Zd!Zd"Zd#Zd0d%Z d0d&Z!d'Z"xZ#S)1ReferenceFileSystemaView byte ranges of some other file as a file system Initial version: single file system target, which must support async, and must allow start and end args in _cat_file. Later versions may allow multiple arbitrary URLs for the targets. This FileSystem is read-only. It is designed to be used with async targets (for now). We do not get original file details from the target FS. Configuration is by passing a dict of references at init, or a URL to a JSON file containing the same; this dict can also contain concrete data for some set of paths. Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md rFNT@Brhc ptjd i|||_| |_| |_i|_i|_i|_| |_| |_ t|trtd i|p|pid|i}tjj|fi|\}}||rqtj|dfi|5}t$d|t)j|}dddn #1swxYwY||| nIt$d|t/||| |_n||| t|trEd|D|_d|jvrt5d|jd<dS|8t|jt8r |jd n|j}||j|<|{|jD]a}t=|r |}tj|\}}|r"||jvrt5|fi|pi}||j|<b||jD]}t=|r |}t|t@rT|d rLtj|d \}}||jvrt5|fi|pi}||j|<n|r"||jvrt5|fi|pi}||j|<|pt5d|jd<|jD]I\}}|j!stE||j# |j|<+|j#|j#z rtId JdS) a Parameters ---------- fo : dict or str The set of references to use for this instance, with a structure as above. If str referencing a JSON file, will use fsspec.open, in conjunction with target_options and target_protocol to open and parse JSON at this location. If a directory, then assume references are a set of parquet files to be loaded lazily. target : str For any references having target_url as None, this is the default file target to use ref_storage_args : dict If references is a str, use these kwargs for loading the JSON file. Deprecated: use target_options instead. target_protocol : str Used for loading the reference file, if it is a path. If None, protocol will be derived from the given path target_options : dict Extra FS options for loading the reference file ``fo``, if given as a path remote_protocol : str The protocol of the filesystem on which the references will be evaluated (unless fs is provided). If not given, will be derived from the first URL that has a protocol in the templates or in the references, in that order. remote_options : dict kwargs to go with remote_protocol fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) Directly provide a file system(s): - a single filesystem instance - a dict of protocol:filesystem, where each value is either a filesystem instance, or a dict of kwargs that can be used to create in instance for the given protocol If this is given, remote_options and remote_protocol are ignored. template_overrides : dict Swap out any templates in the references file with these - useful for testing. simple_templates: bool Whether templates can be processed with simple replace (True) or if jinja is needed (False, much slower). All reference sets produced by ``kerchunk`` are simple in this sense, but the spec allows for complex. max_gap, max_block: int For merging multiple concurrent requests to the same remote file. Neighboring byte ranges will only be merged when their inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 to only merge when it requires no extra bytes. Pass a negative number to disable merging, appropriate for local target files. Neighboring byte ranges will only be merged when the size of the aggregated range is <= ``max_block``. Default is 256MB. cache_size : int Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. Only used for lazily loaded references. kwargs : passed to parent class r<rbzRead reference from URL %sNz$Open lazy reference dict from URL %s)rurrc i|]I\}}|t|tr,tj|dddfi|n|JS):rXr)r/dictrtr r)rrWoptss rrz0ReferenceFileSystem.__init__..sl At !$--F%aggc1ooa&8AADAAA rrnr) asynchronouszEReference-FS's target filesystem must have same value of asynchronousrQ)%rrrtemplate_overridessimple_templates templatesfss _dircachemax_gap max_blockr/r2rvrtrrisfiler loggerinforDload_process_referencesrar4rr r<r1r)callabler r0 async_implr rxr)rforref_storage_argstarget_protocoltarget_optionsremote_protocolremote_optionsruryrzr~rrrrdicref_fsfo2rtextr5r<rrWrs rrzReferenceFileSystem.__init__fsR ""6""" "4 0 " b#   =#;~;GVC!+/::c::KFC}}S!! [T11S11(QKK #-R[%"@"@Q Abk )+DH_ %  "~,,.. , ,C== #%%C$k88== !, 8 8#HGG1E2GGB)+DHX&  "--//  C== #%%Cc4((SV"(+"<">OFF0D"FFB(*DH_ %1z&11HNN$$  DAq< 4QTEVWWW "Q^3  &   s>0C::C>C>c ||}td| |j|}n"#t$r}t ||d}~wwxYwt |tr|}t|dr| }t |trTtd|d| drtj|dd}|ddfSt|dkr-td|d||d }||}}n]|\}} } td|d |d | d | | | z} ||d kr| |z}n| |z}n| }||d kr| |z}n| |z}n| }||j}|||fS) Nzcat: rz Reference: z , type bytesbase64:rXz, whole file => rz => z , offset z, size )_strip_protocolrdebugr4rrr/r2rFr!rr#rbase64 b64decoderJr) rr3startendpartrrsstart1end1start0rend0s r _cat_commonzReferenceFileSystem._cat_commons##D)) ^T^^$$$ 3?4(DD 3 3 3#D))s 2 3 dC  !;;==D 4 $ $ #==??D dE " " $ LL9t999 : : :z** 2'QRR11t# # t99>> LLCtCCTCC D D Dq'C #DFF $ C LLTtTTTTvTTdTT U U UD=D A::#e^FF!E\FF!88!Cg|]}||SrQ)r)rrrs rrz+ReferenceFileSystem.get..hs(CCC1TZZ]]CQCCCr)r expand_pathrtr rcatrUr) rrrrrrutargetsrremotelocals ` rr.zReferenceFileSystem.get`s   GGBKKK  ) <<  v$ 7 7 7eU++  #88CCCCCCCDDDD88E??D 00 2 2MFE~~ UDL111 2 2rraisec t|tr |rtt|tr"|st d|Drtt ||j}i}|D]\}}|j|} ggggf\} } } } |D]} | |\}}}t|ttfs9 | || || || |#t$r}|dkr|dkr|||<Yd}~d}~wwxYwg}g}g}g}t}t| | | | D]\}}}}t|tr|||<"|i||| || || || |t| | | | D]a\}}}}|X||vrT| || || || |bt!t|t|t|d|j|j\}}}| |||}t| | | | D]\}}}}||vr t||||D]|\}}}}||kr-||)t|t(r|||<,|||||<:||kr<||kr6||kr0t|t(r|||<g|||z ||z pd||<}|D]`\} }!t|!t(rF| |jvr=|| }"t-| |j| }#|"|#_|dkr|#|dkr|#|| <at1|dkr(t|trd|vrt3|S|S)Nc3K|]}d|vV dS)*NrQrs rrz*ReferenceFileSystem.cat..rs&7O7OQq7O7O7O7O7O7OrromitT)sortr~rrXr)r/r2rFr0anyr=r4rr|rr#r9rrrUaddrr~r cat_rangesrr1r __cause__rJr+)$rr3ron_errorr proto_dictr;protor:ruurlsstartsends valid_pathsrur^rerrurls2starts2ends2paths2 whole_files new_paths new_startsnew_ends bytes_outrdnsnebrWvexnew_exs$ rrzReferenceFileSystem.catos dC  &Y &% % dD ! ! &y &C7O7O$7O7O7O4O4O &% %%dDO<< &,,..I BI BLE5%B.0"b"n +D&$  * **"..q11GAq!%a%66! !KKNNNMM!$$$KKNNN&&q)))))%%%7**6))!$A %EGEF%%K!$kBB % % 1aa'' %CFFY OOA&&&LLOOONN1%%%LLOOOMM!$$$!$kBB % % 1a=Qk%9%9LLOOONN1%%%LLOOOMM!$$$/BU W U  . /// +Iz8 iXFFI"$kBB B B 1a88%(J)%T%T B BMBBQwwBJ"*%a33,%&CFF%&qsVCFFqQ"WWb%a33B%&CFF%&q2vR0@D'@%ACF B BHHJJ$$&& $ $DAq!Y'' $A,@,@V.q$/!2DEE#% w&& L''#CF s88q==Zc22=s$#;;  s$5D00 E:EEc|dd}|||dS|dkr|||dStd|)NversionrX)ryz Unknown reference spec version: )r._process_references0_process_references1r)rr4ryverss rrz'ReferenceFileSystem._process_referencesst~~i.. <  % %j 1 1 1 1 1 QYY  % %jEW % X X X X XFFFGG Grczt|trd|D}||_dS)z&Make reference dict for Spec Version 0cli|]1\}}|t|trtj|n|2SrQ)r/rvrDrE)rrrLs rrz.sIC 3(=(=FTZ___3rN)r/rvrr4)rr4s rrz(ReferenceFileSystem._process_references0sK j$ ' '  * 0 0 2 2J%rcjrjrddli_|dit dfd}|diD],\}}t|trD| dr$tj |ddj|<|j|<_t|trtj|j|<jr|d}d|vrMjr;|dd d d jdij}n ||}t%|d kr|gn||d |d gj|<"|j|<.j|dgdS)Nrr{icN|jdijSNrQ)Templaterenderr{)rjinja2rs r _render_jinjaz?ReferenceFileSystem._process_references1.._render_jinjas*,6??1%%,>>t~>> >rrzbase64:r{{{z}}}rXr0genrQ)rzr{rr4_process_templatesr.rrr/r2rrrrvrDrEreplacerrJr _process_gen)rr4ryrrWrrrs` @rrz(ReferenceFileSystem._process_references1s$   MMM  {B ? ?@@@ 4 ? ? ? ? ?  ?NN62..4466 ' 'DAq!S!! '<< **A)/)9!ABB%)@)@DOA&%&""At$$ '%)Z]]"" 'aD199,-$AIIdC00$WT3//#66&*n66 *M!,,,/FFaKKaSSa1qt_""%&"" t00r1J1JKKLLLLLrci|_|j||j|D]'\}}d|vrddl|ffd |j|<||j|<(dS)NrrcD|jdi|Sr)rr)temprrs rrz8ReferenceFileSystem._process_templates..s7>V__>>>#>#!>#>#r)r{ryrrr)rtmprWrrs @rrz&ReferenceFileSystem._process_templatess  " . JJt. / / /IIKK & &DAqqyy 01%#%#%#%#%#q!!%&q!! & &rc  i}|D][}d|dD fdtj D}|D]}ddl}||djd i||j}||djd i||j}d|vrd|vr}t||djd i||j} t||djd i||j} || | g||<d|vd|vz rtd |g||<]|S) Nc i|]\\}}|t|tr|n>t|dd|d|dd]S)rrstopsteprX)r/r0rr.)rrWrs rrz4ReferenceFileSystem._process_gen..st Aq !!T**OAAquuWa00!F)QUU61=M=MNN r dimensionsc3vK|]3}tt|V4dSr)rvrUrV)rr) dimensions rrz3ReferenceFileSystem._process_gen..!sSS))62233rrrrsrlengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.rQ) rrfrgr)rrrr{rr) rgensr;rproductsprrrrsrrrs @rrz ReferenceFileSystem._process_gens % %C  -3355 I'/1A1A1C1CDH % % 8fooc%j118PP2PPP8fooc%j118PP2PPPOO(c// =H 66=UUUdnUUF!=H 66=UUUdnUUF!$VV4CHH#o(c/:%$K !$uCHH' %( rcdgi|_|j}|D]_\}}t|tt fst |drt|}nt|dkrd}n|\}}}d|vr|dddnd}|}|g}|r>||jvr5| |}| ||r ||jv5| t||ddD]I\} } | |jvsJ| |jvsJ|j|  | dddg|j| <J|j| |d|dadS) NrrrXrrrrrn) dircacher4rr/r#r2r!rJr_parentr9reverserU) rrr3rrrparpar0subdirsparentchilds r_dircache_from_itemsz(ReferenceFileSystem._dircache_from_items;sR _ " " $ $ T TJD$$ -- "z1J1J "4yyTa! 1d,/4KK$++c1%%a((RCDfG %4t}44||D))t$$$ %4t}44 OO   !$Wgabbk!:!: * * DM1111.... f%,,"KCC(* e$$ M#  % %tVT&R&R S S S S3 T Trrsc ||\}}}t|trtj|||St |\} } |||j| j||f||d|St|||f||d|S)N) block_size cache_options) rr/r#rrr r|_open ReferenceFile) rr3moderrrrrrr<rs rrzReferenceFileSystem._openYs$($4$4T$:$:! VT k5 ) ) 8:k&+677 7$[11 ! >dl+48H%+&+        "'       rc td|||}t|jt r? |j||S#t$rYnwxYwtd|d|j s| | |}|t||r|Sd|DS)Nzlist %s'z' is not a known keycg|] }|d SrrQ)ros rrz*ReferenceFileSystem.ls..s'''a& '''r) rrrr/r4rarrrrr_ls_from_cache)rr3rrr;s rrzReferenceFileSystem.lsss Y%%%##D)) do': ; ; D ))$777    #$B$B$B$BCC C} (  % % ' ' '!!$'' ;#D)) )  J''3''''s A'' A43A4c V||p||Sr)rrrr3rs rrzReferenceFileSystem.existss%zz$44;;t#4#44rc|jr |jvSt|jtr|jvSt fd|jDS)Nc3HK|]}|dVdS)rNr)rrr3s rrz,ReferenceFileSystem.isdir..s5IIAq||tJJJ//IIIIIIr)rr/r4rarHrrr3s `rrzReferenceFileSystem.isdirsq = J4=( ( )< = = J4?22444 4IIIIIIIII Irc||jvSrrr!s rrzReferenceFileSystem.isfilest&&rc$K|j||fi|Sr)r)rr3rrs r_lszReferenceFileSystem._lss"twtV..v...rc :|rtjf|||d|Sr6tfdjD}ntj}|r)jsfd|DS|S)N)maxdepthwithdirsrc3FK|]}||VdSrr)rrWr3s rrz+ReferenceFileSystem.find..s4HHQQ\\$5G5GHqHHHHHHrcHi|]}||dS)r)r)rrWrs rrz,ReferenceFileSystem.find..s.<<.s'888!ai4&7&7A&7&7&7rrrr) r4r.r/r2r#rJrr r|r)rr3rr;out0protrs ` rrzReferenceFileSystem.infos'o!!$'' ?#U|,, F $fc#hhGGGSA $fc!fEEE!%vtDDE''$%%C8888s888D F $k1EEE 76? "$T_T%:1%=>>GD!"htn11$/$2G2JKKDGFOAwrc0K||Sr)rrs r_infozReferenceFileSystem._infosyyrcrK|j|d|jdSr)r4rJrrIrs r_rm_filezReferenceFileSystem._rm_filesA  $    r overwritecK|dkr||rt||j|<|jdS)Nr)rFileExistsErrorr4rrI)rr3rrrs r _pipe_filezReferenceFileSystem._pipe_filesN 8   D 1 1 ! ! $ rcK|dkr||rtt|d5}||j|<dddn #1swxYwY|jdS)Nrrs)rr5r readr4rrI)rrrrrrs r _put_filezReferenceFileSystem._put_files 8   E 2 2 ! ! %   .!%&VVXXDOE " . . . . . . . . . . . . . . . sAA"%A"c i}|jD]u\}}t|trV |d||<4#t $r/dt j|z||<YlwxYw|||<vtj |dfi|5}| tj d|d ddddS#1swxYwYdS)z+Write modified references into new locationasciirrrX)rrN)r4rr/r#r"UnicodeDecodeErrorr b64encodertr r rDrErF)rrsrr;rWrrs r save_jsonzReferenceFileSystem.save_jsonsjO))++  DAq!U## IXXg..CFF)III(6+;A+>+>>FFHHCFFFIA [d 6 6o 6 6 F! GGDJ1c::;;BBDD E E E F F F F F F F F F F F F F F F F F Fs#A6B B %=C//C36C3) NNNNNNNNTrprqrhrj)F)Frr)rsNN)T)NFF)r3)$r!r"r#rkr<cachablerrrrrrr rr.rrrrrrrrrrrrr$r*rr0r2r6r9r>r$r%s@rroroTs  HH  ______B*!*!*!*!X B B B BBBBB&&&/? , , , , 2 2 2 2````DHHHH % % %MMMMB & & &###JTTT<    4(((($555JJJ'''////"*  F F F F F F FrrocReZdZ d fd ZedZfdZd ZxZS) rrsdefaultT readaheadNc tj||f||||||d| |j|j\} |_|_t| \} } |jj| |_ | |_ d|_ dS)N)rr autocommitr cache_typer) rrrurr3rrr r|src_fssrc_path_f)rrur3rrrDrErrrrr<rrs rrzReferenceFile.__init__s    !!!'  -1G,?,? ,J,J) TZ$[11 !gk(+ # rc|j |jjr6|jj|jf|j|j|jdd|j|_|jS)Nnone)rrrDrE) rHclosedrFrrGr blocksizerDrrs rrzReferenceFile.fs_ 7?dgn?'dk' Y>?!  + DGwrc|j|jtSr)rHcloser)rrs rrNzReferenceFile.closes, 7  GMMOOOww}}rc||jz}t||jz|j}|j||j||z Sr)rminrrseekr8)rrrs r _fetch_rangezReferenceFile._fetch_rangesR "# "DH--  Ev{{3;'''r)rsrATrBNN) r!r"r#rrlrrNrRr$r%s@rrrs  :  X  (((((((rr)2r collectionsrrfloggingrr functoolsrrtypingrr fsspec.corert fsspec.specrujsonrDry fsspec.asynrfsspec.callbacksr r r r #fsspec.implementations.asyn_wrapperr fsspec.utilsrrr getLoggerr RuntimeErrorrr+r6r=abc ValuesViewr? ItemsViewrSr_MutableMappingrarorrQrrrds   )))))))),,,,,,  ('''''------8888888888FFFFFFEEEEEEEEEE  - . .TTTTTLTTT"""???  B B B B B[_/ B B BAAAAAKO-AAA z@z@z@z@z@+/8z@z@z@zN FN FN FN FN F/N FN FN Fb4(4(4(4(4((4(4(4(4(4(sA AA