h3 jddlZddlZddlZddlmZddlmZmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZmZddlZdd lmZmZmZdd lmZdd l m!Z!d d l"m#Z#d dl$m%Z%m&Z&d dl'm(Z(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1d dl2m3Z3m4Z4m5Z5ej6dej7Z8eGddZ9Gddej:Z;Gddej<j=Z>Gddej<j=Z?de@de@fdZAde@de@fd ZBd!e@d"eeCdefd#ZDd$e;d!e@d%e@d&eEd'e@f d(ZFdS))N)deque) dataclassfield)datetime)chain)Path)AnyDictIteratorListNoReturnOptionalTupleUnion)quoteunquote)_DEFAULT_CALLBACK NoOpCallback TqdmCallback) isfilelike)Response) constants)CommitOperationCopyCommitOperationDelete)EntryNotFoundErrorRepositoryNotFoundErrorRevisionNotFoundError) hf_hub_urlhttp_get)HfApiLastCommitInfoRepoFile)HFValidationErrorhf_raise_for_status http_backoffzy (^refs\/convert\/\w+) # `refs/convert/parquet` revisions | (^refs\/pr\/\d+) # PR revisions cxeZdZUdZeed<eed<eed<eed<eddZeeed <d efd Z dS) HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path. repo_typerepo_idrevision path_in_repoNF)defaultrepr _raw_revisionreturnctj|jd|jz}|jr'|d|jd|jdS|jtj kr4|dt|jd|jdS|d|jdS)N@/) rREPO_TYPES_URL_PREFIXESgetr)r*r/r,rstripr+DEFAULT_REVISION safe_revision)self repo_paths f/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/huggingface_hub/hf_file_system.py unresolvez"HfFileSystemResolvedPath.unresolve0s599$."MMPTP\\   BJJ$"4JJt7HJJQQRUVV V ]i8 8 8TT- ">">TTARTT[[\_`` `55$"355<r<r(r($s__NNN LLLMMM$)5E#B#B#BM8C=BBBB3BBBBBBr>r(ceZdZdZdZdZddddeedee edfffdZ d ed ed eed e e ee ffd Z d1ded eed efdZd1deed dfdZ d2deded eedeed df dZd1ded eed dfdZ d3dede deed eed df dZ d4dede de d eed eeeeeefff dZ d5dede de d eede f d Zded ee eeeeefffd! Zded eeffd" Z d6dedeed#e de de d eed eeeeeeeefffffd$ Zd1d%ed&ed eed dfd'Zded efd(Z d7dede d eed eeeffd)Z!d*Z"d+Z#d,Z$ded efd-Z%e&dfd8fd. Z'e(d/Z)d0Z*xZ+S)9 HfFileSystema8 Access a remote Hugging Face Hub repository as if were a local file system. [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: token (`str` or `bool`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . Usage: ```python >>> from huggingface_hub import HfFileSystem >>> fs = HfFileSystem() >>> # List files >>> fs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> fs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` r2hfNendpointtokenrJrKctj|i||p tj|_||_t |||_i|_dS)NrI) super__init__rENDPOINTrJrKr!_api_repo_and_revision_exists_cache)r:rJrKargsstorage_options __class__s r<rNzHfFileSystem.__init__fs^ $2/222 6I$6  85999  ,,,r>r)r*r+r0c|||f|jvr |j|||tjd|j|||f<d|j||df<nj#t t f$r(}d|f|j|||f<d|f|j||df<Yd}~n6d}~wt$r&}d|f|j|||f<d|j||df<Yd}~nd}~wwxYw|j|||fS)N)r+r)timeout)TNF)rQrP repo_inforHF_HUB_ETAG_TIMEOUTrr$r)r:r)r*r+es r<_repo_and_revision_existz%HfFileSystem._repo_and_revision_existxsT w )1U U U ^ ##h)YMj$Xb4i(5STS]4i$5OPP,->? \ \ \W\^_W_4i(5STSXZ[S[4i$5OPPPPPP( ^ ^ ^W\^_W_4i(5STS]4i$5OPPPPPP ^ 3Y4RSSs#(AB8"B B8B33B8pathchdttdttdttfd}||}|std|dddzt jvr?d|vrtd|dd\}}t j|}n t j }| ddkr&d |vr|d d\}}d|vrt |}|[|d | fvrCtd |d}| }n|dd\}}nd }|t!||}||||\} } | st%|| nd }d|dd d } d|dd d } |dd} d|ddd }| }| }||||\} } | s]t)| t*t,fr1| }|}||||\} }| st%|| nyt%|| nh|}d }d |vr3|d d\}}|t!||}nd }||||\} }| std||n t j}t1||||| S)a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. revision_in_pathr+r0cL||||krtd|d|dn|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)r]r+s r<%_align_revision_in_path_with_revisionzHHfFileSystem.resolve_path.._align_revision_in_path_with_revisions^##/4D4P4P$-8H--jr--- ,Or>z0Access to repositories lists is not implemented.r4rrr3Nr2)r/)rrC_strip_protocolNotImplementedErrorsplitrr5valuesREPO_TYPES_MAPPINGREPO_TYPE_MODELcountSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprrZ_raise_file_not_foundjoin isinstancerr$r8r()r:r[r+r`r)r*r]matchr,repo_and_revision_existerrrepo_id_with_namespacepath_in_repo_with_namespacerepo_id_without_namespacepath_in_repo_without_namespace_s r< resolve_pathzHfFileSystem.resolve_paths( &sm 7?}  c]    ##D)) 2%&XYY Y ZZ__Q # %)J)Q)Q)S)S S S$)*\]]]"jja00OIt!4Y?II!1I ::c??Q  d{{,0JJsA,>,>))***7>>?OPPE(X$ 9N-N-N'B'F'FrK['\'\'c'cdg'h'h +0;;==((9I9O9OPSUV9W9W6(,,#%L@@IYAZAZ\dee/3/L/LYX_ai/j/j,'.5)$444#' ),$**S//"1"2E)F)F&.1hhtzz#qrr7J.K.K+,0JJsOOA,>)14$**S//!"":M1N1N.0: /3/L/LYX_ai/j/j,'.9!#(?AR'STT9";'E 595R5RS\^ego5p5p2/6=1$<<<-dC888GLd{{,0JJsA,>,>))@@IYAZAZ\dee#' )-)F)FyRY[c)d)d & #Q* ^)*\]]]'3889S' 7Hlbrssssr>c|s4|j|jdS||}|}|r2|j|d||}|2|jsW|j|j|j dfd|j|j|j |j fddSdS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) dircacheclearrQryr=pop_parentr,r)r*r+)r:r[ resolved_paths r<invalidate_cachezHfFileSystem.invalidate_caches  M   ! ! !  0 6 6 8 8 8 8 8 --d33M **,,D * !!$---||D)) * !- 488-:QS`Shjn9oquvvv488",m.C]E[\^b  r>rbmode block_sizeHfFileSystemFilec |d|vrtd|dkrt||f|||d|St||f|||d|S)Naz/Appending to remote files is not yet supported.r)rr+r)rcHfFileSystemStreamFiler)r:r[rr+rkwargss r<_openzHfFileSystem._opensk $;;%&WXX X ??)$t4(_ittmstt t#D$nTHYcnngmnn nr>c D|||}|j|j|j|j|j|j|d|d| | dS)Nr+commit_messagecommit_description)r,r*rKr)r+rrr[) ryrP delete_filer,r*rKr)r+r6rr=)r:r[r+rrs r<_rmzHfFileSystem._rms))$)BB  &3!)*#-"+!::&677%zz*>??     =#:#:#<#<=====r>F recursivemaxdepthc ||}||||}fd|D}d|D} d|d} | |rdndz } | |d |dndz } j|j|jj| |j|d | |d   | d S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrr+cng|]1}||j2SrE)isdirryr,).0r[r:s r< z#HfFileSystem.rm..Bs?hhh$W[WaWabfWgWgh**400=hhhr>c.g|]}t|S))r,)r)rr,s r<rz#HfFileSystem.rm..Cs$iii<+FFFiiir>zDelete  z recursively r2Nz up to depth rr)r*r)rK operationsr+rrr) ry expand_pathrP create_commitr*r)rKr+r6rr=) r:r[rrr+rrpaths paths_in_reporrs ` r<rmzHfFileSystem.rm"s1<))$)BB   XX` aahhhh%hhh ii[hiii *4***I=..2=8L44444RTT !)#-*!"+!::&6GG%zz*>??     =#:#:#<#<=====r>Tdetailrefreshc ||}| |jf||d|}n|#t$ro|jst d|j|f||d|}fd|D}t|dkrt dYnwxYw|r|n d|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `List[Union[str, Dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rr+Nc,g|]}|dk|SnamerEror[s r<rz#HfFileSystem.ls..ys'777QvY$%6%61%6%6%6r>rcg|] }|d SrrErrs r<rz#HfFileSystem.ls..|s":":":1V9":":":r>)ryr=_ls_treerr,rnr~len)r:r[rrr+rrouts ` r<lszHfFileSystem.lsSs:))$)BB &&(( 2$-SgSSFSSCC! 2 2 2 - 2%dD111$- T 2 2aGhaaZ`aaC7777c777C3xx1}}%dD111 2:ss":":c":":"::sAA6B98B9 expand_infoc ||}|}t|j|j|jd|j}g}|jvr|sj|} || g} |rtd| D} | r| } | djvr| | dnGj| d} || | d| D| g} |rfd|D} |r| s|r| r tj | | z}|ds||ks|t!| | vr|dn|fd |D}jD]5}|dzrj|d6jd||d || nj|j|j|||j|j }|D]}t3|t4r/|dz|j z|jd |j|j|j|jd}n|dz|j zdd|j |jd}|d}j!|g || ||S)Nrr2r,r/c*g|]}|ddk|Stype directoryrEr path_infos r<rz)HfFileSystem._ls_tree..s(fff9YvEVZeEeEeYEeEeEer>rc*g|]}|ddk|SrrErs r<rz)HfFileSystem._ls_tree..s)nnn9YW]M^bmMmMmYMmMmMmr>cVg|]%}|d |d&S) last_commitNr)r~)rrr:s r<rz)HfFileSystem._ls_tree..s3$f$f$fQ}M]MeT\\!F)%<%r4cPg|]"}|ddz |#S)rr4) startswith)rr common_paths r<rz)HfFileSystem._ls_tree..s6UUUQQvY-A-A+PSBS-T-TUqUUUr>T)rrr+r)rexpandr+r)filersizerblob_idlfsrsecurityrrrrrtree_idr)"ryr=r(r)r*r+r/r{extendrpopleftappendosr[ commonprefixendswithrr7r~rr}rrPlist_repo_treer,rpr#rrrrrr setdefault)r:r[rrr+rr root_pathrcached_path_infosdirs_not_in_dircache dirs_to_visitdir_infodirs_not_expanded common_prefix cached_pathtreercache_path_info parent_pathrs` @r<rzHfFileSystem._ls_tree~s))$)BB &&((,  #  !  "'5    )++  4=  $ d 3  JJ( ) ) )#%  !&ff0Afff!! $ ,4466H't}<<,33HV4DEEEE,0M(6:J,K) #4555%,,nn8Innn$ !#  g$f$f$f$fc$f$f$f! 2   HY !# 4 45IL]5] ^ ^ %--c225$ 11$.BDU(V(VVV"((---m44 VUUU#UUU#'===K"--kC.?@@= ))+t<<< !!+t444 MM#"+ $!)$/ "9++%*#"&/'1 ,D" , , i22 )C). @ ) &#,#4(}'0'<$-$6''OO!*C). @ ! +#,#4'0'< ''O#ll?6+BCC  ((b99@@QQQ ?++++ r>c/K|||d}tj|g|Ri|Ed{VdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[Tuple[str, List[str], List[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r+rN)ryr6r=rMwalk)r:r[rRrrTs r<rzHfFileSystem.walksw   :0F0F GGQQSS577<6t666v66666666666r>c |||d}tj|fi|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `List[str]`: List of paths matching the pattern. r+r)ryr6r=rMglob)r:r[rrTs r<rzHfFileSystem.globsS   :0F0F GGQQSSuww|D++F+++r>withdirsc  |r tj|f|||||d|S|||}|} |j|fd||jd| |s d D n)|j|fd|ji|} | ddkr| g zn d D n1#t$r$|j|fd|i|dd kr|ii ni YnwxYwt } |s| S fd | DS) a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[List[str], Dict[str, Dict[str, Any]]]`: List of paths or dict of file information. )rrrrr+rT)rrr+c*g|]}|ddk|SrrErs r<rz%HfFileSystem.find..6s&BBBQ6k)A)Aq)A)A)Ar>r+rrc i|] }|d| SrrErs r< z%HfFileSystem.find..;s---A1V9a---r>rc"i|] }|| SrErE)rrrs r<rz%HfFileSystem.find..@s666D#d)666r>) rMfindryr=rr+inforsorted) r:r[rrrrr+rrrnamesrrTs @r<rzHfFileSystem.finds@  577<'(6SZemqw ))$)BB &&(( .$-qgP]PfqqjpqqC UBB#BBB&DIdVV]5KVvVV +4V+< +K+KykC''QT-----CC"   ty;;;F;;FCvMMRj  s  7L6666666 6sB--+CCpath1path2c |||}|||}|j|jko|j|jk}|rd|d|}|j|j|j|j|d||ddt|j|j|jgn| |d |j5} | } d d d n #1swxYwYd|d|}|j | |j|j|j |j|j|d||d | | | | d S) a Copy a file within or between repositories. Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rzCopy z to rrr2)src_path_in_repor, src_revision)r*r)r+rrrrNpath_or_fileobjr,r*rKr)r+rrr)ryr)r*rPrr+r6rr,openread upload_filerKrr=) r:rrr+rresolved_path1resolved_path2 same_reporfcontents r<cp_filezHfFileSystem.cp_fileBs<&**58*DD**58*DD  $(@ @ u^E[_m_uEu   7U7777N I # #&.(2'0%zz*:NKK#)::.BB#G#G')7)D%3%@%3%< $    5$1HII #Q&&(( # # # # # # # # # # # # # # #7U7777N I ! ! '+8&.j(2'0%zz*:NKK#)::.B#C#C "    >#;#;#=#=>>> >#;#;#=#=>>>>>s.DDDc F|j|fii|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rTrdater)r:r[rrs r<modifiedzHfFileSystem.modified}s=tyAA!@F!@M4!@!@AAM"6**r>c f ||}||dd}|jsedddd |rZ|j|j|j|jd }i dt|j |j |j d  n}d | }|s||jvr||||jvr3fd |j|D} | st!d| d |s |r r d |j|j|j||j|j} | st!d| d} t%|j|j|jd|j} t)| t*r/| dz| jz| jd| j| j| j| jd n| dz| jzdd| j| jd |s fddD J S)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `Dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rrFrrN)rrrr)r)r+)oidtitler)rrc,g|]}|dk|SrrErs r<rz%HfFileSystem.info..s'SSSa6dARARARARARr>r)rr+r)r2rr4rrrc"i|] }|| SrErE)rkrs r<rz%HfFileSystem.info..sGGG1c!fGGGr>)rrr)ryr=r6r,rPlist_repo_commitsr*r)r+r" commit_idr created_atr~r{rrnget_paths_infor(r/rpr#r[rrrrrr)r:r[rr+rrrrrout1 paths_inforrrs ` @r<rzHfFileSystem.infos0))$)BB &&((jj 5  )D H## C  "i99!)]5LWdWm: ##1'19JQ\Qg$$$C,,t,,K %;dm#C#C $$$dm++SSSS4=#=SSS6)$5551g% H#+++#+#mBTB\!Y55!)!.&*3+5 6 "6)$555&qM 4!+!)!*!#"/"=  )++ i22 )C). @ ) &#,#4(}'0'<$-$6CC!*C). @ ! +#,#4'0'< C#HGGGG.FGGGC r>c |ddr|||j|fi|dS#YdSxYw)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)r6rr)r:r[rs r<existszHfFileSystem.existss`& zz)U++ ,%%d+++ DId % %f % % %4 55s 9=Acd ||ddkS#t$rYdSwxYw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rrF)rOSErrorr:r[s r<rzHfFileSystem.isdirsC 99T??6*k9 9   55 s ! //cR ||ddkS#YdSxYw)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rrFrr s r<isfilezHfFileSystem.isfile s1 99T??6*f4 4 55s!&c||}t|j|j|j|j|j}||r|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r)r+rJz /resolve/z/tree/r) ryrr*r,r)r+rJrreplace)r:r[rurls r<rzHfFileSystem.url2sx))$//   !  &#-"+]     ::d   8++k8Q77C r>c |d}t|dhz }t|tt frt |dkrtj||f||d|St|r|}n-| |rtj |ddSt|ttfr3tj tj|dd}|t#|d}d}|} ||| } ||| d } ||  t-t/| j| j| j| j|j ||| d|jt|t r|jnd | | |r|!dSdS#|r|!wwxYw) a^ Copy single remote file to local. Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r+r)callbackoutfileT)exist_okNFwbrrr*r+filenamer)rJ)r temp_filedisplayed_filename expected_size resume_sizeheaders _tqdm_bar)"r6setkeysrprrrrMget_filerrrmakedirsrCrr[dirnamertellryrset_sizer rr*r+r,r)rJrP_build_hf_headerstqdmseekclose) r:rpathlpathrrrr+unhandled_kwargs close_file initial_posresolve_remote_pathrrTs r<r!zHfFileSystem.get_fileIs[*::j))v{{}}-- <(\<$@AA `SIYEZEZ]^E^E^$577#E5_8W__X^__ _ e   GG ZZ    K - - - -4 ec4[ ) ) ? K.. > > > > ?5$''GJllnn #///II %( ;;FC -((( /7090=1;!] "#(+ 3355+5h +M+MW(--SW     LL % % %    z   s B H..Ic td)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.rcr:s r< transactionzHfFileSystem.transactions""LMMMr>c td)z@Begin write transaction for deferring files, non-context versionr1r2r3s r<start_transactionzHfFileSystem.start_transactions""LMMMr>N)rNN)FNN)TFN)FFNF)NFFFN)FNr0N),r?r@rArB root_markerprotocolrrCrboolrNr ExceptionrZr(ryrintrrrr r r rrr rrrrrrrr rrrrr!propertyr4r6 __classcell__rTs@r<rGrG:s&&PKH #'(, 3-T3_% $TT'*T6>smT tXi(( )TTTT&]t]t]t ]tIa]t]t]t]t~Xc]d>"&$( o o o o3- o SM o  o o o o > > >x} >$ > > > >  "&"& />/>/>/>3- /> 3- /> />/>/>/>d`d);););!%);7;);OWX[}); eCc3h'( )););););\ "&! ffff f 3- f  ffffP77(5d3iQUVYQZAZ;[2\777777 ,,49,,,,,,&#'"&:7:7:73-:7 :7  :7  :73-:7 tCy$sDcN233 4:7:7:7:7:7:7x9?9?S9?9? 9?[_9?9?9?9?v+S+x++++ ccctcx}cbfgjlogobpccccJ8$$./@E E E E E E E NNNXNNNNNNNNr>rGceZdZddededeeffd ZfdZdeded e fd Z dd Z dd e d dfdZ dfd Zd efdZxZS)rNfsr[r+c ( ||||_n>#t$r1}d|ddvrt|d|d}~wwxYwt j||jfi||dS)Nrwrr2B. Make sure the repository and revision exist before writing data.)ryrFileNotFoundErrorr6rMrNr=)r:rBr[r+rrYrTs r<rNzHfFileSystemFile.__init__s !#!I!ID      fjj,,,,']]]    T/99;;FFvFFF s A,AAcht|dsdStSNrhasattrrM__del__r:rTs r<rKzHfFileSystemFile.__del__.t_--  Fww   r>startendr0cLdd|d|dz i|jj}t|jj|jj|jj|jj|jj }td||dtj }t||jS) Nrangezbytes=-rrGETiiii)rretry_on_status_codesrV)rBrPr&rrr*r+r,r)rJr&rHF_HUB_DOWNLOAD_TIMEOUTr%r)r:rNrOrrrs r< _fetch_rangezHfFileSystemFile._fetch_ranges /e//cAg// gl,,.. &.'0'4(2W%       "65     Ayr>c<tjdd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerr3s r<_initiate_uploadz!HfFileSystemFile._initiate_uploads!4GERRRr>Ffinalc |jd|j}|j||r|j|jj|jj |j j |j j |jj |j j|j j|jd|jdt%j|jj |j|j dSdS)Nrrrrr)bufferr(rrwriter)rBrPrrrr,r*rKr)r+rr6rremoverr=)r:r_blocks r< _upload_chunkzHfFileSystemFile._upload_chunks1    "" U###   N " " " GL $ $ $ 3!/<*2gm,6+4#{/?@@#';??3G#H#H %    Idn) * * * G $ $'1133 %       r>rc\|jdkr||dkrx|jdkrm|j|jdd5}|}|xjt |z c_|cdddS#1swxYwYt|S)a)Read remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rNrr)r)rlocrBrr[rrrM)r:lengthrrrTs r<rzHfFileSystemFile.reads 9  &.FbLLdhRSmmdi!<< ffhhCHH$                ww||F###s3BB Bc@|j|jSr7rBrr[r3s r<rzHfFileSystemFile.urlw{{49%%%r>r7r8)Fr)r?r@rArGrCrrNrKr=bytesrXr^r;rerrr?r@s@r<rrs  < s hsm      !!!!! #CE,SSSS4D* $ $ $ $ $ $&S&&&&&&&&r>rc eZdZ ddedededeed ed ef fd Zdd ed efdZddefdZ defdZ fdZ dZ xZ S)rrNrnonerBr[rr+r cache_typec |dkrtd||dkrtd|d|vrtd|d ||||_nB#t$r5}d|d d vrt|d |Yd}~nd}~wwxYw|jdd |_tj||jf|||d |d|_ |dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got roz?HfFileSystemStreamFile only supports cache_type='none' but got rDz;HfFileSystemStreamFile only supports reading but got mode=''rrr2rE)rrrrrp) r_ryrrFr6r=detailsrMrNresponse) r:rBr[rr+rrprrYrTs r<rNzHfFileSystemStreamFile.__init__su ??fZdffgg g   k_ikkll l $;;b[_bbbcc c !#!I!ID      fjj,,,,']]]-,,,,  !% 2 < < > >MM  ",,.. 59j]g  kq   -1  s A'' B&1+B!!B&rgwhencecb|dkr|dkrdS||jkr|dkrdStd)NrrzCannot seek streaming HF file)rgr_)r:rgrvs r<r(zHfFileSystemStreamFile.seeks? !88! F $(??v{{ F8999r>rrhc |dkr|fnd}|jt|jj|jj|jj|jj|jj}td||jj ddtj |_t|j |jjj|}n#t"$r|jt|jj|jj|jj|jj|jj}td|dd |jzi|jj ddtj |_t|j |jjj|}n(#t"$r|jwxYwYnwxYw|xjt)|z c_|S) NrrErrSrTT)rrUstreamrVRangez bytes=%d-)rurrr*r+r,r)rBrJr&rPr&rrVr%rawrr<r)rgr)r:rh read_argsrrs r<rzHfFileSystemStreamFile.reads !'1VII" = *2+4+8,6) C) 6688&:!9 DM  . . . ($-#()4CC    M   ! ! !*2+4+8,6) C) +"8]DGLcht|dsdStSrHrIrLs r<rKzHfFileSystemStreamFile.__del__PrMr>cZt|j|j|j|j|jjffSr7)reopenrBr[r blocksizecacherr3s r< __reduce__z!HfFileSystemStreamFile.__reduce__Vs$DIt~tzWWWr>)rNrro)rrl)r?r@rArGrCrr=rNr(rrrKrr?r@s@r<rrs  "&    3-   >:::S:::://3////b&S&&&&!!!!! XXXXXXXr>rr+r0cXt|r|nt|Sr7)rirq safe_quoters r<r9r9Zs(288BB \88 S[H\H\\r>sc$t|dS)Nr2)safe)r)rs r<rr^s    r>r[rsc|}t|tr|d}n5t|tr|d}nt|tr|d}t ||)Nz (repository not found)z (revision not found)z (invalid repository id))rprrr$rF)r[rsmsgs r<rnrnbs~ C#.//0... C. / /0,,, C* + +0/// C c)r>rBrrrpc4|||||S)Nrs)r)rBr[rrrps r<rrms 774dzj7 Q QQr>)Grrer\ collectionsr dataclassesrrr itertoolsrpathlibrtypingr r r r r rrr urllib.parserrfsspecfsspec.callbacksrrr fsspec.utilsrrequestsrr2r _commit_apirrerrorsrrr file_downloadrr hf_apir!r"r#utilsr$r%r&compileVERBOSErir(AbstractFileSystemrGspecAbstractBufferedFilerrrCr9rr<rnr=rrEr>r<rs ((((((((NNNNNNNNNNNNNNNNNNNN'''''''' JJJJJJJJJJ######CCCCCCCCVVVVVVVVVV////////3333333333GGGGGGGGGG)bj J  BBBBBBB B*e Ne Ne Ne Ne N6,e Ne Ne NPP&P&P&P&P&v{7P&P&P&fbXbXbXbXbXV[=bXbXbXJ]C]C]]]]##**(9*=*(****R|R3RcRsRPSRRRRRRr>