gh! ddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZdZdaGdd e Zed gd Gd d ejZGddeZdS)N)suppress)cached_propertywraps)parse_qs)AbstractFileSystem)"get_package_version_without_importinfer_storage_options mirror_fromtokenizec<tfd}|S)Nc |i|S#t$rK}|js|j^}}t|trd|vrt t j||d}~wwxYw)Nzdoes not exist)OSErrorargs isinstancestrFileNotFoundErrorerrnoENOENT)rkwargs exceptionmessagefuncs d/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/fsspec/implementations/arrow.pywrapperz wrap_exceptions..wrappers 4((( (   > &^NGd'3'' ,<,G,G' g>>IM s A AAA )r)rrs` rwrap_exceptionsrs3 4[[    [  NcfeZdZdZdZfdZedZedZ e dZ ddZ d Z d Zd Zed Zed ZedZeddZeddZeddZeddZedZedZdfd ZfdZxZS)ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem. Parameters ---------- fs : pyarrow.fs.FileSystem /c ftda||_tjdi|dS)Npyarrow)rPYARROW_VERSIONfssuper__init__)selfr$r __class__s rr&zArrowFSWrapper.__init__3s:#D))6{ ??4  8D rFc |}ddlm}fdj||D}|r|Sd|DS)Nr) FileSelectorc:g|]}|Sr") _make_entry).0entryr's r z%ArrowFSWrapper.ls..Ns7      U # #   rcg|] }|d S)namer")r>r?s rr@z%ArrowFSWrapper.ls..Us777eE&M777r)r9 pyarrow.fsr;r$ get_file_info)r'r4detailrr;entriess` rlszArrowFSWrapper.lsJs##D))++++++    ..||D/A/ABB     8N77w777 7rc ||}|j|g\}||Sr*)r9r$rDr=)r'r4rinfos rrIzArrowFSWrapper.infoWsB##D))&&v..%%%rc~||} ||dS#t$rYdSwxYw)NTF)r9rIrr'r4s rexistszArrowFSWrapper.exists\sT##D))  IIdOOO4!   55 s . <<cddlm}|j|jurd}n]|j|jurd}nL|j|jur >C!B> C! E!D3?D& D3&D* *D3-D* .D33EE Ec ||d}||d}|j||dS)Nr)r9r_r$rf)r'rirjrs rmvzArrowFSWrapper.mvs`$$U++22377$$U++22377  UE"""""rcd||}|j|dSr*)r9r$rhrKs rrm_filezArrowFSWrapper.rm_files1##D)) D!!!!!rNc||d}||r-|r|j|dSt d|j|dS)Nrz0Can't delete directories without recursive=False)r9r_isdirr$ delete_dir ValueErrorrh)r'r4 recursivemaxdepths rrmzArrowFSWrapper.rms##D))0055 ::d   & U""4((((( !STTT G   % % % % %rr\Tc j|dkr|r |jj}nE|jj}n8|dkr |jj}n%|dkr |jj}nt d|i}|dks|s6t tdddkrd|d<||fi|}t|||||fi|S) Nr\r^abz'unsupported mode for Arrow filesystem: .r compression) r$open_input_fileopen_input_streamopen_output_streamopen_append_streamrvintr#split ArrowFile) r'r4mode block_sizeseekablermethod_kwargsstreams rr`zArrowFSWrapper._opens 4<< 302 T\\W/FF T\\W/FFOtOOPP P 4<>_>[ 8 8 8 8&&&    &  _ ##_# ""_"&&&_&III_I,666_6111_1!!_!11_1FFFFFF111111111rrr) readseektellwritereadablewritablecloserRrc"eZdZddZdZdZdS)rNc h||_||_||_||_|x|_|_||_dSr*)r4rr$r blocksizerr)r'r$rr4rrrs rr&zArrowFile.__init__s7   +55 rc|Sr*r"r,s r __enter__zArrowFile.__enter__s rc*|Sr*)r)r'rs r__exit__zArrowFile.__exit__szz||rr*)rrrr&rrr"rrrrsFrrcJeZdZdZdZ d fd ZedZxZS) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem to connect it's interface with fsspechdfsdefaultrNc lddlm}|||||||} tjdd| i|dS)a; Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication replication: int set replication factor of file for write operations. default value is 3. extra_conf: None or dict Passed on to HadoopFileSystem r)r)r0r1user kerb_ticket replication extra_confr$Nr")rCrr%r&) r'r0r1rrrrrrr$r(s rr&zHadoopFileSystem.__init__sj6 0/////  ##!     ))B)&)))))rct|}i}|ddr |d|d<|ddr |d|d<|ddr |d|d<|ddrIt|d}|ddrt|dd|d<|S)Nr0usernamerr1 url_queryrr)r getrr)r4r8outqueriess r_get_kwargs_from_urlsz&HadoopFileSystem._get_kwargs_from_urls"s#D)) 7764  &f+CK 77:t $ $ *j/CK 7764  &f+CK 77; % % Ds;/00G{{=$// D%()?)B%C%CM" r)rrNNrN) rrrrr-r& staticmethodrrrs@rrrs~--H  %*%*%*%*%*%*N  \     rr)riorXrard contextlibr functoolsrr urllib.parser fsspec.specr fsspec.utilsrr r r rr#rIOBaserrr"rrrs  ,,,,,,,,!!!!!!******$j1j1j1j1j1'j1j1j1Z          $;;;;;~;;;;;r