OOId0 ddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z mZmZmZdZdaGdde Zed gd Gd d ejZGd deZdS)N)suppress)cached_propertywraps)AbstractFileSystem)"get_package_version_without_importinfer_storage_options mirror_fromtokenizec<tfd}|S)Nc |i|S#t$rK}|js|j^}}t|trd|vrt t j||d}~wwxYw)Nzdoes not exist)OSErrorargs isinstancestrFileNotFoundErrorerrnoENOENT)rkwargs exceptionmessagefuncs .wrappers 4((( (   > &^NGd'3'' ,<,G,G' g>>IM s A AAA )r)rrs` rwrap_exceptionsrs3 4[[    [  NcjeZdZdZdZfdZedZedZ e dZ ddZ d Z d Zd Zed Zed ZeZedZeddZeddZeddZeddZedZedZdfd ZfdZxZS)ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem. Parameters ---------- fs : pyarrow.fs.FileSystem /c ftda||_tjdi|dS)Npyarrow)rPYARROW_VERSIONfssuper__init__)selfr#r __class__s rr%zArrowFSWrapper.__init__2s:#D))6{ ??4  8D rFc |}ddlm}fdj||D}|r|Sd|DS)Nr) FileSelectorc:g|]}|Sr!) _make_entry).0entryr&s r z%ArrowFSWrapper.ls..Ms7      U # #   rcg|] }|d S)namer!)r=r>s rr?z%ArrowFSWrapper.ls..Ts777eE&M777r)r8 pyarrow.fsr:r# get_file_info)r&r3detailrr:entriess` rlszArrowFSWrapper.lsIs##D))++++++    ..||D/A/ABB     8N77w777 7rc ||}|j|g\}||Sr))r8r#rCr<)r&r3rinfos rrHzArrowFSWrapper.infoVsB##D))&&v..%%%rc~||} ||dS#t$rYdSwxYw)NTF)r8rHrr&r3s rexistszArrowFSWrapper.exists[sT##D))  IIdOOO4!   55 s . <<cddlm}|j|jurd}n]|j|jurd}nL|j|jur >C!B> C! E!D3?D& D3&D* *D3-D* .D33EE Ec ||d}||d}|j||dS)Nr)r8r^r#re)r&rhrirs rmvzArrowFSWrapper.mvs`$$U++22377$$U++22377  UE"""""rcd||}|j|dSr))r8r#rgrJs rrm_filezArrowFSWrapper.rm_files1##D)) D!!!!!rNc||d}||r-|r|j|dSt d|j|dS)Nrz0Can't delete directories without recursive=False)r8r^isdirr# delete_dir ValueErrorrg)r&r3 recursivemaxdepths rrmzArrowFSWrapper.rms##D))0055 ::d   & U""4((((( !STTT G   % % % % %rr[Tc j|dkr|r |jj}nE|jj}n8|dkr |jj}n%|dkr |jj}nt d|i}|dks|s6t tdddkrd|d<||fi|}t|||||fi|S) Nr[r]abz'unsupported mode for Arrow filesystem: .r compression) r#open_input_fileopen_input_streamopen_output_streamopen_append_streamruintr"split ArrowFile) r&r3mode block_sizeseekablermethod_kwargsstreams rr_zArrowFSWrapper._opens 4<< 302 T\\W/FF T\\W/FFOtOOPP P 4<>_>[ 8 8 8 8&&&    &  _ ##_# G""_"&&&_&III_I,666_6111_1!!_!11_1FFFFFF111111111rrr) readseektellwritereadablewritablecloserQrc"eZdZddZdZdZdS)rNc h||_||_||_||_|x|_|_||_dSr))r3rr#r blocksizerr)r&r#rr3rrrs rr%zArrowFile.__init__s7   +55 rc|Sr)r!r+s r __enter__zArrowFile.__enter__s rc*|Sr))r)r&rs r__exit__zArrowFile.__exit__szz||rr))rrrr%rrr!rrrrsFrrcHeZdZdZdZ dfd ZedZxZS) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem to connect it's interface with fsspechdfsdefaultrNc jddlm}||||||}tjdd|i|dS)a Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication extra_conf: None or dict Passed on to HadoopFileSystem r)r)r/r0user kerb_ticket extra_confr#Nr!)rBrr$r%) r&r/r0rrrrrr#r's rr%zHadoopFileSystem.__init__sg0 0/////  #!     ))B)&)))))rct|}i}|ddr |d|d<|ddr |d|d<|ddr |d|d<|S)Nr/usernamerr0)rget)r3r7outs r_get_kwargs_from_urlsz&HadoopFileSystem._get_kwargs_from_urlss#D)) 7764  &f+CK 77:t $ $ *j/CK 7764  &f+CK r)rrNNN) rrrrr,r% staticmethodrrrs@rrrs{--H   !*!*!*!*!*!*F  \     rr)riorWr`rc contextlibr functoolsrr fsspec.specr fsspec.utilsrrr r rr"rIOBaserrr!rrrs  ,,,,,,,,******$l1l1l1l1l1'l1l1l1^          $33333~33333r