OOIdddlZddlZddlZddlZddlZddlZddlmZddlZ ddl Z n#e $rddl Z YnwxYwddl mZddlmZddlmZmZmZddlmZmZmZejdZGd d eZd Zd Zd ZGddej j!Z"Gddej j#Z$Gddej j%Z&GddeZ'dS)N) lru_cache)AsyncFileSystem)_DEFAULT_CALLBACK) filesystemopensplit_protocol) isfilelikemerge_offset_ranges other_pathszfsspec.referencec$eZdZfdZdZxZS)ReferenceNotReachablecPtj|||_||_dSN)super__init__ referencetarget)selfrrargs __class__s @lib/python3.11/site-packages/fsspec/implementations/reference.pyrzReferenceNotReachable.__init__s)$" c&d|jd|jS)Nz Reference "z" failed to fetch target )rrrs r__str__zReferenceNotReachable.__str__sST^SSdkSSSr)__name__ __module__ __qualname__rr __classcell__rs@rrrsN TTTTTTTrrcPt|dSNr)listvalues)ds r_firstr'"s   A rc||}t|ttfr+|drt |ddn|dSdSr#)get isinstancer$tupler )path referencesrefs r_prot_in_referencesr/&s\ ..  C#e}%%?,/F>~c!f%%a((A>??rct|trt|||giSi}|D];}t||}||g|<|Sr)r*strr/ setdefaultappend)pathsr-outr,protocols r_protocol_groupsr7,sy%A#E:66@@ C22&tZ88 x$$++D1111 JrceZdZdZdS)RefsValuesViewc#K|jjD]*}tj|V+|jjEd{V|jD]Z}|j|}|j dkr|j|dzV:|j |Ed{V[dS)Nr/0) _mapping zmetadatar%jsondumpsencode_itemslistdir_get_chunk_sizessize_generate_all_records)rvalfield chunk_sizess r__iter__zRefsValuesView.__iter__7s=*1133 + +C*S//((** * * * *='..000000000]**,, B BE-88??K1$$mEDL1111}::5AA A A A A A A A A  B BrNrrrrIrrr9r96s( B B B B Brr9ceZdZdZdS) RefsItemsViewc~t|j|jSr)zipr<keysr%rs rrIzRefsItemsView.__iter__Ds.4=%%'')=)=)?)?@@@rNrJrKrrrMrMCs(AAAAArrMceZdZdZedZedZ ddZddZd Z d Z d Z d Z d Z dZdZdZdZdZdZdZdS)LazyReferenceMapperzVInterface to read parquet store as if it were a standard kerchunk references dict.cddl}|Sr#)numpy)rnps rrUzLazyReferenceMapper.npMs rcddl}|Sr#)pandas)rpds rrXzLazyReferenceMapper.pdSs rNc>|_i_i_d_|t jdn|_jdjdgd5}| jd<dddn #1swxYwYtj jd}|d_ |d_ jrjdz_nd _t| fd }|_dS) a Parameters ---------- root : str Root of parquet store fs : fsspec.AbstractFileSystem fsspec filesystem object, default is local filesystem. cache_size : int Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. Nfile/z .zmetadatarb record_sizemetadataz/{field}/refs.{record}.parqz{field}/refs.{record}.parq)maxsizecj||}j|5}j|ddddn #1swxYwYfdjD}|fS)N)rGrecord fastparquet)enginec,i|]}||jSrK)r%).0cdfs r zCLazyReferenceMapper.__init__..open_refs..s!888Ar!u|888r)urlformatfsrrX read_parquetcolumns)rGrbr,frefsrhrs @r open_refsz/LazyReferenceMapper.__init__..open_refszs8??v?>>Dd## CqW))!M)BB C C C C C C C C C C C C C C C8888RZ888Dt8OsA""A&)A&)rootrHrAdirsfsspecrrlrjoinreadr>loadsr^r=rjrrq)rrrrl cache_sizerometrqs` rrzLazyReferenceMapper.__init__YsZ"   /1z&#F+++r W\\#((DI|#<==t D D 1()DK % 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1j\233}-Z 9 4y#@@DHH3DH : & & &     ' & #s0BB BTc|jz/LazyReferenceMapper.listdir..s)???1AGGCOOA&???rc3HK|]}||d|VdS).N startswithr~s r z.LazyReferenceMapper.listdir..s8"R"Ra"R S@Q@Q"R1"R"R"R"R"R"RrcLg|]!}tj|"SrK)osr,basename)rfr,s rrz/LazyReferenceMapper.listdir..s(BBB$rw''--BBBr)rsr=setsorted)rrrslistings rrBzLazyReferenceMapper.listdirsm 9 ?????DF"R"Rd"R"R"RRRSSDI)  CBB'BBBGrcR ||jvr |j|S||jvr1tj|j|Sd|vrt ||d\}} ||\} }|dkrdS|||\} n*#tttf$rt |wxYwgd} fd|D}|d} | | S|dd} | ddddgkr | dd} | S)Nr\rr)r,offsetrDrawc8g|]}|vr|ndSrrK)rfrgrpris rrz5LazyReferenceMapper._load_one_key..s-IIIAAIIT!WR[[4IIIrr|) rAr=r>r?r@KeyErrorr}_key_to_recordrq ValueError TypeErrorFileNotFoundError) rkeyrGsub_keyrb chunk_size_rn selectionrdatarprs @@r _load_one_keyz!LazyReferenceMapper._load_one_keys[ $+  ;s# # DN " ":dnS12299;; ; ^^3-- 3w %)%8%8%=%= "FB QsnnUF33GAttI'89   3--  333IIIIIIII m ?J"~ 81v  8D s?B: B::'C!cV|d\}}||}|jdkrdS|jd|dD}|j||}||jz}||jz}|||jfS)Nr\r)rrrc,g|]}t|SrK)intrfrgs rrz6LazyReferenceMapper._key_to_record..s"D"D"Da3q66"D"D"Drr)r}rCrDrUarrayravel_multi_indexr^) rrrGchunkrH chunk_idx chunk_numberrbrs rrz"LazyReferenceMapper._key_to_recordsyy~~ u++E22  q 7GMM"D"D5;;s3C3C"D"D"DEE w00KHH !11 D, ,r;+++rcD||jvr|j|d}|j|d|j|dz }|j|t |j|<|j|S)Nz/.zarrayshapechunks)rHr=rUrceilastyper)rrGzarray size_ratios rrCz$LazyReferenceMapper._get_chunk_sizess ( ( (^u$6$6$67Fvg77$'--x ;;J'+gll:&>&>&E&Ec&J&JD U #&&rc|||\}}|dd}|jjdkr d|DS|jjdkr d|DSd|DS)NF)nameindexc34K|]}t|VdSrr$rfts rrz7LazyReferenceMapper._generate_record..s(((DGG((((((rr|c3&K|] }|dV dS)rNrKrs rrz7LazyReferenceMapper._generate_record..s&%%QAaD%%%%%%rc3dK|]+}|dst|ddn|dV,dS)rNrrs rrz7LazyReferenceMapper._generate_record..sABB!qt5D2A2KKK1BBBBBBr)rq itertuplesrnrD)rrGrbrhrits r_generate_recordz$LazyReferenceMapper._generate_recordsuf--A ]]E] 2 2 :?a  ((R((( ( Z_ ! !%%"%%% %CBrBBB Brc#K||}t|j|j||jz }t |D]}|||Ed{VdSr)rCrrUrproductr^ranger)rrGrnrecrbs rrEz)LazyReferenceMapper._generate_all_recordss**511 47<< ; ;d>N NOOPPDkk < r?r@r)rrrFs r __getitem__zLazyReferenceMapper.__getitem__si $+  +c"C{ $. :dnS12299;; ;!!#&&&rc||j|<dSr)rA)rrvalues r __setitem__zLazyReferenceMapper.__setitem__s  Crcf||jvr |j|=dS||jvr |j|=dSd|j|<dSr)rAr=)rrs r __delitem__zLazyReferenceMapper.__delitem__sP $+   C   DN " "s### $DK   rc2d}|D]Q}|dr|dz }||}|j|}||z }R|t |jz }|t |jz }|S)Nrrr|)rBrrCrUrlenr=rA)rcountrGrHnchunkss r__len__zLazyReferenceMapper.__len__s\\^^ ! !E$$ ! "33E:: '//+66  T^$$$ T[!!! rc#2K|jEd{V|jEd{V|D]e}||}|jdkr|dzV*|jj|}|D])}|dzdd|DzV*fdS)Nrr;r\rc,g|]}t|SrK)r1rs rrz0LazyReferenceMapper.__iter__..s-B-B-Bc!ff-B-B-Br)r=rArBrCrDrUndindexru)rrGrHindsinds rrIzLazyReferenceMapper.__iter__s>!!!!!!!;\\^^ D DE//66K1$$dl""""47?K0D D DckCHH-B-Bc-B-B-B$C$CCCCCC D  D Dr)NrYT)rrr__doc__propertyrUrXrrBrrrCrrEr%rrrrrrIrKrrrRrRHsAX X  ,#,#,#,#\4 , , ,''' C C C<<< $$$###'''!!!$$$    D D D D DrrRceZdZdZdZ d'fd Zd(d Zd(d Zd(d Zd Z d Z e fdZ d)dZ d*dZd+dZdZd+dZdZdZdZd,dZd-dZdZdZdZd-dZd.fd Zd!Zd"Zd#Zd$Zd%Z d&Z!xZ"S)/ReferenceFileSystemaView byte ranges of some other file as a file system Initial version: single file system target, which must support async, and must allow start and end args in _cat_file. Later versions may allow multiple arbitrary URLs for the targets. This FileSystem is read-only. It is designed to be used with async targets (for now). This FileSystem only allows whole-file access, no ``open``. We do not get original file details from the target FS. Configuration is by passing a dict of references at init, or a URL to a JSON file containing the same; this dict can also contain concrete data for some set of paths. Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md rNT@BrYc ztjd i|||_| |_| |_i|_i|_i|_| |_| |_ t|trtd i|p|pid|i}tjj|fi|\}}||rt||d5}t$d|t)j|}dddn #1swxYwY||| n.t/||| |_n||| t|trEd|D|_d|jvrt5d|jd<dS|8t|jt8r |jdn|j}||j|<|{|jD]a}t=|r |}tj|\}}|r"||jvrt5|fi|pi}||j|<b||jD]}t=|r |}t|t@rT|drLtj|d\}}|r"||jvrt5|fi|pi}||j|<|r"||jvrt5|fi|pi}||j|<|pt5d|jd<dS) a Parameters ---------- fo : dict or str The set of references to use for this instance, with a structure as above. If str referencing a JSON file, will use fsspec.open, in conjunction with target_options and target_protocol to open and parse JSON at this location. If a directory, then assume references are a set of parquet files to be loaded lazily. target : str For any references having target_url as None, this is the default file target to use ref_storage_args : dict If references is a str, use these kwargs for loading the JSON file. Deprecated: use target_options instead. target_protocol : str Used for loading the reference file, if it is a path. If None, protocol will be derived from the given path target_options : dict Extra FS options for loading the reference file ``fo``, if given as a path remote_protocol : str The protocol of the filesystem on which the references will be evaluated (unless fs is provided). If not given, will be derived from the first URL that has a protocol in the templates or in the references, in that order. remote_options : dict kwargs to go with remote_protocol fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) Directly provide a file system(s): - a single filesystem instance - a dict of protocol:filesystem, where each value is either a filesystem instance, or a dict of kwargs that can be used to create in instance for the given protocol If this is given, remote_options and remote_protocol are ignored. template_overrides : dict Swap out any templates in the references file with these - useful for testing. simple_templates: bool Whether templates can be processed with simple replace (True) or if jinja is needed (False, much slower). All reference sets produced by ``kerchunk`` are simple in this sense, but the spec allows for complex. max_gap, max_block: int For merging multiple concurrent requests to the same remote file. Neighboring byte ranges will only be merged when their inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 to only merge when it requires no extra bytes. Pass a negative number to disable merging, appropriate for local target files. Neighboring byte ranges will only be merged when the size of the aggregated range is <= ``max_block``. Default is 256MB. cache_size : int Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. Only used for lazily loaded references. kwargs : passed to parent class r6r]zRead reference from URL %sN)rlrxc i|]I\}}|t|tr,tj|dddfi|n|JS):r|r)r*dictrtrr})rfkoptss rriz0ReferenceFileSystem.__init__..sl At !$--F%aggc1ooa&8AADAAA rr[rrK)!rrrtemplate_overridessimple_templates templatesfss _dircachemax_gap max_blockr*r1rrtcore url_to_fsisfilerloggerinfor>load_process_referencesrRr-rrr6r+r%callabler r$)rforref_storage_argstarget_protocoltarget_optionsremote_protocolremote_optionsrlrrrrrxkwargsdicref_fsrotextr.r6rrs rrzReferenceFileSystem.__init__sR ""6""" "4 0 " b#   =#;~;GVC .r99S99JFB}}R  [[T**(aKK #-R[%"@"@Q Abk )+DH_ %  "~,,.. , ,C== #%%C$k88== !, 8 8#HGG1E2GGB)+DHX&  "--// 0 0C== #%%Cc4((0SV0"(+"<">OFF0D"FFB(*DH_ %1z&11s0C==DDc 8||}td| |j|}n#t$rt |wxYwt |tr|}t |trTtd|d| drtj |dd}|ddfSt|dkr-td|d||d}||}}n]|\}}} td|d |d |d | || z} ||dkr||z}n| |z}n|}||dkr||z}n| |z}n| }||j}|||fS) Nzcat: z Reference: z , type bytesbase64:r|z, whole file => rz => z , offset z, size )_strip_protocolrdebugr-rrr*r1r@bytesrbase64 b64decoderr) rr,startendpartrjstart1end1start0rDend0s r _cat_commonzReferenceFileSystem._cat_commons##D)) ^T^^$$$ *?4(DD * * *#D)) ) * dC  !;;==D dE " " $ LL9t999 : : :z** 2'QRR11t# # t99>> LLCtCCTCC D D Dq'C #DFF $ C LLTtTTTTvTTdTT U U UD=D A::#e^FF!E\FF!88!CK||||\}}}t|tr |||St|\}} |j||||d{VdS#t $r} t||| d} ~ wwxYwN)rr)rr*rr r _cat_file Exceptionr rr,rrr part_or_urlrrr6res rr zReferenceFileSystem._cat_files$($4$4TC$4$P$P! VT k5 ) ) *uSy) )$[11 ! B(8$..{%S.QQ Q Q Q Q Q Q Q Q Q B B B'k:: A Bs)A<< BBBc *||||\}}}t|tr |||St|\}} |j||||S#t $r} t||| d} ~ wwxYwr )rr*rr rcat_filer rr s rrzReferenceFileSystem.cat_files$($4$4TC$4$P$P! VT k5 ) ) *uSy) )$[11 ! B8H%..{&d.SS S B B B'k:: A Bs"A22 B<B  Bc ||j|<dS)z2Temporarily add binary data or reference as a fileNr-)rr,rrs r pipe_filezReferenceFileSystem.pipe_files %rcK||rtj|dS||d{V}t |d5}||ddddS#1swxYwYdSNT)exist_okwb)isdirrmakedirsr rwrite)rrpathlpathrrros r _get_filezReferenceFileSystem._get_files ::e   5;ut444 4^^E******** %   ! GGDMMM                  sA<<BBc ||rtj|dS|j|fi|}|t |t |r||n=t|d5}||dddn #1swxYwY| t |dSr) rrrrset_sizerr rrabsolute_update)rrrcallbackrrros rget_filezReferenceFileSystem.get_files ::e   5;ut444 4t}U--f--#d))$$$ e    KK    eT"" a                  T+++++sB33B7:B7Fc |rd||}tjdd}t ||}|r"fd|D}n|}t ||D]%\}} ||vr|| ||&dS)N) recursiver[T) auto_mkdirc>g|]}||SrK)r)rfrrs rrz+ReferenceFileSystem.get..s(CCC1TZZ]]CQCCCr)ls expand_pathrtrr catrOr) rrrr&rrltargetsrremotelocals ` rr)zReferenceFileSystem.gets   GGBKKK  ) <<  v$ 7 7 7eU++  #88CCCCCCCDDDD88E??D 00 2 2MFE~~ UDL111 2 2rraisec @t|tr |rtt|tr"|st d|Drtt ||j}i}|D]\}}|j|} ggg} } } |D]} | | \}}}| || || |\#t$r}|dkr|dkr||| <Yd}~}d}~wwxYwg}g}g}g}t}t| | | |D]\}}}} t|tr||| <"|i||| || || || | t| | | |D]a\}}}} |X||vrT| || || || | bt!t|t|t|d|j|j\}}}| |||}t| | | |D]\}}}} | |vr t||||D]|\}}}}||kr-||)t|t(r||| <,||||| <:||kr<||kr6||kr0t|t(r||| <g|||z ||z pd|| <}|D]`\}}t|t(rF||jvr=||} t-||j|}!| |!_|dkr|!|dkr|!||<at1|dkr(t|trd|vrt3|S|S)Nc3K|]}d|vV dS)*NrKr~s rrz*ReferenceFileSystem.cat..s&7O7OQq7O7O7O7O7O7Orr0omitT)sortrrr|r3)r*r1NotImplementedErrorr$anyr7r-rrrr3rrrOraddr rr cat_rangesr copyr __cause__rr')"rr,r&on_errorr proto_dictr5protor4rlurlsstartsendsrusrurls2starts2ends2paths2 whole_files new_paths new_startsnew_ends bytes_outrUnsnebrvexnew_exs" rr,zReferenceFileSystem.cats dC  &Y &% % dD ! ! &y &C7O7O$7O7O7O4O4O &% %%dDO<< &,,..D BD BLE5%B!#R$&D # # #"..q11GAq!KKNNNMM!$$$KKNNNN(###7**6))!"A #EGEF%%K!$e<< % % 1aa'' %CFFY OOA&&&LLOOONN1%%%LLOOOMM!$$$!$e<< % % 1a=Qk%9%9LLOOONN1%%%LLOOOMM!$$$/BU W U  . /// +Iz8 iXFFI"$e<< B B 1a88%(J)%T%T B BMBBQwwBJ"*%a33,%&CFF%&qsVCFFqQ"WWb%a33B%&CFF%&q2vR0@D'@%ACF B BHHJJ$$&& $ $DAq!Y'' $A,@,@V.q$/!2DEE#% w&& L''#CF s88q==Zc22=s$#;;  sAC88 DDDc|dd}|||dS|dkr|||dStd|)Nversionr|)rz Unknown reference spec version: )r)_process_references0_process_references1r)rr-rverss rrz'ReferenceFileSystem._process_referencesqst~~i.. <  % %j 1 1 1 1 1 QYY  % %jEW % X X X X XFFFGG Grc||_dS)z&Make reference dict for Spec Version 0Nr)rr-s rrUz(ReferenceFileSystem._process_references0|s $rcjrjrddli_|dit dfd}|diD]\}}t|trD| dr$tj |ddj|<|j|<^jr|d}d|vrMjr;| dd  d d jdij}n ||}t|d kr|gn||d |d gj|<|j|<j|dgdS)NrricN|jdijSNrK)Templaterenderr)rBjinja2rs r _render_jinjaz?ReferenceFileSystem._process_references1.._render_jinjas*,6??1%%,>>t~>> >rrpzbase64:r{{{z}}}r|rgenrK)rrr^r-_process_templatesr)rrr*r1rrrreplacerkrupdate _process_gen)rr-rr_rrPrBr^s` @rrVz(ReferenceFileSystem._process_references1s$   MMM  {B ? ?@@@ 4 ? ? ? ? ?  ?NN62..4466 ' 'DAq!S!! '<< **A)/)9!ABB%)@)@DOA&%&"" 'aD199,-$AIIdC00$WT3//#66&*n66 *M!,,,/FFaKKaSSa1qt_""%&"" t00r1J1JKKLLLLLrci|_|j||j|D]'\}}d|vrddl|ffd |j|<||j|<(dS)Nr`rcD|jdi|Sr[)r\r])temprr^s rz8ReferenceFileSystem._process_templates..s7>V__>>>#>#!>#>#r)rrrfrr^)rtmprrPr^s @rrdz&ReferenceFileSystem._process_templatess  " . JJt. / / /IIKK & &DAqqyy 01%#%#%#%#%#q!!%&q!! & &rc  i}|D][}d|dD fdtj D}|D]}ddl}||djd i||j}||djd i||j}d|vrd|vr}t||djd i||j} t||djd i||j} || | g||<d|vd|vz rtd |g||<]|S) Nc i|]\\}}|t|tr|n>t|dd|d|dd]S)rrstopstepr|)r*r$rr))rfrrPs rriz4ReferenceFileSystem._process_gen..stAqa&&K11155!,,aivq9I9IJJr dimensionsc3vK|]3}tt|V4dSr)rrOrP)rfr% dimensions rrz3ReferenceFileSystem._process_gen..sSS))62233rrrrjrlengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.rK) r itertoolsrr%r^r\r]rrr) rgensr5rcproductsprr^rrjrrtrss @rrgz ReferenceFileSystem._process_gens % %C -3355 I '/1A1A1C1CDH % % 8fooc%j118PP2PPP8fooc%j118PP2PPPOO(c// =H 66=UUUdnUUF!=H 66=UUUdnUUF!$VV4CHH#o(c/:%$K !$uCHH' %( rcdgi|_|j}|D]\}}t|tt frt |}nt |dkrd}n|\}}}d|vr|dddnd}|}|r||jvr|g|j|<|jd|vr|dddndg |ddd| |}|r ||jv||j| |d|ddS)Nr%r|r\r directoryrtyperDr[) dircacher-rr*rr1rrsplitr2r3_parent) rrr,rrDrrparpar0s r_dircache_from_itemsz(ReferenceFileSystem._dircache_from_itemsstR _ " " $ $ T TJD$$ -- &4yyTa!%5$,/4KK$++c1%%a((RCD *4t}44&( d# ((.1TkkDKKQ''**r2&$ QGGHHH||D))  *4t}44 M#  % %tVT&R&R S S S S# T Trr]c T||}tj|Sr)rioBytesIO)rr,mode block_size cache_optionsrrs r_openzReferenceFileSystem._opens#}}T""z$rc ||}|js|||}|t ||r|Sd|DS)Ncg|] }|d SrrK)rfos rrz*ReferenceFileSystem.ls..s'''a& '''r)rr}r_ls_from_cacher)rr,detailrr5s rr*zReferenceFileSystem.lssx##D))} (  % % ' ' '!!$'' ;#D)) )  J''3''''rc V||p||Sr)rrrr,rs rexistszReferenceFileSystem.existss%zz$44;;t#4#44rcd|jr |jvStfd|jDS)Nc3HK|]}|dVdS)r\Nr)rfrr,s rrz,ReferenceFileSystem.isdir..s5IIAq||tJJJ//IIIIIIr)r}r7r-rr,s `rrzReferenceFileSystem.isdirsB = J4=( (IIIIIIIII Irc||jvSrrrs rrzReferenceFileSystem.isfilest&&rc$K|j||fi|Sr)r*)rr,rrs r_lszReferenceFileSystem._lss"twtV..v...rc :|rtjf|||d|Sr6tfdjD}ntj}|r)jsfd|DS|S)N)maxdepthwithdirsrc3FK|]}||VdSrr)rfrr,s rrz+ReferenceFileSystem.find..s4HHQQ\\$5G5GHqHHHHHHrcHi|]}||dS)r)r)rfrrs rriz,ReferenceFileSystem.find..s.<<.&s'888!ai4&7&7A&7&7&7rrzrrD) r-r*r1rrr*r rrD)rr,rr5out0protrs ` rrzReferenceFileSystem.infos& 4? " "/$'C#U|,, F $fc#hhGGGSA $fc!fEEE!%vtDDE''$%%C8888s888D F $k1EEE 76? "$T_T%:1%=>>GD!"htn11$/$2G2JKKDGFOAwrc0K||Sr)rrs r_infozReferenceFileSystem._info/syyrcrK|j|d|jdSr)r-popr}clearrs r_rm_filezReferenceFileSystem._rm_file2sA  $    rcPK||j|<|jdSr)r-r}r)rr,rs r _pipe_filezReferenceFileSystem._pipe_file8s, $ rcKt|d5}||j|<dddn #1swxYwY|jdS)Nr])rrvr-r}r)rrrros r _put_filezReferenceFileSystem._put_file=s %   .!%&VVXXDOE " . . . . . . . . . . . . . . . s<AAc i}|jD]u\}}t|trV |d||<4#t $r/dt j|z||<YlwxYw|||<vtj |dfi|5}| tj d|d ddddS#1swxYwYdS)z+Write modified references into new locationasciirrr|)rTrpN)r-rr*rdecodeUnicodeDecodeErrorr b64encodertrrr>r?r@)rrjstorage_optionsr5rrPros r save_jsonzReferenceFileSystem.save_jsonCsjO))++  DAq!U## IXXg..CFF)III(6+;A+>+>>FFHHCFFFIA [d 6 6o 6 6 F! GGDJ1c::;;BBDD E E E F F F F F F F F F F F F F F F F F Fs#A6B B %=C//C36C3) NNNNNNNNTrrrY)NN)F)Fr0r)r]NNr)NFF)#rrrrr6rrr rrrrr#r)r,rrUrVrdrgrrr*rrrrrrrrrrrr r!s@rrr s  H  R2R2R2R2R2R2h(!(!(!(!TBBBBBBBB&&&/@ , , , , 2 2 2 2ZZZZxHHHH%%%MMMM> & & &"""HTTT,     ( ( ( (555JJJ'''////$*   F F F F F F Frr)(r collectionsrruloggingr functoolsr fsspec.corertujsonr> ImportErrorasynr callbacksrrrrr utilsr r r getLoggerr RuntimeErrorrr'r/r7abc ValuesViewr9 ItemsViewrMMutableMappingrRrrKrrrse   KKKKK#"""""))))))3333333333@@@@@@@@@@  - . .TTTTTLTTT???  B B B B B[_/ B B BAAAAAKO-AAA @D@D@D@D@D+/8@D@D@DFD FD FD FD FD F/D FD FD FD FD Fs ) 55