çgAAddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl Z ddl m Z m Z ddlmZmZejdZGdd e ZGd d e ZdS) N)suppress)quote)AbstractBufferedFileAbstractFileSystem)infer_storage_optionstokenizewebhdfsc\eZdZdZeejZdZ d#fd Z e dZ d Z d$d Z d%d ZedZedZedZdZd&dZdZdZdZd'dZdZdZdZd(dZdZdZd&dZ dZ!d&dZ"d Z#d!Z$d"Z%xZ&S))WebHDFSa Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways. Four auth mechanisms are supported: insecure: no auth is done, and the user is assumed to be whoever they say they are (parameter ``user``), or a predefined value such as "dr.who" if not given spnego: when kerberos authentication is enabled, auth is negotiated by requests_kerberos https://github.com/requests/requests-kerberos . This establishes a session based on existing kinit login and/or specified principal/password; parameters are passed with ``kerb_kwargs`` token: uses an existing Hadoop delegation token from another secured service. Indeed, this client can also generate such tokens when not insecure. Note that tokens expire, but can be renewed (by a previously specified user) and may allow for proxying. basic-auth: used when both parameter ``user`` and parameter ``password`` are provided. )r webHDFSFNTc  |jrdStjdi| | rdndd|d|d|_||_|pi|_i|_| pi|_|||td||jd<||_ ||_ ||td n | ||jd <| ||jd <|r|td | |_ | |_ | d t|||_dS)a Parameters ---------- host: str Name-node address port: int Port for webHDFS kerberos: bool Whether to authenticate with kerberos for this connection token: str or None If given, use this token on every call to authenticate. A user and user-proxy may be encoded in the token and should not be also given user: str or None If given, assert the user name to connect with password: str or None If given, assert the password to use for basic auth. If password is provided, user must be provided also proxy_to: str or None If given, the user has the authority to proxy, and this value is the user in who's name actions are taken kerb_kwargs: dict Any extra arguments for HTTPKerberosAuth, see ``_ data_proxy: dict, callable or None If given, map data-node addresses. This can be necessary if the HDFS cluster is behind a proxy, running on Docker or otherwise has a mismatch between the host-names given by the name-node and the address by which to refer to them from the client. If a dict, maps host names ``host->data_proxy[host]``; if a callable, full URLs are passed, and function must conform to ``url->data_proxy(url)``. use_https: bool Whether to connect to the Name-node using HTTPS instead of HTTP session_cert: str or Tuple[str, str] or None Path to a certificate file, or tuple of (cert, key) files to use for the requests.Session session_verify: str, bool or None Path to a certificate file to use for verifying the requests.Session. kwargs Nhttpshttpz://:z /webhdfs/v1z_If passing a delegation token, must not set user or proxy_to, as these are encoded in the token delegationzQIf passing a password, the user must also beset in order to set up the basic-authz user.namedoaszJIf using Kerberos auth, do not specify the user, this is handled by kinit.webhdfs_)_cachedsuper__init__urlkerb kerb_kwargsparsproxy ValueErroruserpassword session_certsession_verify_connectr _fsid)selfhostportkerberostokenr r!proxy_tor data_proxy use_httpsr"r#kwargs __class__s Z/var/www/html/ai-engine/env/lib/python3.11/site-packages/fsspec/implementations/webhdfs.pyrzWebHDFS.__init__-ssr <  F""6"""!*6ggSS4SS$SSS &," %2  8#7  ',DIl #    | < )- +&   (DIf   (2  ), 6t 4 466 c|jSN)r%r&s r0fsidz WebHDFS.fsids zr1c@tj|_|jr|j|j_|j|j_|jrddlm }|di|j |j_ |j /|j *ddlm}||j |j |j_ dSdSdS)Nr)HTTPKerberosAuth) HTTPBasicAuthr)requestsSessionsessionr"certr#verifyrrequests_kerberosr7rauthr r! requests.authr8)r&r7r8s r0r$zWebHDFS._connects'))   2 $ 1DL "1  9 E : : : : : : 0 0 D D43C D DDL  9 T]%> 3 3 3 3 3 3 - di G GDL    ! %>%>r1getc ||jt|pddz}|}||j||d<td|||j |||||} | j dvr | } | dd } | dd } | d vrt| | d vrt| | d vrt| t!| #tt"f$rYnwxYw| | S)Nz/=)safeopzsending %s with %s)methodrparamsdataallow_redirects)iiiiiRemoteExceptionmessage exception)IllegalArgumentExceptionUnsupportedOperationException)SecurityExceptionAccessControlException)FileNotFoundException) _apply_proxyrrcopyupdaterupperloggerdebugr;request status_codejsonrPermissionErrorFileNotFoundError RuntimeErrorKeyErrorraise_for_status) r&rErFpathrHredirectr.rargsouterrmsgexps r0_callz WebHDFS._calls5$+G+G+G GHH{{}} DIXXZZT  )3777l""<<>>$ #   ?7 7 7 ,hhjj+,Y7+,[9WWW$S//)KKK)#...555+C000&s+++)      s 0EEErbc L|p|j}t|||||j|||S)a^ Parameters ---------- path: str File location mode: str 'rb', 'wb', etc. block_size: int Client buffer size for read-ahead or write buffer autocommit: bool If False, writes to temporary file that only gets put in final location upon commit replication: int Number of copies of file on the cluster, write mode only permissions: str or int posix permissions, write mode only kwargs Returns ------- WebHDFile instance )mode block_sizetempdir autocommit replication permissions) blocksize WebHDFilerl)r&r`rjrkrmrnror.s r0_openz WebHDFS._opensBB 14>   !L!##    r1cV|d|d<|d|d<|S)Ntypelengthsize)lower)infos r0 _process_infozWebHDFS._process_infos-F|))++V H~V  r1c,t|dS)Nr`)r)clsr`s r0_strip_protocolzWebHDFS._strip_protocols$T**622r1ct|}|dd|ddd|vr|d|d<|S)Nr`protocolusernamer )rpop)urlpathrcs r0_get_kwargs_from_urlszWebHDFS._get_kwargs_from_urlss\#G,,   D!!!   ''*--CK r1c|d|}|d}||d<||S)N GETFILESTATUSr` FileStatusname)rgrZry)r&r`rcrxs r0rxz WebHDFS.infosFjjtj44xxzz,'V !!$'''r1cF|d|}|dd}|D];}|||ddz|dz|d<<|rt |d St d |DS) N LISTSTATUSr FileStatusesr/ pathSuffixrc|dS)Nrr)is r0zWebHDFS.ls..s qyr1)keyc3&K|] }|dV dS)rNr).0rxs r0 zWebHDFS.ls..s&994$v,999999r1)rgrZryrstripsorted)r&r`detailrcinfosrxs r0lsz WebHDFS.lssjjDj11 >*<8 G GD   t $ $ $;;s++c1D4FFDLL  :%%8%8999 999599999 9r1cd|d|}|dS)z8Total numbers of files, directories and bytes under pathGETCONTENTSUMMARYrContentSummaryrgrZ)r&r`rcs r0content_summaryzWebHDFS.content_summarys,jj,4j88xxzz*++r1cp|d|d}d|jvrh||jd}|j|}||dS||dS)z/Checksum info of file, giving method and resultGETFILECHECKSUMF)r`raLocation FileChecksum)rgheadersrRr;rAr_rZ)r&r`rclocationout2s r0ukeyz WebHDFS.ukeysjj*jFF  $ $((Z)@AAH<##H--D  ! ! # # #99;;~. .  " " "88::n- -r1c`|d}|dS)zGet user's home directoryGETHOMEDIRECTORYPathr)r&rcs r0home_directoryzWebHDFS.home_directory*s'jj+,,xxzz&!!r1c|r|d|}n|d}|d}|td|dS)zRetrieve token which can give the same authority to other uses Parameters ---------- renewer: str or None User who may use this token; if None, will be current user GETDELEGATIONTOKEN)renewerTokenNz1No token available for this user/security context urlString)rgrZr)r&rrcts r0get_delegation_tokenzWebHDFS.get_delegation_token/sf  3**17*CCCC**122C HHJJw  9PQQ Q~r1cf|dd|}|dS)z/Make token live longer. Returns new expiry timeRENEWDELEGATIONTOKENputrFr*longr)r&r*rcs r0renew_delegation_tokenzWebHDFS.renew_delegation_token@s-jj/UjKKxxzz&!!r1c6|dd|dS)z Stop the token from being usefulCANCELDELEGATIONTOKENrrNrg)r&r*s r0cancel_delegation_tokenzWebHDFS.cancel_delegation_tokenEs! *5 FFFFFr1c8|dd||dS)aSet the permission at path Parameters ---------- path: str location to set (file or directory) mod: str or int posix epresentation or permission, give as oct string, e.g, '777' or 0o777 SETPERMISSIONr)rFr` permissionNr)r&r`mods r0chmodz WebHDFS.chmodIs$ ?5t LLLLLr1cFi}|||d<|||d<|jdd|d|dS)zChange owning user and/or groupNownergroupSETOWNERrrFr`)rr)r&r`rrr.s r0chownz WebHDFS.chownVsJ  #F7O  #F7O Ae$AA&AAAAAr1c8|d|d|dS)a9 Set file replication factor Parameters ---------- path: str File location (not for directories) replication: int Number of copies of file on the cluster. Should be smaller than number of data nodes; normally 3 on most systems. SETREPLICATIONr)r`rFrnNr)r&r`rns r0set_replicationzWebHDFS.set_replication_s% #$u+ VVVVVr1c 6|dd|dS)NMKDIRSrrrr&r`r.s r0mkdirz WebHDFS.mkdirms  8E 55555r1c|dur$||rt|||dS)NF)existsFileExistsErrorr)r&r`exist_oks r0makedirszWebHDFS.makedirspsD u  T!2!2 !$'' ' 4r1c 8|dd||dS)NRENAMEr)rFr` destinationr)r&path1path2r.s r0mvz WebHDFS.mvus" 8E5 IIIIIr1c @|dd||rdnddS)NDELETEdeletetruefalse)rFr` recursiver)r&r`rr.s r0rmz WebHDFS.rmxs<  )6ffw      r1c 0||dSr3)rrs r0rm_filezWebHDFS.rm_files  r1c 8||5}d||dtjdg} ||d5}t j||dddn #1swxYwY|||nP#t$rCtt5| |dddn #1swxYwYwxYw ddddS#1swxYwYdS)Nrz.tmp.wb) openjoin_parentsecrets token_hexshutil copyfileobjr BaseExceptionrr\r)r&lpathrpathr.lstream tmp_fnamerstreams r0cp_filezWebHDFS.cp_files YYu   $,,u"5"57Vw?PQS?T?T7V7V!WXXI YYy$//97&w888999999999999999 5))))    /00''GGI&&&''''''''''''''' *                  srADB1-B B1B B1B B10D1C>C1% C>1C5 5C>8C5 9C>>DDDc|jr*t|jr||}n=|jr6|jD]\}}|||d}|S)N)rcallableitemsreplace)r&rkvs r0rRzWebHDFS._apply_proxysy : 5(4:.. 5zz(++HH Z 5 ((** 5 51#++Aq!44r1) rFNNNNNNFNT)rANNT)rhNTNNFr3)NN)'__name__ __module__ __qualname____doc__strtempfile gettempdirrlr~rpropertyr5r$rgrr staticmethodry classmethodr|rrxrrrrrrrrrrrrrrrrrR __classcell__r/s@r0r r s*c%(%''((G#H  c7c7c7c7c7c7JXHHH$F+ + + + Z\ 33[3\((( : : : :,,, . . .""" """" GGG M M MBBBB W W W666 JJJ       r1r cBeZdZdZfdZd dZdZdZdZdZ xZ S) rqz"A file living in HDFS over webHDFSc Htj||fi||}|dd|dd|dd|dd|dd|_|d}|dddurQ|j|_tj |ttj |_dSdS)NrornirlrmF) rrrSrArror`targetosrruuiduuid4)r&fsr`r.rlr/s r0rzWebHDFile.__init__sT,,V,,, ::mT * * 2 JJ}d + + + ::mT * * 2 JJ}d + + +!::mS99**Y'' ::lE * *e 3 3)DK Wc$*,,.?.?@@DIII 4 3r1Fc|jj|j|jddi}|dS)zWrite one part of a multi-block file upload Parameters ========== final: bool This is the last block, so should complete file, if self.autocommit is True. content-typeapplication/octet-stream)rHrT)r r;postrbuffergetvaluer_)r&finalrcs r0 _upload_chunkzWebHDFile._upload_chunksZgo"" M%%''#%?@#   tr1c|j}d|jvrd\}}n d\}}d|d<|jj|||jfddi|}|j|jd}d |jvr|jj |d d i }| |jjd d|jfddi|}|j|jd|_ dSdS)zCreate remote file/uploada)APPENDPOST)CREATEPUTr overwriteraFrwrr)rrrN) r.rSrjr rgr`rRrr;rr_r)r&r.rErFrcrrs r0_initiate_uploadzWebHDFile._initiate_uploads!!## $)  )JB(JB"(F; dgmB LLELVLL7'' J(?@@ $)  7?&&>3M"N'D  ! ! # # # 47=649WWuWPVWWD G00j1IJJDMMM  r1ct|d}t|j|}||ks ||jkrdS|jd|j|||z d}|d|jvrK|jd}|jj |j |}|j S|j S)Nrr1OPENF)r`offsetrurar) maxminrvr rgr`r_rr;rArRcontent)r&startendrcrrs r0 _fetch_rangezWebHDFile._fetch_rangesE1 $)S!! C<<5DI--3gmm 5uu     $ ${:.H7?&&tw';';H'E'EFFD< ; r1cP|j|j|jdSr3)r rr`rr4s r0commitzWebHDFile.commits"  49dk*****r1cD|j|jdSr3)r rr`r4s r0discardzWebHDFile.discards  49r1r) rrrrrrrr&r(r*rrs@r0rqrqs,, A A A A A"KKK( +++r1rq)loggingr rrrr  contextlibr urllib.parserr9specrrutilsrr getLoggerrVr rqrr1r0r1s   ;;;;;;;;33333333  9 % %DDDDD DDDN IIIII$IIIIIr1