
    gA                        d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZmZmZmZmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZm Z m!Z!m"Z" d
dl#m$Z$ d
dl%m&Z&m'Z' erd
dl(m)Z)  e j*        e+          Z, ej-        d          Z.dZ/dddZ0e G d d                      Z1e"	 	 	 d9dee1         dee2         de2de2dee2         dee2         deee2e2f                  deee3         ee3         f         fd            Z4 G d de          Z5 G d  d!e          Z6	 	 	 d9d"d#d$edee2         deee2e2f                  dee2         ddfd%Z7d&e3fd'Z8d(e3fd)Z9d*e3fd+Z:d"d#d,e2ddfd-Z;d"d#d.ed/e<d,e2ddf
d0Z=d.ed1e1d/e<dee2         fd2Z>d3ee         d4e2de6fd5Z?d"d#d6ee2         d/e<dee         fd7Z@d"d#d6ee2         d/e<dee         fd8ZAdS ):z.Git LFS related type definitions and utilities    N)	dataclass)ceil)getsize)Path)TYPE_CHECKINGBinaryIODictIterableListOptionalTuple	TypedDict)unquote)	constants   )build_hf_headersfix_hf_endpoint_in_urlget_sessionhf_raise_for_statushttp_backoffloggingtqdmvalidate_hf_hub_args)SliceFileObj)sha256sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)AcceptzContent-Typec                       e Zd ZU dZeed<   eed<   eed<   edefd            Z	edefd            Z
ed	efd
            ZdS )
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   sizesamplepathc                     t          |          }t          j        |d          5 }|                    d          d d         }t	          |          }d d d            n# 1 swxY w Y    | |||          S )Nrb   r!   r   r"   )r   ioopenpeekr   )clsr#   r!   filer"   shas         O/var/www/html/ai-engine/env/lib/python3.11/site-packages/huggingface_hub/lfs.py	from_pathzUploadInfo.from_pathN   s    t}}WT4   	$DYYs^^DSD)Fd##C	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ sS8888s   -AA"%A"datac                     t          |                                          } | t          |          |d d         |          S )Nr&   )r!   r"   r   )r   digestlen)r+   r0   r-   s      r.   
from_byteszUploadInfo.from_bytesV   s>    Tll!!##sD		$tt*SAAAA    fileobjc                    |                     d          }|                    dt          j                   t	          |          }|                                }|                    dt          j                    | |||          S )Nr&   r   r'   )readseekr(   SEEK_SETr   tell)r+   r6   r"   r-   r!   s        r.   from_fileobjzUploadInfo.from_fileobj[   sp    c""Q$$$'""||~~Q$$$sS8888r5   N)__name__
__module____qualname____doc__bytes__annotations__intclassmethodstrr/   r4   r   r<    r5   r.   r    r    ;   s           MMM
IIIMMM9S 9 9 9 [9 Be B B B [B 98 9 9 9 [9 9 9r5   r    upload_infostoken	repo_typerepo_idrevisionendpointheadersreturnc                 8   ||nt           j        }d}|t           j        v rt           j        |         }| d| | d}dddgd | D             d	d
}	|dt          |          i|	d<   i t          t          |          |pi }t                                          |||	          }
t          |
           |
	                                }|
                    dd          }t          |t                    st          d          d |D             d |D             fS )a  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        revision (`str`, *optional*):
            The git revision to upload to.
        headers (`dict`, *optional*):
            Additional headers to include in the request

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If an argument is invalid or the server response is malformed.
        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            If the server returned an error.
    N /z.git/info/lfs/objects/batchuploadbasic	multipartc                 P    g | ]#}|j                                         |j        d $S )oidr!   )r   hexr!   ).0rR   s     r.   
<listcomp>z'post_lfs_batch_info.<locals>.<listcomp>   sG     
 
 

  }((** 
 
 
r5   r   )	operation	transfersobjects	hash_algonameref)rH   rM   jsonr]   zMalformed response from serverc                 4    g | ]}d |vt          |          S error)_validate_batch_actionsrY   objs     r.   rZ   z'post_lfs_batch_info.<locals>.<listcomp>   s+    OOO#G3<N<N	 	%	%<N<N<Nr5   c                 4    g | ]}d |v t          |          S rd   )_validate_batch_errorrg   s     r.   rZ   z'post_lfs_batch_info.<locals>.<listcomp>   s&    III'S..	s	#	#...r5   )r   ENDPOINTREPO_TYPES_URL_PREFIXESr   LFS_HEADERSr   r   postr   rb   get
isinstancelist
ValueError)rG   rH   rI   rJ   rK   rL   rM   
url_prefix	batch_urlpayloadresp
batch_infor]   s                r.   post_lfs_batch_inforx   e   sw   P $/xxY5GHJI5556yA
NNjN'NNNI{+
 

 '
 
 
  G  '("3"34


'
'
' =bG
 ==iwGGDJnnY--Ggt$$ ;9::: 	POOOOIIwIII r5   c                   $    e Zd ZU eed<   eed<   dS )PayloadPartT
partNumberetagN)r=   r>   r?   rC   rB   rE   rF   r5   r.   rz   rz      s"         OOO
IIIIIr5   rz   c                   4    e Zd ZU dZeed<   ee         ed<   dS )CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rW   partsN)r=   r>   r?   r@   rE   rB   r   rz   rF   r5   r.   r~   r~      s2         II	HHHr5   r~   r[   r   lfs_batch_actionc                    t          |           |                    d          }|%t                              d| j         d           dS |d         d         }t          |           |d                             d          }|t          |           |                    di           }|                    d          }	t          |d	         |
          }
|	M	 t          |	          }	n(# t          t          f$ r t          d|	 d          w xY wt          | ||	|
           nt          | |
           |t          |           t          |d	         |          }t                                          |t          ||          | j        j                                        | j        j        d          }t'          |           t                              | j         d           dS )aw  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        headers (`dict`, *optional*):
            Headers to include in the request, including authentication and user agent headers.

    Raises:
        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If `lfs_batch_action` is improperly formatted
        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            If the upload resulted in an error
    actionsNzContent of file z. is already present upstream - skipping uploadrR   verifyheader
chunk_sizehref)rL   zTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.)r[   r   r   
upload_url)r[   r   )rH   rM   rV   ra   z: Upload successful)rf   ro   loggerdebugpath_in_repo_validate_lfs_actionr   rC   rr   	TypeError_upload_multi_part_upload_single_partr   rn   r   upload_infor   rX   r!   r   )r[   r   rH   rM   rL   r   upload_actionverify_actionr   r   r   
verify_urlverify_resps                r.   
lfs_uploadr      s   8 ,---""9--Gn	(>nnnooo %Y/9M'''$Y/33H==M ]+++ x,,FL))J'f(=QQQJ	ZJJI& 	 	 	ugquuu  	 	Yv*akllllliJGGGG  ]++++M&,A8LL
!mm(($5'BBB".599;;YEZE_`` ) 
 

 	K(((
LLI*???@@@@@s   C* *%D
lfs_actionc                     t          |                     d          t                    r=|                     d          7t          |                     d          t                    st	          d          | S ).validates response from the LFS batch endpointr   r   Nz"lfs_action is improperly formatted)rp   ro   rE   dictrr   )r   s    r.   r   r     se     	:>>&))3//?^^H%%-JNN8<T<TVZ1[1[-=>>>r5   lfs_batch_actionsc                    t          |                     d          t                    r(t          |                     d          t                    st	          d          |                     di                               d          }|                     di                               d          }|t          |           |t          |           | S )r   rW   r!   z)lfs_batch_actions is improperly formattedr   rR   r   )rp   ro   rE   rC   rr   r   )r   r   r   s      r.   rf   rf     s    (,,U33S99 FjIZI^I^_eIfIfhk>l>l FDEEE%)))R88<<XFFM%)))R88<<XFFM ]+++ ]+++r5   lfs_batch_errorc                    t          |                     d          t                    r(t          |                     d          t                    st	          d          |                     d          }t          |t
                    rPt          |                    d          t                    r(t          |                    d          t                    st	          d          | S )r   rW   r!   z'lfs_batch_error is improperly formattedre   messagecode)rp   ro   rE   rC   rr   r   )r   
error_infos     r.   rj   rj     s    **511377 DJGZGZ[aGbGbdg<h<h DBCCC $$W--J:t$$Dz~~i00#66D z~~f--s33D
 BCCCr5   r   c                     |                      d          5 }t          d||d          }t          |           ddd           dS # 1 swxY w Y   dS )a  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises:
     [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
        If the upload resulted in an error.
    T	with_tqdmPUTi  i  i  i  r0   retry_on_status_codesN)as_filer   r   )r[   r   r6   responses       r.   r   r   ,  s      
		T		*	* &gzWklllH%%%& & & & & & & & & & & & & & & & & &s   #AAAr   r   c                    t          || j        |          }t          j        }t          j        rJt	          | j        t                    s0t	          | j        t                    st          j	        d           d}|rt          | ||          nt          | ||          }t                                          |t          || j        j                                                  t"                    }t%          |           dS )z@
    Uploads file using HF multipart LFS transfer protocol.
    )r   r   r   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)r[   sorted_parts_urlsr   )rb   rM   N)_get_sorted_parts_urlsr   r   HF_HUB_ENABLE_HF_TRANSFERrp   path_or_fileobjrE   r   warningswarn_upload_parts_hf_transfer_upload_parts_iterativelyr   rn   _get_completion_payloadr   rX   rm   r   )r[   r   r   r   r   use_hf_transferresponse_headerscompletion_ress           r.   r   r   B  s   
 /f)J_lvwww  9O+	 94c::	  94d;;	 
 		
 	
 	
   	x!IIZgqrrrr&N_lvwww  !]]''$%5y7L7S7W7W7Y7YZZ (  N
 '''''r5   r   c                     d t          d |                                 D             d           D             }t          |          }|t          |j        |z            k    rt          d          |S )Nc                     g | ]\  }}|S rF   rF   )rY   _r   s      r.   rZ   z*_get_sorted_parts_urls.<locals>.<listcomp>f  s,     
 
 
Az 	
 
 
r5   c                     g | ]>\  }}|                                 r%t          |          d k    ,t          |d          |f?S )r   
   )isdigitr3   rC   )rY   part_numr   s      r.   rZ   z*_get_sorted_parts_urls.<locals>.<listcomp>i  s\       (Hj##%% +.h--!*;*; Xr""J/*;*;*;r5   c                     | d         S )Nr   rF   )ts    r.   <lambda>z(_get_sorted_parts_urls.<locals>.<lambda>n  s
    !A$ r5   )keyz0Invalid server response to upload large LFS file)sorteditemsr3   r   r!   rr   )r   r   r   sorted_part_upload_urls	num_partss        r.   r   r   e  s    
 
# ,2LLNN  
 
 
 

 
 
 +,,ID)J67777KLLL""r5   r   rW   c                     g }t          |           D ]U\  }}|                    d          }||dk    rt          d| d|dz              |                    |dz   |d           V||dS )Nr|   rP   zInvalid etag (`z`) returned for part r   )r{   r|   )rW   r   )	enumeratero   rr   append)r   rW   r   part_numberr   r|   s         r.   r   r   w  s     "E()9:: 	
 	
Vzz&!!<42::[t[[+XY/[[\\\)Ao 	
 	
 	
 	
 '''r5   r   c           	      d   g }|                      d          5 }t          |          D ]n\  }}t          |||z  |          5 }t          d||d          }t	          |           |                    |j                   d d d            n# 1 swxY w Y   o	 d d d            n# 1 swxY w Y   |S )NTr   )	seek_from
read_limitr   r   r   )r   r   r   r   r   r   rM   )	r[   r   r   rM   r6   part_idxpart_upload_urlfileobj_slicepart_upload_ress	            r.   r   r     sW    G			T		*	* 8g)23D)E)E 	8 	8%Ho$x/%   
8 ".?Vj# # # $O4446777
8 
8 
8 
8 
8 
8 
8 
8 
8 
8 
8 
8 
8 
8 
8	88 8 8 8 8 8 8 8 8 8 8 8 8 8 8 Ns5   *B%=B B%BB%BB%%B),B)c                    	 ddl m} n# t          $ r t          d          w xY wdt	          j        |          j        v }|st          j        d           | j	        j
        }| j        }t          |          dk    rd|dd           }t                                          t          j        k    rd	nd }t#          d
d	|d||d          5 }	  |d| j        ||dddd|r	d|j        ini }	n"# t(          $ r}
t+          d          |
d }
~
ww xY w|s|                    |           |	cd d d            S # 1 swxY w Y   d S )Nr   )multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.callbackzYou are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`.(   u   (…)iTBzhuggingface_hub.lfs_upload)unit
unit_scaletotalinitialdescdisabler_            )	file_path
parts_urlsr   	max_filesparallel_failuresmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.rF   )hf_transferr   ImportErrorrr   inspect	signature
parametersr   r   r   r!   r   r3   r   getEffectiveLevelr   NOTSETr   r   update	ExceptionRuntimeError)r[   r   r   r   supports_callbackr   r   r   progressoutputes              r.   r   r     s'   
0000000 
 
 
Q
 
 	

 #g&78H&I&I&TT 
 c	
 	
 	
 !&E!D
4yy2~~#tCDDz## //11W^CCdd$G	)
 
 
  
	%% #3,%"%  5FMJ002 FF  	 	 	*  	
 ! 	#OOE"""5                 s>   	 #D4C-,D4-
D7DDD44D8;D8)NNN)Br@   r   r(   rer   dataclassesr   mathr   os.pathr   pathlibr   typingr   r   r	   r
   r   r   r   r   urllib.parser   huggingface_hubr   utilsr   r   r   r   r   r   r   r   
utils._lfsr   	utils.shar   r   _commit_apir   
get_loggerr=   r   compile	OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDrm   r    rE   r   rx   rz   r~   r   r   rf   rj   r   rC   r   r   r   r   r   rF   r5   r.   <module>r     s-   5 4  				 				  ! ! ! ! ! !                   \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \             % % % % % %	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 % $ $ $ $ $ * * * * * * * *  0//////		H	%	%BJ())	5  -2  &9 &9 &9 &9 &9 &9 &9 &9R  #"(,K K:&KC=K K 	K
 smK smK d38n%K 4:tDz!"K K K K\    9   
         (,"CA CA#CACA C=CA d38n%	CA
 smCA 
CA CA CA CALT    t    4    &#7 &S &T & & & &, ("6  (  (RU  (cf  (ko  (  (  (  (F#4 #j #c #VZ[^V_ # # # #$(d4j (s (GY ( ( ( (#8<S	OR	$Z   (5#58<S	5OR5	$Z5 5 5 5 5 5r5   