
    g[                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m	Z	 d dl
mZ d dlmZ d dlmZmZmZmZmZ ddlmZ ddlmZmZmZ dd	lmZmZmZmZ dd
lmZmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' erddl(m)Z)  ej*        e+          Z,dZ-dZ.dZ/ddddddddddde0dee0ef         de0dee0         de1deeee0         e0f                  deeee0         e0f                  dee2         d e1d!e2fd"Z3 G d# d$e j4                  Z5eeef         Z6 G d% d&          Z7d'e7ddde0de0de0f
d(Z8d'e7d)eee5ee6         f                  fd*Z9d+e6d)dfd,Z:d-ee6         ddde0de0de0d)dfd.Z;d+e6ddde0de0de0d)dfd/Z<d-ee6         ddde0de0de0d)dfd0Z= G d1 d2e          Z>d+e6d)e>fd3Z?d4d5d)ee6         fd6Z@d4d5d7e2d)ee6         fd8ZAd4d5d)ee6         fd9ZBd:e0d)dfd;ZCdS )<    N)datetime)Path)Lock)TYPE_CHECKINGListOptionalTupleUnion   )	constants)CommitOperationAdd
UploadInfo_fetch_upload_modes)LocalUploadFileMetadataLocalUploadFilePathsget_local_upload_pathsread_upload_metadata)DEFAULT_REVISION
REPO_TYPES)DEFAULT_IGNORE_PATTERNSfilter_repo_objectstqdm)_format_size)sha_fileobj)HfApi
   K      FT<   )revisionprivateallow_patternsignore_patternsnum_workersprint_reportprint_report_everyapir   repo_idfolder_path	repo_typer    r!   r"   r#   r$   r%   r&   c                4    t          d          t          vrt          dt                     t          t                                                                                                                    st          d d          |g }nt          |t                    r|g}|t          z  }|(t          j                    pd}t          |dz
  d          }                     |d	          }t                              d
|            |j        t#          fd                    d          D             ||          }fd|D             }t                              dt'          |           d           fdt)          |d          D             }t+          |           fdt-          |          D             }|D ]}|                                 |	r$t1          d                                z              t5          j                    }	 t5          j        d           t5          j                    |z
  |
k    r6|	r!t9                                                     t5          j                    }                                rt=          j        d           n|D ]}|                                 t                                                                         t=          j        d           dS )zUpload a large folder to the Hub in the most resilient way possible.

    See [`HfApi.upload_large_folder`] for the full documentation.
    NzFor large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryr      T)r(   r*   r!   exist_okzRepo created: c              3      K   | ]?}|                                 |                                                              V  @d S N)is_filerelative_toas_posix).0pathr)   s     `/var/www/html/ai-engine/env/lib/python3.11/site-packages/huggingface_hub/_upload_large_folder.py	<genexpr>z/upload_large_folder_internal.<locals>.<genexpr>b   sO      iidZ^ZfZfZhZhi		+	&	&	/	/	1	1iiiiii    z**/*)r"   r#   c                 0    g | ]}t          |          S  )r   )r3   relpathr)   s     r5   
<listcomp>z0upload_large_folder_internal.<locals>.<listcomp>f   s$    bbb7(g>>bbbr7   zFound z candidate files to uploadc                 >    g | ]}|t          |j                  fS r9   )r   path_in_repo)r3   pathsr)   s     r5   r;   z0upload_large_folder_internal.<locals>.<listcomp>j   s<        
$[%2DEEF  r7   zRecovering from metadata files)descc                 R    g | ]#}t          j        t          d           $S ))statusr'   r(   r*   r    )targetkwargs)	threadingThread_worker_job)r3   _r'   r(   r*   r    rA   s     r5   r;   z0upload_large_folder_internal.<locals>.<listcomp>q   sY         	 "&$ 		
 		
 		
  r7   z

zIs done: exiting main loopzUpload is complete!) 
ValueErrorr   r   r   
expanduserresolveis_dir
isinstancestrr   os	cpu_countmaxcreate_repologgerinfor(   r   globlenr   LargeUploadStatusrangestartprintcurrent_reporttimesleep_print_overwriteis_doneloggingjoin)r'   r(   r)   r*   r    r!   r"   r#   r$   r%   r&   nb_coresrepo_urlfiltered_paths_list
paths_listitemsthreadsthreadlast_report_tsrA   s   `````              @r5   upload_large_folder_internalri   0   s}   & I
 
 	
 
""JjJJKKK#{##..0088::K OMKMMMNNN	OS	)	) ,*+..O<>>&Q(Q,** w)W_cddH
KK+++,,,G .iiiik>N>Nv>V>Viii%'  
 cbbbNabbbJ
KKDZDDDEEE   *+KLLL  E u%%F        {##  G     0fv,,...///Y[[N
19;;'+=== : !6!6!8!8999!Y[[N>> 	L5666   
KK%%''(((L&'''''r7   c                       e Zd Z ej                    Z ej                    Z ej                    Z ej                    Z ej                    Z	dS )	WorkerJobN)
__name__
__module____qualname__enumautoSHA256GET_UPLOAD_MODEPREUPLOAD_LFSCOMMITWAITr9   r7   r5   rk   rk      sO        TY[[FdikkODIKKMTY[[F49;;DDDr7   rk   c                   B    e Zd ZdZdee         fdZdefdZde	fdZ
dS )rV   zBContains information, queues and tasks for a large upload process.re   c                    || _         t          j                    | _        t          j                    | _        t          j                    | _        t          j                    | _        t                      | _        d| _	        d| _
        d| _        d| _        d| _        d | _        t          j                    | _        | j         D ]}|\  }}|j        | j                            |           )|j        | j                            |           K|j        dk    r"|j        s| j                            |           x|j        s| j                            |           t.                              d|j         d           d S )Nr   lfszSkipping file z! (already uploaded and committed))re   queueQueuequeue_sha256queue_get_upload_modequeue_preupload_lfsqueue_commitr   locknb_workers_sha256nb_workers_get_upload_modenb_workers_preupload_lfsnb_workers_commitnb_workers_waitinglast_commit_attemptr   now_started_atsha256putupload_modeis_uploadedis_committedrR   debugr=   )selfre   itemr>   metadatas        r5   __init__zLargeUploadStatus.__init__   ss   
7<{}}@E">Ckmm 7<{}}FF	&'/0'-.%&''(48 #<>> J 	e 	eD"OE8&!%%d++++%-*..t4444%..x7K.(,,T2222* e!%%d++++ce.@cccdddd	e 	er7   returnc                    d}d}d}d}d}d}d}d}d}	d}
d}| j         5  | j        D ]\  }}|j        r|
dz  }
|	|j        z  }	|dz  }|j        |dz  }||j        z  }|j        dk    r|dz  }|j        |dz  }|j        r|dz  }||j        z  }|j        r|dz  }||j        z  }t          |	          }t          j
                    }|                    d          }|| j        z
  }t          |                              d          d         }d}|d| d	| d
z  }|dz  }|dz  }|d| d| d	t          |           d| d	z  }|d| d| d	t          |           d| d	z  }|dk    r	|d| dz  }|d| d| d	t          |           d| d	z  }|d|
 dz  }|dz  }|d| j         dz  }|d| j         dz  }|d| j         dz  }|d| j         dz  }|d| j         dz  }|dz  }|cddd           S # 1 swxY w Y   dS )z<Generate a report of the current status of the large upload.r   r   Nrx   z%Y-%m-%d %H:%M:%S.z
---------- z (z) z----------
z	Files:   zhashed /z) | zpre-uploaded: )z (+z unsure)z | committed: z | ignored: 
z	Workers: z	hashing: z | zget upload mode: zpre-uploading: zcommitting: z	waiting: z3---------------------------------------------------)r   re   should_ignoresizer   r   r   r   r   r   r   strftimer   rM   splitr   r   r   r   r   )r   	nb_hashedsize_hashednb_preuploadednb_lfsnb_lfs_unsuresize_preuploadednb_committedsize_committed
total_sizeignored_filestotal_filesrG   r   total_size_strr   now_strelapsedelapsed_strmessages                       r5   rZ   z LargeUploadStatus.current_report   ss   	
Y /	 /	#z 4 48) !Q&Mhm+
q ?.NI8=0K'500aKF'/!Q&M' 6"a'N$5$( 4 A%L"hm3N)*55N,..Cll#677GD,,Gg,,,,S11!4K%G5755k5555G&G{"Glll[llL<U<UllXfllllGuuuuu<P`CaCauudruuuuGq  88888vvv{vvlSaFbFbvvesvvvvG7m7777G{"G>4#9>>>>GO4+JOOOOGK)FKKKKGAd&<AAAAG>4#:>>>>GxG_/	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	 /	s   GG::G>G>c                     | j         5  t          d | j        D                       cd d d            S # 1 swxY w Y   d S )Nc              3   8   K   | ]\  }}|j         p|j        V  d S r/   r   r   r3   rG   r   s      r5   r6   z,LargeUploadStatus.is_done.<locals>.<genexpr>
  s1      ee;1hx,F0Feeeeeer7   )r   allre   r   s    r5   r^   zLargeUploadStatus.is_done  s    Y 	f 	feeZ^Zdeeeee	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	fs   377N)rl   rm   rn   __doc__r   
JOB_ITEM_Tr   rM   rZ   boolr^   r9   r7   r5   rV   rV      s}        LLed:. e e e e>= = = = =~f f f f f f fr7   rV   rA   c                 p   	 d}t          |           }|dS |\  }}|t          j        k    r|d         }	 t          |           | j                            |           nk# t          $ r  t          $ rT}	t          	                    d|	            t          j                     | j                            |           Y d}	~	nd}	~	ww xY w| j        5  | xj        dz  c_        ddd           n# 1 swxY w Y   n;|t          j        k    r	 t!          |||||           nQ# t          $ r  t          $ r:}	t          	                    d|	            t          j                     Y d}	~	nd}	~	ww xY w|D ]u}|\  }
}|j        r|j        dk    r| j                            |           5|j        d	k    r| j                            |           [| j                            |           v| j        5  | xj        dz  c_        ddd           n# 1 swxY w Y   n|t          j        k    r|d         }	 t/          |||||           | j                            |           nk# t          $ r  t          $ rT}	t          	                    d
|	            t          j                     | j                            |           Y d}	~	nd}	~	ww xY w| j        5  | xj        dz  c_        ddd           n# 1 swxY w Y   n6|t          j        k    r	 t5          |||||           np# t          $ r  t          $ rY}	t          	                    d|	            t          j                     |D ]}| j                            |           Y d}	~	nd}	~	ww xY w| j        5  t7          j                    | _        | xj        dz  c_        ddd           n# 1 swxY w Y   nX|t          j        k    rHt7          j        t@                     | j        5  | xj!        dz  c_!        ddd           n# 1 swxY w Y   6)a  
    Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded
    and committed. If no tasks are available, the worker will wait for 10 seconds before checking again.

    If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up.

    Read `upload_large_folder` docstring for more information on how tasks are prioritized.
    TNr   zFailed to compute sha256: r   )r'   r(   r*   r    zFailed to get upload mode: rx   regularzFailed to preupload LFS: zFailed to commit: )"_determine_next_jobrk   rq   _compute_sha256r|   r   KeyboardInterrupt	ExceptionrR   error	traceback
format_excr{   r   r   rr   _get_upload_moder   r   r}   r~   r   rs   _preupload_lfsr   rt   _commitr[   r   r   ru   r\   WAITING_TIME_IF_NO_TASKSr   )rA   r'   r(   r*   r    next_jobjobre   r   erG   r   s               r5   rF   rF     sG   U/AE 'v..F
U )"""8D.%%%,006666$    . . .=!==>>>$&&&#''--------.
  . .((A-((. . . . . . . . . . . . . . . I---' CI`hiiiii$    ' ' '>1>>???$&&&&&&&&'  	; 	;"8) '500.2248888)Y66'++D1111044T:::: 7 711Q6117 7 7 7 7 7 7 7 7 7 7 7 7 7 7 I+++8D5tg]effff#''----$    5 5 5<<<===$&&&*..t444444445
  5 5//14//5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 I$$$239W_`````$    2 2 25!55666$&&&! 2 2D'++D11112 2 2 2 22
  . .-1Y[[*((A-((. . . . . . . . . . . . . . . IN""J/000 / /))Q.))/ / / / / / / / / / / / / / /iU/s   )A C3A
CCC..C25C2D" "E060E++E02HHH5.I$ $K8A
KKK33K7:K7L& &N:ANN)OOOP++P/2P/r   c                    | j         5  | j        dk    r| j                                        dk    r|| j        ut          j                    | j        z
  dk    rV| xj        dz  c_        t                              d           t          j	        t          | j                  fcd d d            S | j        dk    rs| j                                        dk    rV| xj        dz  c_        t                              d           t          j	        t          | j                  fcd d d            S | j                                        dk    rW| xj        dz  c_        t                              d           t          j        t          | j        d	          fcd d d            S | j                                        dk    ra| j        dk    rV| xj        dz  c_        t                              d
           t          j        t%          | j                  fcd d d            S | j                                        dk    ra| j        dk    rV| xj        dz  c_        t                              d           t          j        t%          | j                  fcd d d            S | j                                        dk    rb| j        dk    rW| xj        dz  c_        t                              d           t          j        t          | j        d	          fcd d d            S | j                                        dk    rm| j        dk    st,          j        sV| xj        dz  c_        t                              d           t          j        t%          | j                  fcd d d            S | j                                        dk    rV| xj        dz  c_        t                              d           t          j        t%          | j                  fcd d d            S | j                                        dk    rW| xj        dz  c_        t                              d           t          j        t          | j        d	          fcd d d            S | j        dk    r| j                                        dk    r|| j        ut          j                    | j        z
  dk    rV| xj        dz  c_        t                              d           t          j	        t          | j                  fcd d d            S | j        dk    r| j                                        dk    r| j                                        dk    r| j                                        dk    r| j                                        dk    rw| j        dk    rl| j        dk    ra| j        dk    rV| xj        dz  c_        t                              d           t          j	        t          | j                  fcd d d            S t1          d | j        D                       r(t                              d           	 d d d            d S | xj        dz  c_        t                              dt8           d           t          j        g fcd d d            S # 1 swxY w Y   d S )Nr   i,  r   z;Job: commit (more than 5 minutes since last commit attempt)r   zJob: commit (>100 files ready)r   z&Job: get upload mode (>10 files ready)2   z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSzJob: sha256zJob: get upload moder   z-Job: commit (1 min since last commit attempt)zJob: commitc              3   8   K   | ]\  }}|j         p|j        V  d S r/   r   r   s      r5   r6   z&_determine_next_job.<locals>.<genexpr>  s1      aa[Q&@(*@aaaaaar7   z.All files have been processed! Exiting worker.zNo task available, waiting... (zs))r   r   r~   qsizer   r[   rR   r   rk   rt   _get_items_to_commitr|   r   rr   _get_nr}   r   rs   _get_oner{   r   rq   r   HF_HUB_ENABLE_HF_TRANSFERr   re   rS   r   r   ru   )rA   s    r5   r   r   t  s]   	 c( c( $))#))++a//*6	f886AA$$)$$LLVWWW$&:6;N&O&OPc( c( c( c( c( c( c( c( %**v/B/H/H/J/Jc/Q/Q$$)$$LL9:::$&:6;N&O&OP!c( c( c( c( c( c( c( c(& )//11R77--2--LLABBB-vf6RTV/W/WX-c( c( c( c( c( c( c( c(2 '--//!338W[\8\8\++q0++LLPQQQ+Xf6P-Q-QR9c( c( c( c( c( c( c( c(>  &&((1,,1IQ1N1N$$)$$LLIJJJ$hv/B&C&CDEc( c( c( c( c( c( c( c(J )//11A55&:[_`:`:`--2--LLUVVV-vf6RTV/W/WXQc( c( c( c( c( c( c( c(X '--//!33+q00	8[0++q0++LL-...+Xf6P-Q-QRcc( c( c( c( c( c( c( c(h  &&((1,,$$)$$LL'''$hv/B&C&CDoc( c( c( c( c( c( c( c(t )//11A55--2--LL/000-vf6RTV/W/WX{c( c( c( c( c( c( c( c(B $))#))++a//*6	f886AA$$)$$LLHIII$&:6;N&O&OPQc( c( c( c( c( c( c( c(Z $))#))++a//#))++q00,224499*0022a77(A--1Q66/144$$)$$LL'''$&:6;N&O&OPqc( c( c( c( c( c( c( c(v aaTZT`aaaaa 	(KKHIII{c( c( c( c( c( c( c( c(B %%*%%LLW;SWWWXXXNB'Gc( c( c( c( c( c( c( c( c( c( c( c( c( c( c( c( c( c(se   BZ8-A1Z8+A'Z8A1Z8A1Z8A2Z8A=Z8$A&Z8A'Z8BZ8/C)Z8%9Z8+A Z88Z<?Z<r   c                     | \  }}|j         X|j                            d          5 }t          |                                          |_         ddd           n# 1 swxY w Y   |                    |           dS )z1Compute sha256 of a file and save it in metadata.Nrb)r   	file_pathopenr   hexsave)r   r>   r   fs       r5   r   r     s    OE8_!!$'' 	31)!nn0022HO	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3MM%s   'AA!Are   c                     d | D             }t          ||||                                |           t          | |          D ]7\  }}|\  }}	|j        |	_        |j        |	_        |	                    |           8dS )zmGet upload mode for each file and update metadata.

    Also receive info if the file should be ignored.
    c                 ,    g | ]}t          |          S r9   _build_hacky_operationr3   r   s     r5   r;   z$_get_upload_mode.<locals>.<listcomp>  !    @@@$'--@@@r7   )	additionsr*   r(   headersr    N)r   _build_hf_headerszip_upload_moder   _should_ignorer   r   )
re   r'   r(   r*   r    r   r   additionr>   r   s
             r5   r   r     s    
 A@%@@@I%%''    eY//  hx'4!)!8e	 r7   c                     | \  }}t          |           }|                    ||||g           d|_        |                    |           dS )z'Preupload LFS file and update metadata.)r(   r*   r    r   TN)r   preupload_lfs_filesr   r   )r   r'   r(   r*   r    r>   r   r   s           r5   r   r     se    OE8%d++H*	      HMM%r7   c                     d | D             }|                     ||||d           | D ]!\  }}d|_        |                    |           "dS )zCommit files to the repo.c                 ,    g | ]}t          |          S r9   r   r   s     r5   r;   z_commit.<locals>.<listcomp>  r   r7   z(Add files using upload-large-folder tool)r(   r*   r    
operationscommit_messageTN)create_commitr   r   )re   r'   r(   r*   r    r   r>   r   s           r5   r   r     s}    @@%@@@IA     !  x $e r7   c                       e Zd ZddZdS )HackyCommitOperationAddr   Nc                 p    t          | j        t                    rt          | j                  | _        d S d S r/   )rL   path_or_fileobjr   rM   r   s    r5   __post_init__z%HackyCommitOperationAdd.__post_init__!  s:    d*D11 	=#&t';#<#<D   	= 	=r7   )r   N)rl   rm   rn   r   r9   r7   r5   r   r      s(        = = = = = =r7   r   c                    | \  }}t          |j        |j                  }|j                            d          5 }|                    d          d d         }d d d            n# 1 swxY w Y   |j        t          d          t          t          	                    |j                  |j
        |          |_        |S )N)r=   r   r   i   z&sha256 must have been computed by now!)r   r   sample)r   r=   r   r   peekr   rH   r   bytesfromhexr   upload_info)r   r>   r   	operationfiler   s         r5   r   r   &  s    OE8'U5GY^YhiiiI			d	#	# &t3%& & & & & & & & & & & & & & &ABBB&emmHO.L.LS[S`iopppIs   A%%A),A)ry   zqueue.Queue[JOB_ITEM_T]c                 "     | j                     gS r/   get)ry   s    r5   r   r   6  s    EIKK=r7   nc                 p      fdt          t            j                    |                    D             S )Nc                 .    g | ]} j                     S r9   r   )r3   rG   ry   s     r5   r;   z_get_n.<locals>.<listcomp>;  s!    >>>AIEIKK>>>r7   )rW   minr   )ry   r   s   ` r5   r   r   :  s7    >>>>s;5;==!'<'<!=!=>>>>r7   c                     g }d\  }}	  | j                     dk    r|S |t          k    s|t          k    r|S  | j                    }|                    |           |\  }}|j        dk    r|dz  }n|dz  }m)zXSpecial case for commit job: the number of items to commit depends on the type of files.)r   r   Tr   rx   r   )r   MAX_NB_LFS_FILES_PER_COMMITMAX_NB_REGULAR_FILES_PER_COMMITr   appendr   )ry   re   r   
nb_regularr   rG   r   s          r5   r   r   >  s     !EFJ5;==AL 000JBa4a4aL uy{{T85((aKFF!OJ!r7   reportc                 J   | dz  } t          j                    j        t          fd|                                 D                       }t          |          D ]@}t          j                            d           t          j                            d           At          j                            |            t          j                            dt          |                                 d                   z
  z             t          j        
                                 dS )zPrint a report, overwriting the previous lines.

    Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout`
    to print the report.

    Note: works well only if no other process is writing to `sys.stdout`!
    r   c              3   B   K   | ]}t          |          z  d z   V  dS )r   N)rU   )r3   lineterminal_widths     r5   r6   z#_print_overwrite.<locals>.<genexpr>c  s3      SSt3t99.2SSSSSSr7   z[Kz[Fr   N)shutilget_terminal_sizecolumnssum
splitlinesrW   sysstdoutwriterU   flush)r  nb_linesrG   r  s      @r5   r]   r]   V  s     dNF-//7N SSSSv?P?P?R?RSSSSSH 8__ # #
$$$
"""" JVJSNS1B1B1D1DR1H-I-IIJKKKJr7   )Dro   r_   rN   ry   r  r  rD   r[   r   r   pathlibr   r   typingr   r   r   r	   r
    r   _commit_apir   r   r   _local_folderr   r   r   r   r   r   utilsr   r   r   utils._cache_managerr   	utils.shar   hf_apir   	getLoggerrl   rR   r   r   r   rM   r   intri   Enumrk   r   rV   rF   r   r   r   r   r   r   r   r   r   r   r]   r9   r7   r5   <module>r     s|     				   



                            > > > > > > > > > > > > > >       L L L L L L L L L L v v v v v v v v v v v v 3 3 3 3 3 3 3 3 E E E E E E E E E E . . . . . . " " " " " "  		8	$	$ "$ !  #6:7;!% d( d( d(	d(d( sDy!d(
 d( smd( d( U49c>23d( eDIsN34d( #d( d( d( d( d( d(X    	    ')@@A
cf cf cf cf cf cf cf cfLd/d/	d/ d/ 	d/
 d/ d/ d/ d/Nd( 1 d(huYPTU_P`E`?a6b d( d( d( d(X*     D, 7 S UX dg lp    ( ' C C [^ cg    4
# ' C C [^ cg    (= = = = =0 = = = 0G     - $z2B    ?+ ? ?Z8H ? ? ? ? 9 d:>N    0S T      r7   