
    gA                     2   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlZddlmZ dd	lmZ  ej        e          Zd
Z G d de          Z G d d          ZdefdZdee         fdZ G d de	          Z G d d          ZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs.

Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client
/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ```    N)ArgumentParser)AbstractContextManager)DictListOptional   )logging   )BaseTransformersCLICommandzlfs-multipart-uploadc                   .    e Zd ZdZedefd            ZdS )LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ transformers-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ transformers-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    parserc                    |                      dd          }|                    dt          d           |                    d            |                      t          d	          }|                    d
            d S )Nzlfs-enable-largefileszeDeprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                      t          |           S N)LfsEnableCommandargss    U/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/commands/lfs.py<lambda>z1LfsCommands.register_subcommand.<locals>.<lambda>?       5Ed5K5K     )funczgDeprecated: use `huggingface-cli` instead. Command will get called by git-lfs, do not call it directly.c                      t          |           S r   )LfsUploadCommandr   s    r   r   z1LfsCommands.register_subcommand.<locals>.<lambda>H   r   r   )
add_parseradd_argumentstrset_defaultsLFS_MULTIPART_UPLOAD_COMMAND)r   enable_parserupload_parsers      r   register_subcommandzLfsCommands.register_subcommand6   s    ))#w * 
 
 	""6:k"lll""(K(K"LLL))(O * 
 
 	""(K(K"LLLLLr   N)__name__
__module____qualname____doc__staticmethodr   r&    r   r   r   r   $   sL         " MN M M M \M M Mr   r   c                       e Zd Zd Zd ZdS )r   c                     || _         d S r   r   selfr   s     r   __init__zLfsEnableCommand.__init__L       			r   c                    t          j        d           t          j                            | j        j                  }t          j                            |          st          d           t          d           t          j
        d                                d|           t          j
        dt                                           d|           t          d           d S )	NzcManaging repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead.z)This does not look like a valid git repo.r
   z=git config lfs.customtransfer.multipart.path transformers-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)warningswarnosr   abspathr   isdirprintexit
subprocessrunsplitr#   )r0   
local_paths     r   r>   zLfsEnableCommand.runO   s    q	
 	
 	
 W__TY^44
w}}Z(( 	=>>>GGGKQQSS[_eo	
 	
 	
 	
 	Z<XZZ``bb	
 	
 	
 	

 	011111r   Nr'   r(   r)   r1   r>   r,   r   r   r   r   K   s2          2 2 2 2 2r   r   msgc                     t          j        |           dz   } t          j                            |            t          j                                         dS )z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflushrB   s    r   	write_msgrL   b   sD    
*S//D
 CJSJr   returnc                  p   t          j        t          j                                                                                  } d|                     d          |                     d          fv rdS |                     d          dvr.t                              d           t          j	        d           | S )z$Read Line delimited JSON from stdin.	terminater   eventN)downloaduploadzReceived unexpected messager
   )
rE   loadsrG   stdinreadlinestripgetloggercriticalr<   rK   s    r   read_msgrZ   i   s    
*SY''))//11
2
2Cswwv(8(8999t
www5555666Jr   c                   F    e Zd ZdZdededefdZd Zd Zdd	Z	d
 Z
d ZdS )	FileSlicezq
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    filepath	seek_from
read_limitc                 >    || _         || _        || _        d| _        d S )Nr   )r]   r^   r_   n_seen)r0   r]   r^   r_   s       r   r1   zFileSlice.__init__   s"     "$r   c                 x    t          | j        d          | _        | j                            | j                   | S )Nrb)openr]   fseekr^   r0   s    r   	__enter__zFileSlice.__enter__   s0    dmT**DN###r   c                     t          j        | j                                                  j        }t          | j        || j        z
            S r   )r8   fstatre   filenost_sizeminr_   r^   )r0   total_lengths     r   __len__zFileSlice.__len__   s7    x0084?L4>$ABBBr   c                     | j         | j        k    rdS | j        | j         z
  }| j                            |dk     r|nt	          ||                    }| xj         t          |          z  c_         |S )Nr   r   )ra   r_   re   readrm   len)r0   nremaining_amountdatas       r   rr   zFileSlice.read   sl    ;$/))3?T[8v{{q1uu++#aAQ:R:RSSs4yy r   c              #   :   K   |                      d          V  d S )Ni  @ )rt   )rr   rg   s    r   __iter__zFileSlice.__iter__   s&      ii/i*******r   c                 8    | j                                          d S r   )re   closer/   s     r   __exit__zFileSlice.__exit__   s    r   N)rp   )r'   r(   r)   r*   r!   intr1   rh   ro   rr   rx   r{   r,   r   r   r\   r\   x   s           #      
C C C   + + +    r   r\   c                       e Zd Zd Zd ZdS )r   c                     || _         d S r   r   r/   s     r   r1   zLfsUploadCommand.__init__   r2   r   c           	      x   t          j        t          j                                                                                  }|                    d          dk    r|                    d          dk    s(t          ddddi           t          j        d	           t          i            	 t                      }|t          j        d           |d         }|d         }|d         d         }|d         d         }t          |                    d                    }t          |                                          }g }	t          |          D ]\  }
}t          ||
|z  |          5 }t!          j        ||          }|                                 |	                    |j                            d          |
d	z   d           t          d||
d	z   |z  |d           d d d            n# 1 swxY w Y   t!          j        |||	d          }|                                 t          d|d           )NrP   init	operationrR   error    zWrong lfs init operation)codemessager
   Tr   oidr   actionhrefheader
chunk_size)r^   r_   )rv   etag)r   
partNumberprogress)rP   r   
bytesSoFarbytesSinceLast)r   parts)rE   complete)rP   r   )rE   rS   rG   rT   rU   rV   rW   rL   r<   rZ   r|   poplistvalues	enumerater\   requestsputraise_for_statusappendheaderspost)r0   init_msgrB   r   r]   completion_urlr   r   presigned_urlsr   ipresigned_urlrv   rs                 r   r>   zLfsUploadCommand.run   s    :ci002288::;;W%%//HLL4M4MQY4Y4Yw8R S STUUUHQKKK 	"0	9**C{
 e*C6{H ]62N]8,FVZZ5566J(,V]]__(=(=NE$-n$=$=   =x1z>jYYY ]a ]>>>A&&(((LL$%IMM&$9$9*+a%    %/#&+,q5J*>.8	                 * "   A    
377888a0	9s   +A8G//G3	6G3	NrA   r,   r   r   r   r      s7          A9 A9 A9 A9 A9r   r   )r*   rE   r8   r=   rG   r6   argparser   
contextlibr   typingr   r   r   r   utilsr	    r   
get_loggerr'   rX   r#   r   r   rL   rZ   r\   r   r,   r   r   <module>r      s  ` `  				     



  # # # # # # - - - - - - ' ' ' ' ' ' ' ' ' '        ( ( ( ( ( ( 
	H	%	%  6 $M $M $M $M $M, $M $M $MN2 2 2 2 2 2 2 2.4    (4.    " " " " "& " " "JE9 E9 E9 E9 E9 E9 E9 E9 E9 E9r   