
    Ng                         d dl Z d dlZd dlZd dlmZmZmZ d dlmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ  G d d	          Z G d
 de          Z G d de          ZdS )    N)AnyListOptional)urljoin)Document)HTTPBasicAuth)
BaseLoader)UnstructuredBaseLoaderc            
       Z    e Zd ZdZdededefdZdededed	ee         d
ef
dZ	d
efdZ
dS )LakeFSClientzClient for lakeFS.lakefs_access_keylakefs_secret_keylakefs_endpointc                 *   d                     |ddg          | _        t          ||          | _        	 t	          j        t          | j        d          | j                  }|                                 d S # t          $ r t          d          w xY w)N/apizv1/healthcheckauthz<lakeFS server isn't accessible. Make sure lakeFS is running.)
join_LakeFSClient__endpointr   _LakeFSClient__authrequestsgetr   raise_for_status	Exception
ValueError)selfr   r   r   health_checks        g/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/document_loaders/lakefs.py__init__zLakeFSClient.__init__   s     ((OUE#BCC#$57HII	#<77dk  L ))+++++ 	 	 	N  	s   AA8 8Breporefpathpresignreturnc           	      T   ||d}t           j                            |          }t          | j        d| d| d|           }t          j        || j                  }|                                 |	                                }	t          t          d |	d                             S )N)prefixr%   zrepositories/z/refs/z/objects/ls?r   c                 "    | d         | d         fS )Nr$   physical_address )ress    r    <lambda>z)LakeFSClient.ls_objects.<locals>.<lambda>1   s    S[#.@*AB     results)urllibparse	urlencoder   r   r   r   r   r   jsonlistmap)
r   r"   r#   r$   r%   qpeqpobjects_ls_endpointolsr	olsr_jsons
             r    
ls_objectszLakeFSClient.ls_objects$   s     11l$$R((%OOTOOOO#OO
 
 |/dkBBBIIKK	BBIiDX 
 
 	
r.   c                     | j         dz   }t          j        || j                  }|                                 |                                }|d         d         S )Nconfigr   storage_configpre_sign_support)r   r   r   r   r   r3   )r   config_endpointresponser=   s       r    is_presign_supportedz!LakeFSClient.is_presign_supported5   sU    /H4<dkBBB!!###&'(:;;r.   N)__name__
__module____qualname____doc__strr!   r   boolr   r;   rB   r+   r.   r    r   r      s          	   $

!
),
7?~
	
 
 
 
"<d < < < < < <r.   r   c                       e Zd ZU dZeed<   eed<   eed<   	 	 	 dded	ed
edee         dee         dee         fdZdeddfdZdeddfdZ	deddfdZ
dee         fdZddZdS )LakeFSLoaderzLoad from `lakeFS`.r"   r#   r$   Nmain r   r   r   c                     t          |||          | _        ||dk    rdnt          |          | _        ||dk    rdnt          |          | _        |dnt          |          | _        dS )a  

        :param lakefs_access_key: [required] lakeFS server's access key
        :param lakefs_secret_key: [required] lakeFS server's secret key
        :param lakefs_endpoint: [required] lakeFS server's endpoint address,
               ex: https://example.my-lakefs.com
        :param repo: [optional, default = ''] target repository
        :param ref: [optional, default = 'main'] target ref (branch name,
               tag, or commit ID)
        :param path: [optional, default = ''] target path
        NrL   rK   )r   _LakeFSLoader__lakefs_clientrG   r"   r#   r$   )r   r   r   r   r"   r#   r$   s          r    r!   zLakeFSLoader.__init__D   ss    *  ,0/ 
  
 ,$"**BB#d))	 [C2II663s88,BBCII			r.   r&   c                     || _         d S N)r$   )r   r$   s     r    set_pathzLakeFSLoader.set_path`       			r.   c                     || _         d S rP   )r#   )r   r#   s     r    set_refzLakeFSLoader.set_refc   s    r.   c                     || _         d S rP   )r"   )r   r"   s     r    set_repozLakeFSLoader.set_repof   rR   r.   c                 h   |                                   | j                                        }g }| j                            | j        | j        | j        |          }|D ]R}t          |d         | j        | j        |d         |          }|                    |	                                           S|S )N)r"   r#   r$   r%      r   )
 _LakeFSLoader__validate_instancerN   rB   r;   r"   r#   r$   UnstructuredLakeFSLoaderextendload)r   	presigneddocsobjsobjlakefs_unstructured_loaders         r    r\   zLakeFSLoader.loadi   s      """(==??	!#..ty) / 
 
  	; 	;C)AA	48SVY* *& KK27799::::r.   c                     | j         | j         dk    rt          d          | j        | j        dk    rt          d          | j        t          d          d S )NrL   zBno repository was provided. use `set_repo` to specify a repositoryz3no ref was provided. use `set_ref` to specify a refz6no path was provided. use `set_path` to specify a path)r"   r   r#   r$   r   s    r    __validate_instancez LakeFSLoader.__validate_instancew   sk    9	RT   8tx2~~RSSS9UVVV r.   )NrK   rL   )r&   N)rC   rD   rE   rF   rG   __annotations__r   r!   rQ   rT   rV   r   r   r\   rY   r+   r.   r    rJ   rJ   =   s@        
III	HHH
III ## 6 66 6 	6
 sm6 c]6 sm6 6 6 68S T    3 4    S T    d8n    W W W W W Wr.   rJ   c                   \     e Zd ZdZ	 	 	 ddedededed	ed
ef fdZdefdZ	de
fdZ xZS )rZ   z(Load from `lakeFS` as unstructured data.rK   rL   Turlr"   r#   r$   r%   unstructured_kwargsc                      t                      j        di | || _        || _        || _        || _        || _        dS )zInitialize UnstructuredLakeFSLoader.

        Args:

        :param lakefs_access_key:
        :param lakefs_secret_key:
        :param lakefs_endpoint:
        :param repo:
        :param ref:
        Nr+   )superr!   rg   r"   r#   r$   r%   )r   rg   r"   r#   r$   r%   rh   	__class__s          r    r!   z!UnstructuredLakeFSLoader.__init__   sJ    ( 	//.///		r.   r&   c                 ,    | j         | j        | j        dS )Nr"   r#   r$   rm   rc   s    r    _get_metadataz&UnstructuredLakeFSLoader._get_metadata   s    	$(DIFFFr.   c                    ddl m} d}| j        rt          j                    5 }| d| j                            d          d          }t          j        t          j        	                    |          d           t          j        | j                  }|                                 t          |d	          5 }|                    |j                   d d d            n# 1 swxY w Y    ||
          cd d d            S # 1 swxY w Y   d S | j                            |          st%          d          | j        t'          |          d          } ||
          S )Nr   )	partitionzlocal://r   T)exist_okwb)mode)filenamez>Non pre-signed URLs are supported only with 'local' blockstore)unstructured.partition.autorp   r%   tempfileTemporaryDirectoryr$   splitosmakedirsdirnamer   r   rg   r   openwritecontent
startswithr   len)r   rp   local_prefixtemp_dir	file_pathrA   file
local_paths           r    _get_elementsz&UnstructuredLakeFSLoader._get_elements   s   999999!< 	2,.. 5('DD$)//#*>*>r*BDD	BGOOI66FFFF#<11))+++)$/// 14JJx/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 y)4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 $$\22 	2P   #l"3"3"5"56J9j1111s6   BD;C"D"C&	&D)C&	*DD	D	)rK   rL   T)rC   rD   rE   rF   rG   rH   r   r!   dictrn   r   r   __classcell__)rk   s   @r    rZ   rZ      s        22    	
    #     6Gt G G G G2t 2 2 2 2 2 2 2 2r.   rZ   )rz   rw   urllib.parser0   typingr   r   r   r   r   langchain_core.documentsr   requests.authr   )langchain_community.document_loaders.baser	   1langchain_community.document_loaders.unstructuredr
   r   rJ   rZ   r+   r.   r    <module>r      s=   				      & & & & & & & & & &              - - - - - - ' ' ' ' ' ' @ @ @ @ @ @ T T T T T T+< +< +< +< +< +< +< +<\BW BW BW BW BW: BW BW BWJ52 52 52 52 525 52 52 52 52 52r.   