
    NgC                        d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ  ej                    Zd
ZddZddZddZ G d de          Z G d de          ZdS )    )annotationsN)sha1)Thread)AnyDictIterableListOptionalTuple)Document)
Embeddings)VectorStore)BaseSettingsSettingsConfigDictFsstrargsr   returnboolc                    |D ]	}|| vr dS 
dS )z
    Check if a string has multiple substrings.
    Args:
        s: The string to check
        *args: The substrings to check for in the string

    Returns:
        bool: True if all substrings are present in the string, False otherwise
    FT )r   r   as      f/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/vectorstores/starrocks.pyhas_mul_sub_strr      s-       A::55 4    Nonec                6    t           rt          |            dS dS )z[
    Print a debug message if DEBUG is True.
    Args:
        s: The message to print
    N)DEBUGprint)r   s    r   debug_outputr    "   s%      a r   
connectionqueryList[dict[str, Any]]c                \   |                                  }|                    |           |j        }g }|                                D ]A}i }t	          |          D ]\  }}||         d         }	|||	<   |                    |           Bt          |           |                                 |S )z
    Get a named result from a query.
    Args:
        connection: The connection to the database
        query: The query to execute

    Returns:
        List[dict[str, Any]]: The result of the query
    r   )cursorexecutedescriptionfetchall	enumerateappendr    close)
r!   r"   r%   columnsresultvalueridxdatumks
             r   get_named_resultr3   ,   s       F
NN5 GF""  #E** 	 	JCQAAaDDa
LLNNNMr   c                      e Zd ZU dZdZded<   dZded<   dZded	<   d
Zded<   dddddZ	ded<   dZ
ded<   dZded<   d!dZ edddd          Zd S )"StarRocksSettingsa  StarRocks client configuration.

    Attribute:
        StarRocks_host (str) : An URL to connect to MyScale backend.
                             Defaults to 'localhost'.
        StarRocks_port (int) : URL port to connect with HTTP. Defaults to 8443.
        username (str) : Username to login. Defaults to None.
        password (str) : Password to login. Defaults to None.
        database (str) : Database name to find the table. Defaults to 'default'.
        table (str) : Table name to operate on.
                      Defaults to 'vector_table'.

        column_map (Dict) : Column type map to project column name onto langchain
                            semantics. Must have keys: `text`, `id`, `vector`,
                            must be same size to number of columns. For example:
                            .. code-block:: python

                                {
                                    'id': 'text_id',
                                    'embedding': 'text_embedding',
                                    'document': 'text_plain',
                                    'metadata': 'metadata_dictionary_in_json',
                                }

                            Defaults to identity map.
    	localhostr   hostiF#  intportrootusername passwordiddocument	embeddingmetadata)r>   r?   r@   rA   zDict[str, str]
column_mapdefaultdatabase	langchaintableitemr   r   c                "    t          | |          S N)getattr)selfrG   s     r   __getitem__zStarRocksSettings.__getitem__p   s    tT"""r   z.envutf-8
starrocks_ignore)env_fileenv_file_encoding
env_prefixextraN)rG   r   r   r   )__name__
__module____qualname____doc__r7   __annotations__r9   r;   r=   rB   rD   rF   rL   r   model_configr   r   r   r5   r5   E   s          6 DDHH  	" "J     HE# # # # &%!	  LLLr   r5   c                       e Zd ZdZ	 d4d5 fdZd6dZed7d            Zd8dZd9dZ		 	 	 d:d;dZ
e	 	 	 	 d<d=d"            Zd>d#Z	 d4d?d)Z	 d@dAd.Z	 	 d@dBd/Z	 d@dCd1ZdDd2Zed>d3            Z xZS )E	StarRocksa  `StarRocks` vector store.

    You need a `pymysql` python package, and a valid account
    to connect to StarRocks.

    Right now StarRocks has only implemented `cosine_similarity` function to
    compute distance between two vectors. And there is no vector inside right now,
    so we have to iterate all vectors and compute spatial distance.

    For more information, please visit
        [StarRocks official site](https://www.starrocks.io/)
        [StarRocks github](https://github.com/StarRocks/starrocks)
    Nr@   r   configOptional[StarRocksSettings]kwargsr   r   r   c                P   	 ddl }n# t          $ r t          d          w xY w	 ddlm} || _        n# t          $ r d | _        Y nw xY wt	                                                       ||| _        nt                      | _        | j        sJ | j        j        r| j        j	        sJ | j        j
        r| j        j        r| j        j        sJ dD ]}|| j        j
        v sJ t          |                    d                    }d| j        j         d	| j        j         d
| j        j
        d          d| j        j
        d          d| j        j
        d          d| j        j
        d          d| _        || _        d| _        d| _        || _        d| _        t+          | j                    |j        d| j        j        | j        j	        | j        j        | j        j        | j        j        d|| _        t+          | j                   t5          | j        | j                   dS )zStarRocks Wrapper to LangChain

        embedding_function (Embeddings):
        config (StarRocksSettings): Configuration to StarRocks Client
        r   NzVCould not import pymysql python package. Please install it with `pip install pymysql`.)tqdmc                    | S rI   r   )xr^   s     r   <lambda>z$StarRocks.__init__.<locals>.<lambda>   s    Q r   )r>   r@   r?   rA   testzCREATE TABLE IF NOT EXISTS .z
(    
    r>   z string,
    r?   r@   z array<float>,
    rA   zf string
) ENGINE = OLAP PRIMARY KEY(id) DISTRIBUTED BY HASH(id)   PROPERTIES ("replication_num" = "1")\)rf   'DESC)r7   r9   userr=   rD   r   )pymysqlImportErrorr`   pgbarsuper__init__r\   r5   r7   r9   rB   rD   rF   lenembed_queryschemadimBSmust_escapeembedding_function
dist_orderr    connectr;   r=   r!   r3   )	rK   r@   r\   r^   rj   r`   r2   rr   	__class__s	           r   rn   zStarRocks.__init__   s   	NNNN 	 	 	@  	
	/!!!!!!DJJ 	/ 	/ 	/..DJJJ	/ 	 DKK+--DK{{4DK$4444{%T$+*>T4;CTTTT< 	/ 	/A...... )''//00 K0 37;3D 	[D!  
[J'  
[K(	 
 
[J'   &"+ T[!!! *'/ 
!!%[)[)
 
 
 
 	T[!!!$+66666s    "4 A	A	r.   r   c                F     d                      fd|D                       S )Nr<   c              3  D   K   | ]}|j         v rj         | n|V  d S rI   )rt   rs   ).0crK   s     r   	<genexpr>z'StarRocks.escape_str.<locals>.<genexpr>   s>      VV1!t/?*?*?$'1QVVVVVVr   )join)rK   r.   s   ` r   
escape_strzStarRocks.escape_str   s+    wwVVVVPUVVVVVVr   c                    | j         S rI   )ru   rK   s    r   
embeddingszStarRocks.embeddings   s    &&r   transacr   column_namesIterable[str]c           
         d                     |          }t          |                               j        j        d                   g }|D ]J}d                      fdt          |          D                       }|                    d| d           Kd j        j         d j        j         d| dd                     |           d		}|S )
N,r@   c                    g | ]C\  }}|k    r&d                      t          |                     d ndt          |           DS )rg   zarray<float>)r   r   )r{   r0   _nembed_tuple_indexrK   s      r   
<listcomp>z/StarRocks._build_insert_sql.<locals>.<listcomp>   sn        "b "333 8DOOCGG4477775CGG55	  r   ()z1
                INSERT INTO
                    re   z))
                VALUES
                z
                )	r~   tupleindexr\   rB   r)   r*   rD   rF   )rK   r   r   ks_datani_strr   s   `      @r   _build_insert_sqlzStarRocks._build_insert_sql   s   XXl##!,//55K";/
 
  	# 	#A     &/q\\  	 	A LLQ""""[) ,0K,= @B  %	   r   c                z    |                      ||          }t          |           t          | j        |           d S rI   )r   r    r3   r!   )rK   r   r   _insert_querys       r   _insertzStarRocks._insert   s=    ..wEE]###-88888r       texts	metadatasOptional[List[dict]]
batch_sizer8   idsOptional[Iterable[str]]	List[str]c           
        |pd |D             }| j         j        }g }|d         ||d         ||d         | j                            t	          |                    i}|pd |D             }t          t          j        |          ||d         <   t          t          |          t          |          z
            dk    sJ t          |                                 \  }	}
	 d}|                     t          |
 d	t          |          
          D ]}t          ||	                    | j         j        d                                      | j        k    sJ |                    |           t          |          |k    rD|r|                                 t#          | j        ||	g          }|                                 g }t          |          dk    r,|r|                                 |                     ||	           d |D             S # t(          $ rG}t*                              dt/          |           dt1          |           d           g cY d}~S d}~ww xY w)a  Insert more texts through the embeddings and add to the VectorStore.

        Args:
            texts: Iterable of strings to add to the VectorStore.
            ids: Optional list of ids to associate with the texts.
            batch_size: Batch size of insertion
            metadata: Optional column data to be inserted

        Returns:
            List of ids from adding the texts into the VectorStore.

        c                v    g | ]6}t          |                    d                                                     7S )rM   )r   encode	hexdigest)r{   ts     r   r   z'StarRocks.add_texts.<locals>.<listcomp>	  s8    IIIad188G,,--7799IIIr   r>   r?   r@   c                    g | ]}i S r   r   )r{   _s     r   r   z'StarRocks.add_texts.<locals>.<listcomp>  s    !4!4!4"!4!4!4r   rA   r   NzInserting data...)desctotal)targetr   c                    g | ]}|S r   r   )r{   is     r   r   z'StarRocks.add_texts.<locals>.<listcomp>(  s    ###!A###r   	[91m[1m
[0m [95m[0m)r\   rB   ru   embed_documentslistmapjsondumpsro   setzipitemsrl   r   rr   r*   r~   r   r   start	Exceptionloggererrortyper   )rK   r   r   r   r   r^   colmap_r   r   keysvaluesr   ves                 r   	add_textszStarRocks.add_texts   sj   * III5III+(DM3JK $"9"I"I$u++"V"V

 4!4!4e!4!4!4	,/
I,F,FWZ()3w<<#l"3"33449999L..001f	AZZV#6c)nn     ! ! $**T[%;K%HIIJKKtxWWWWq!!!w<<:-- !dl'4IIIAGGIII G7||a FFHHHWd+++##s#### 	 	 	LLS477SSCFFSSSTTTIIIIII	s   D*H	 	
I<IIIOptional[List[Dict[Any, Any]]]text_idsc                L     | ||fi |}|                     ||||           |S )a  Create StarRocks wrapper with existing texts

        Args:
            embedding_function (Embeddings): Function to extract text embedding
            texts (Iterable[str]): List or tuple of strings to be added
            config (StarRocksSettings, Optional): StarRocks configuration
            text_ids (Optional[Iterable], optional): IDs for the texts.
                                                     Defaults to None.
            batch_size (int, optional): Batchsize when transmitting data to StarRocks.
                                        Defaults to 32.
            metadata (List[dict], optional): metadata to texts. Defaults to None.
        Returns:
            StarRocks Index
        )r   r   r   )r   )	clsr   r@   r   r\   r   r   r^   ctxs	            r   
from_textszStarRocks.from_texts-  s<    2 c)V..v..ejIVVV
r   c                H   d| j         j         d| j         j         d}|| j         j         d| j         j         dz  }|d| j         j         dz  }d}d	}|d
||z  dz   z  dz   z  }g d}|d|d         dd|d         dz  }|d|d         ddz  }|d
||z  dz   z  dz   z  }d| j         j         d| j         j         }t          |           t          | j        |          }|D ]+}|d|d         dd|d         dz  }|d|d         ddz  },|d
||z  dz   z  dz   z  }|S )zText representation for StarRocks Vector Store, prints backends, username
            and schemas. Easy to use with `str(StarRocks())`

        Returns:
            repr: string to show connection info and data schema
        z	[92m[1mre   z @ :z[0m

z[1musername: z[0m

Table Schema:
      -   
)namer   keyz|[94mr   24sz
[0m|[96m   z[0m|
zDESC FieldTypeKey)	r\   rD   rF   r7   r9   r;   r    r3   r!   )rK   _reprwidthfieldsr,   q_strrsr/   s           r   __repr__zStarRocks.__repr__J  s    P$+"6OO9JOOODK$DDt{'7DDDDUT[%9UUUU*+d22)))MWQZMMMWQZMMMM>GAJ>>>>>*+d22B,BBt{/@BBUdou55 	A 	AAP7PPP6PPPPE@%@@@@@EE*+d22r   q_embList[float]topk	where_strOptional[str]c                V   d                     t          t          |                    }|rd| }nd}d| j        j        d          d| j        j        d          d| d	| j        j        d
          d| j        j         d| j        j         d| d| j         d| d}t          |           |S )Nr   zWHERE r<   z
            SELECT r?   z, 
                rA   z7, 
                cosine_similarity_norm(array<float>[z],
                  r@   z) as dist
            FROM re   z
            z
            ORDER BY dist z
            LIMIT )	r~   r   r   r\   rB   rD   rF   rv   r    )rK   r   r   r   	q_emb_strr   s         r   _build_query_sqlzStarRocks._build_query_sqld  s    HHSe__--	 	,,,III	K*:6	 	'
3	 	 6?	 	 ;)+6		 	
 +&	 	
 *.):	 	 	 	  ?	 	 	 	 	 	Ur      r"   r2   List[Document]c                R     | j         | j                            |          ||fi |S )a  Perform a similarity search with StarRocks

        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.

            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.

        Returns:
            List[Document]: List of Documents
        )similarity_search_by_vectorru   rp   )rK   r"   r2   r   r^   s        r   similarity_searchzStarRocks.similarity_search{  s@    & 0t/#//669
 
HN
 
 	
r   c           	                               |||          }	  fdt           j        |          D             S # t          $ rG}t                              dt          |           dt          |           d           g cY d}~S d}~ww xY w)a  Perform a similarity search with StarRocks by vectors

        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.

            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.

        Returns:
            List[Document]: List of (Document, similarity)
        c           
         g | ]Q}t          |j        j        d                   t          j        |j        j        d                                      RS )r?   rA   page_contentrA   r   r\   rB   r   loadsr{   r/   rK   s     r   r   z9StarRocks.similarity_search_by_vector.<locals>.<listcomp>  sh       
 	 !"4;#9*#E!F!Z$+*@*L(MNN    r   r   r   r   N)r   r3   r!   r   r   r   r   r   )rK   r@   r2   r   r^   r   r   s   `      r   r   z%StarRocks.similarity_search_by_vector  s    . %%iI>>
	   
 *$/5AA     	 	 	LLS477SSCFFSSSTTTIIIIII	s    ; 
B<BBBList[Tuple[Document, float]]c           	     N                           j                            |          ||          }	  fdt           j        |          D             S # t
          $ rG}t                              dt          |           dt          |           d           g cY d}~S d}~ww xY w)a  Perform a similarity search with StarRocks

        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.

            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.

        Returns:
            List[Document]: List of documents
        c           
         g | ]Y}t          |j        j        d                   t          j        |j        j        d                                      |d         fZS )r?   rA   r   distr   r   s     r   r   zEStarRocks.similarity_search_with_relevance_scores.<locals>.<listcomp>  sv     	 	 	  %&t{'=j'I%J!%Adk.DZ.P,Q!R!R   fI	 	 	r   r   r   r   N)
r   ru   rp   r3   r!   r   r   r   r   r   )rK   r"   r2   r   r^   r   r   s   `      r   'similarity_search_with_relevance_scoresz1StarRocks.similarity_search_with_relevance_scores  s    & %%#//669
 
		 	 	 	 *$/5AA	 	 	 	  	 	 	LLS477SSCFFSSSTTTIIIIII	s    A 
B$<BB$B$c                d    t          | j        d| j        j         d| j        j                    dS )z,
        Helper function: Drop data
        zDROP TABLE IF EXISTS re   N)r3   r!   r\   rD   rF   r   s    r   dropzStarRocks.drop  sB     	ONDK$8NN4;;LNN	
 	
 	
 	
 	
r   c                &    | j         j        d         S )NrA   )r\   rB   r   s    r   metadata_columnzStarRocks.metadata_column  s    {%j11r   rI   )r@   r   r\   r]   r^   r   r   r   )r.   r   r   r   )r   r   )r   r   r   r   r   r   )r   r   r   r   r   r   )Nr   N)r   r   r   r   r   r8   r   r   r^   r   r   r   )NNNr   )r   r   r@   r   r   r   r\   r]   r   r   r   r8   r^   r   r   r[   )r   r   )r   r   r   r8   r   r   r   r   )r   N)
r"   r   r2   r8   r   r   r^   r   r   r   )
r@   r   r2   r8   r   r   r^   r   r   r   )
r"   r   r2   r8   r   r   r^   r   r   r   )r   r   )rT   rU   rV   rW   rn   r   propertyr   r   r   r   classmethodr   r   r   r   r   r   r   r   __classcell__)rx   s   @r   r[   r[   {   s        " /3B7 B7 B7 B7 B7 B7 B7HW W W W ' ' ' X'   49 9 9 9 +/'+7 7 7 7 7r 
 59.2,0    [8   6 IM    0 BF
 
 
 
 
4 #'	" " " " "J BF# # # # #J
 
 
 
 2 2 2 X2 2 2 2 2r   r[   )r   r   r   r   r   r   )r   r   r   r   )r!   r   r"   r   r   r#   ) 
__future__r   r   logginghashlibr   	threadingr   typingr   r   r   r	   r
   r   langchain_core.documentsr   langchain_core.embeddingsr   langchain_core.vectorstoresr   pydantic_settingsr   r   	getLoggerr   r   r   r    r3   r5   r[   r   r   r   <module>r     s   " " " " " "               = = = = = = = = = = = = = = = = - - - - - - 0 0 0 0 0 0 3 3 3 3 3 3 > > > > > > > >				          23 3 3 3 3 3 3 3lk2 k2 k2 k2 k2 k2 k2 k2 k2 k2r   