NgW`ddlmZddlZddlZddlZddlZddlmZmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZddlmZddlmZd Zdd ZddZGddeZdS)) annotationsN)AnyCallableDictIterableListOptionalType)Document) Embeddings guard_import) VectorStore)maximal_marginal_relevancereturnrc tdS)zImport lancedb package.lancedbr d/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/vectorstores/lancedb.pyimport_lancedbrs  " ""rfilterDict[str, str]strcddd|DS)z2Converts a dict filter to a LanceDB filter string.z AND c$g|] \}}|d|dS)z = ''r).0kvs r z#to_lance_filter..s(CCCDAqA1CCCr)joinitems)rs rto_lance_filterr%s+ <<CCFLLNNCCC D DDrcfeZdZdZdddddddddddd ddefd`dZdadbd%Zedcd&Z ddded/Z dfdgd3Z dhdid?Z djdAZ dddkdCZ dldmdGZdndIZ dldodLZ dldodMZ dddpdNZ dqdrdQZ dsdtdWZ dsdudXZe dvdwd\Z dxdyd_ZdS)zLanceDBay`LanceDB` vector store. To use, you should have ``lancedb`` python package installed. You can install it with ``pip install lancedb``. Args: connection: LanceDB connection to use. If not provided, a new connection will be created. embedding: Embedding to use for the vectorstore. vector_key: Key to use for the vector in the database. Defaults to ``vector``. id_key: Key to use for the id in the database. Defaults to ``id``. text_key: Key to use for the text in the database. Defaults to ``text``. table_name: Name of the table to use. Defaults to ``vectorstore``. api_key: API key to use for LanceDB cloud database. region: Region to use for LanceDB cloud database. mode: Mode to use for adding data to the table. Valid values are ``append`` and ``overwrite``. Defaults to ``overwrite``. Example: .. code-block:: python vectorstore = LanceDB(uri='/lancedb', embedding_function) vectorstore.add_texts(['text1', 'text2']) result = vectorstore.similarity_search('text1') Nz /tmp/lancedbvectoridtext vectorstore overwritel2 connection Optional[Any] embeddingOptional[Embeddings]uri Optional[str] vector_keyid_keytext_key table_nameapi_keyregionmodetabledistancererankerrelevance_score_fn"Optional[Callable[[float], float]]limitintc td}||_||_||_||_|dkr|pt jdnd|_| |_| |_ | |_ ||_ ||_ d|_ t| |jjr| |_n| d|_nt%dt|t&r+|j$|drt%d|jt%dt||jjr||_nt|t&|jjfrt%d |j|||_nt|t&rk|dr(|||j|j |_n.|||_t5jd | r t| |jj|jjjfsJ| |_tA| d r| j!nd |_"dS#tF$rt%dwxYw|$|d|_dS)z$Initialize with Lance DB vectorstorer LANCE_API_KEYNz9`reranker` has to be a lancedb.rerankers.Reranker object.zdb://z&API key is required for LanceDB cloud.z#embedding object should be providedzs`connection` has to be a lancedb.db.LanceDBConnection object. `lancedb.db.LanceTable` is deprecated.)r8r9z[api key provided with local uri. The data will be stored locallyname remote_tablezj`table` has to be a lancedb.db.LanceTable or lancedb.remote.table.RemoteTable object.T) set_default)%r _embedding _vector_key_id_key _text_keyosgetenvr8r9r:r<override_relevance_score_fnr@ _fts_index isinstance rerankersReranker _reranker ValueErrorr startswithdbLanceDBConnection _connection LanceTableconnectwarningswarnremoter; RemoteTable_tablehasattrrE _table_nameAssertionError get_table)selfr.r0r2r4r5r6r7r8r9r:r;r<r=r>r@rs r__init__zLanceDB.__init__:s&y))#% !@G2 w<")O"<" ? ? )D    S'**?$@ A A 8  |##*??3#7#7  c3'' ~~g.. +2??dk,;,,((,3??3+?+?( =   !GJ17>3G3ST$ ")%"8"8LEJJn   "    @  ...FFDKKKs AII/Fresultsrscoreboolrcjj}d|vrdn d|vrdnd|s)fdttDSr,|r,fdttDSdSdS)N _distance_relevance_scorecg|]T}tj|d|USmetadata) page_contentrnr rKas_py)ridxrfrds rr"z+LanceDB.results_to_docs..sn  !(!8!=!C!C!E!E$Z05;;==rcg|]t}tj|d||fuSrmrp)rrrrf score_colrds rr"z+LanceDB.results_to_docs..s    %,T^%.s&777Ac$*,,''777rr)rnNdata)r:)rHembed_documentslist enumerateappendrIrJrKrcrX create_tablerar_r8addr:rO) rdr~rrrdocsr}rrr*r0rntbls r add_textszLanceDB.add_textss6$777777_44T%[[AA "5))  IC"3I)2Hy~~s3x8HH KK$iL#c(ND     nn ;"//0@t/LLCDKK|#49----  rrErGOptional[bool]c||r||_|j}n |}n|j} |j|S#t$rYdSwxYw)a  Fetches a table object from the database. Args: name (str, optional): The name of the table to fetch. Defaults to None and fetches current table object. set_default (bool, optional): Sets fetched table as the default table. Defaults to False. Returns: Any: The fetched table object. Raises: ValueError: If the specified table is not found in the database. N)rarX open_table Exception)rdrErG_names rrczLanceDB.get_tablesq&   #' ($E #..u55 5   44 s9 AA`L2col_name vector_colnum_partitions Optional[int]num_sub_vectorsindex_cache_sizemetricNonec||}|r||||||dS|r||dStd)aO Create a scalar(for non-vector cols) or a vector index on a table. Make sure your vector column has enough data before creating an index on it. Args: vector_col: Provide if you want to create index on a vector column. col_name: Provide if you want to create index on a non-vector column. metric: Provide the metric to use for vector index. Defaults to 'L2' choice of metrics: 'L2', 'dot', 'cosine' num_partitions: Number of partitions to use for the index. Defaults to 256. num_sub_vectors: Number of sub-vectors to use for the index. Defaults to 96. index_cache_size: Size of the index cache. Defaults to None. name: Name of the table to create index on. Defaults to None. Returns: None )rvector_column_namerrrz%Provide either vector_col or col_nameN)rc create_indexcreate_scalar_indexrT) rdrrrrrrrErs rrzLanceDB.create_indexs6nnT""  F   #-- /!1       F  # #H - - - - -DEE Errct|d5}tj|dcdddS#1swxYwYdS)z!Get base64 string from image URI.rbzutf-8N)openbase64 b64encodereaddecode)rdr2 image_files r encode_imagezLanceDB.encode_image.s #t__ G #JOO$5$566==gFF G G G G G G G G G G G G G G G G G Gs9AAAurisc @}fd|D}| d|D}d}j1tjdrj|}nt dg}t |D]Q\} } |r|| n d|| i} |j| j|| j || d| iR|)j j | }|_ n|||S) asRun more images through the embeddings and add to the vectorstore. Args: uris List[str]: File path to the image. metadatas (Optional[List[dict]], optional): Optional list of metadatas. ids (Optional[List[str]], optional): Optional list of IDs. Returns: List[str]: List of IDs of the added images. c<g|]}|S))r2)r)rr2rds rr"z&LanceDB.add_images..Gs*@@@CT&&3&//@@@rNcNg|]"}ttj#Srrrs rr"z&LanceDB.add_images..Js&3333tz||$$333r embed_image)rzEembedding object should be provided and must have embed_image method.r)rnr)rcrHr`rrTrrrIrJrKrXrrar_r) rdrrrrr b64_textsr}rrrembrns ` r add_imageszLanceDB.add_images3sW"nnA@@@4@@@ ;33d333C ? &74?M+R+R &44$4??JJW !*--  HC)2Hy~~s3x8HH KK$cL#c(NIcN      ;"//0@t/LLCDKK GGDMMM rqueryr rc ||j}||}t|trt |}|dd}|dd}|dx} rX|||j||  ||} nD|||j| ||} |dkr"|j | |j | } t| d krtjd | S) N prefilterF query_typer(metrics)rr)rhybrid)r=rzNo results found for the query.)r@rcrPdictr%getsearchrIrwhererSrerankto_arrowrxr[r\) rdrr rrErrrrr lance_queryrs r_queryzLanceDB._querygs_ 9 AnnT"" fd # # -$V,,FJJ{E22 ZZ h77 jj++ +7  4;K LLqv33 K 4;K LLqv33   ! !dn&@     7 7 7##%% t99>> M; < < < rCallable[[float], float]c|jr|jS|jdkr|jS|jdkr|jS|jdkr|jSt d|jd)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. cosiner-ipzANo supported normalization function for distance metric of type: z=.Consider providing relevance_score_fn to Chroma constructor.)rNr<_cosine_relevance_score_fn_euclidean_relevance_score_fn%_max_inner_product_relevance_score_fnrTr|s r_select_relevance_score_fnz"LanceDB._select_relevance_score_fns  + 43 3 =H $ $2 2 ]d " "5 5 ]d " "= =O15OOO r List[float]Optional[Dict[str, str]]c ||j}|j||f||d|}|||ddS)zD Return documents most similar to the query vector. NrrErgFrg)r@rrzpop)rdr0r rrErress rsimilarity_search_by_vectorz#LanceDB.similarity_search_by_vectorsY 9 Adk)QKvDKKFKK##Cvzz'5/I/I#JJJrc |||j}||j||fddi|}fd|DS)zZ Return documents most similar to the query vector with relevance scores. NrgTcJg|]\}}|t|f Sr)float)rdocrgr>s rr"zMLanceDB.similarity_search_by_vector_with_relevance_scores..sA   8BUS$$U5\\22 3   r)r@rr)rdr0r rrErdocs_and_scoresr>s @r1similarity_search_by_vector_with_relevance_scoresz9LanceDB.similarity_search_by_vector_with_relevance_scoress| 9 A!<<>>:$: q  $ (.      FU    rc ||j}|dd}|dd}|dd}|jtd|dks|d kr|j|j||}||jd |_|d kr|j |} | |f} n|} |j | |f||d |} | | | Std |j |} |j | |fd|i|} | | | S)zAReturn documents most similar to the query with relevance scores.NrgTrErr(z4search needs an emmbedding function to be specified.ftsr)replacerrz?Full text/ Hybrid search is not supported in LanceDB Cloud yet.r) r@rrHrTr8rOrccreate_fts_indexrK embed_queryrrzNotImplementedError) rdrr rrrgrErrr0rrs rsimilarity_search_with_scorez$LanceDB.similarity_search_with_scores 9 A 7D))zz&$''ZZ h77 ? "STT T   *"8"8|#(?nnT**"%"6"6t~t"6"T"T)) $ ; ;E B BI'/FF"F!dk&!PFPPPP++Cu+===)U33E::I$+iDD6DVDDC''5'99 9rrList[Document]c .|jd|||||dd|}|S)apReturn documents most similar to the query Args: query: String to query the vectorstore with. k: Number of documents to return. filter (Optional[Dict]): Optional filter arguments sql_filter(Optional[string]): SQL filter to apply to the query. prefilter(Optional[bool]): Whether to apply the filter prior to the vector search. Raises: ValueError: If the specified table is not found in the database. Returns: List of documents most similar to the query. F)rr rErrrgr)r)rdrr rErrrrs rsimilarity_searchzLanceDB.similarity_searchs>00d/ 14Cu  PV   r?fetch_k lambda_multrc ||j}|jtd|j|}||||||}|S)a?Return docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. NzBFor MMR search, you must specify an embedding function oncreation.)rr)r@rHrTr'max_marginal_relevance_search_by_vector) rdrr rrrrr0rs rmax_marginal_relevance_searchz%LanceDB.max_marginal_relevance_searchst4 9 A ? "W O//66 ;;  # <   rc * |jd|||d|}ttj|tj|d|p|j| ||} fdt|D} | S)aHReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. )rr r)dtyper()r rc"g|] \}}|v | Srr)rir mmr_selecteds rr"zCLanceDB.max_marginal_relevance_search_by_vector..hs'UUU$!Q1 CTCTACTCTCTrr) rrnparrayfloat32 to_pylistr@rzr) rdr0r rrrrrf candidatesselected_resultsrs @rrz/LanceDB.max_marginal_relevance_search_by_vector>s6$+       2 HYbj 1 1 1 H  ' ' ) )o4:#    ))'22 UUUU)J*?*?UUUrcls Type[LanceDB]r c ftd||||||| | | | | |d |}||||S)N) r.r0r4r5r6r7r8r9r:r<r=r>)rr)r'r)rr~r0rr.r4r5r6r7r8r9r:r<r=r>rinstances r from_textszLanceDB.from_textsksk& !!!1     5I666r delete_all drop_columnsc ||}|r||dS|r=|dd|dS|r-|jt d||dS|r|ddStd)a Allows deleting rows by filtering, by ids or drop columns from the table. Args: filter: Provide a string SQL expression - "{col} {operation} {value}". ids: Provide list of ids to delete from the table. drop_columns: Provide list of columns to drop from the table. delete_all: If True, delete all rows from the table. z id in ('{}'),Nz;Column operations currently not supported in LanceDB Cloud.truez6Provide either filter, ids, drop_columns or delete_all)rcdeleteformatr#r8rrrT)rdrrrrrErrs rrzLanceDB.deletes$nnT""  W JJv       W JJ~,,SXXc]];; < < < < <  W|')Q  .....  W JJv     UVV Vr)r.r/r0r1r2r3r4r3r5r3r6r3r7r3r8r3r9r3r:r3r;r/r<r3r=r/r>r?r@rA)F)rfrrgrhrr)rr1)NN) r~rrrrrrrrr)NF)rEr3rGrrr)NNrrNrN)rr3rr3rrrrrrrr3rEr3rr)r2rrr) rrrrrrrrrr)NNN) rrr rrr/rEr3rrrr)rr) r0rr rrrrEr3rrrr) rrr rrrrrrr)NNNF)rrr rrEr3rr/rrrrrr)NrrN)rrr rrrArrrrrrrr)r0rr rrrArrrrrrrr) NNr(r)r*r+NNr,r-NN)"rrr~rr0r rrr.r/r4r3r5r3r6r3r7r3r8r3r9r3r:r3r<r3r=r/r>r?rrrr')NNNNN)rrrrrr3rrrEr3rrrr)__name__ __module__ __qualname____doc__ DEFAULT_Krerzpropertyr}rrcrrrrrrrrrrr classmethodrrrrrr'r's:%)*.+$, $"($1!% $)#"&"&AE!UGUGUGUGUGn<X +/#' .....bINF#'$((+)+*. $"(F(F(F(F(FTGGGG+/#' 22222n $" $$$$$L8 +/" KKKKK( +/"      2 +/ &:&:&:&:&:V " $# @  +/ *****^  +/ + + + + + Z +/$($, $"($1!% $)"&"&AE####[#N$(%) $,0" !W!W!W!W!W!W!Wrr')rr)rrrr) __future__rrrLrr[typingrrrrrr r numpyrlangchain_core.documentsr langchain_core.embeddingsr langchain_core.utilsrlangchain_core.vectorstoresr&langchain_community.vectorstores.utilsrrrr%r'rrrrsP"""""" FFFFFFFFFFFFFFFFFF------000000------333333MMMMMM #### EEEE T WT WT WT WT WkT WT WT WT WT Wr