NgQddlmZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z mZddlZddlmZddlmZddlmZddlmZdd lmZmZdd lmZdd lmZddZ ddZ!GddeZ"dS)) annotationsN)Path)AnyCallableDictIterableListOptionalTuple)Document) Embeddings guard_import) VectorStore) AddableMixinDocstore)InMemoryDocstore)DistanceStrategyx np.ndarrayreturncx|tjtj|ddddz}|S)z!Normalize vectors to unit length.T)axiskeepdimsg-q=N)npcliplinalgnorm)rs b/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/vectorstores/scann.py normalizer!s3T::E4 H HHA Hrc tdS)z= Import `scann` if available, otherwise raise error. scannrr"r dependable_scann_importr&s   r"cBeZdZdZddejdfdFdZ dGdHd Z dGdId!Z dGdJd$Z dKdLd&Z dMdNd0Z dMdOd3Z dMdPd5Z dMdQd6Ze dRdSd8Ze dGdTd9Ze dGdUd;ZdVdWd?Ze dVdd@dXdBZdYdDZ dMdOdEZdS)ZScaNNa`ScaNN` vector store. To use, you should have the ``scann`` python package installed. Example: .. code-block:: python from langchain_community.embeddings import HuggingFaceEmbeddings from langchain_community.vectorstores import ScaNN model_name = "sentence-transformers/all-mpnet-base-v2" db = ScaNN.from_texts( ['foo', 'bar', 'barz', 'qux'], HuggingFaceEmbeddings(model_name=model_name)) db.similarity_search('foo?', k=1) NF embeddingr indexrdocstorerindex_to_docstore_idDict[int, str]relevance_score_fn"Optional[Callable[[float], float]] normalize_L2booldistance_strategyr scann_config Optional[str]c v||_||_||_||_||_||_||_||_dS)z%Initialize with necessary components.N)r)r*r+r,r2override_relevance_score_fn _normalize_L2 _scann_config) selfr)r*r+r,r.r0r2r3s r __init__zScaNN.__init__3sG#   $8!!2+=())r"texts Iterable[str] embeddingsIterable[List[float]] metadatasOptional[List[dict]]idsOptional[List[str]]kwargsr List[str]c t|jtstd|jdt d)NSIf trying to add texts, the underlying docstore should support adding items, which does notz(Updates are not available in ScaNN, yet.) isinstancer+r ValueErrorNotImplementedError)r9r;r=r?rArCs r __addz ScaNN.__addHsX$-66 @'+}@@@ ""LMMMr"c t|jt|}|j||f||d|S)alRun more texts through the embeddings and add to the vectorstore. Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. r?rA)r)embed_documentslist _ScaNN__add)r9r;r?rArCr=s r add_textszScaNN.add_textsWsB$^33DKK@@ tz%TycTTVTTTr"text_embeddings!Iterable[Tuple[str, List[float]]]c t|jtstd|jdt |\}}|j||f||d|S)aRun more texts through the embeddings and add to the vectorstore. Args: text_embeddings: Iterable pairs of string and embedding to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. Returns: List of ids from adding the texts into the vectorstore. rFrGrM)rHr+rrIziprP)r9rRr?rArCr;r=s r add_embeddingszScaNN.add_embeddingsls{$$-66 @'+}@@@   1ztz%TycTTVTTTr"Optional[bool]c  td)a3Delete by vector ID or other criteria. Args: ids: List of ids to delete. **kwargs: Other keyword arguments that subclasses might use. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. z*Deletions are not available in ScaNN, yet.)rJ)r9rArCs r deletez ScaNN.deletes""NOOOr" List[float]kintfilterOptional[Dict[str, Any]]fetch_kList[Tuple[Document, float]]c  tj|gtj}|jrt |}|j|||n|\}}g} t|dD]\} } | dkr |j| } |j | ttstd| d|od|D}tfd|Dr#| |d| f| |d| f|d F|jt&jt&jfvr t,jn t,j fd | D} | d|S) aReturn docs most similar to query. Args: embedding: Embedding vector to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, Any]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. **kwargs: kwargs to be passed to similarity search. Can include: score_threshold: Optional, a floating point value between 0 to 1 to filter the resulting set of retrieved docs Returns: List of documents most similar to the query text and L2 distance in float for each. Lower score represents more similarity. dtypeNrrzCould not find document for id z, got cJi|] \}}|t|ts|gn|!Sr%)rHrO).0keyvalues r z@ScaNN.similarity_search_with_score_by_vector..sC"U 5$(?(?J%Ur"c3VK|]#\}}j||vV$dSN)metadataget)rgrhridocs r z?ScaNN.similarity_search_with_score_by_vector..s<WW*#us|'',,5WWWWWWr"score_thresholdc6g|]\}}|||fSr%r%)rgro similaritycmprqs r z@ScaNN.similarity_search_with_score_by_vector..sD#C3z?33j!r")rarrayfloat32r7r!r*search_batched enumerater,r+searchrHr rIitemsallappendrnr2rMAX_INNER_PRODUCTJACCARDoperatorgele)r9r)r]r_rarCvectorindicesscoresdocsji_idrtrorqs @@@r &similarity_search_with_score_by_vectorz,ScaNN.similarity_search_with_score_by_vectors09+RZ888   'v&&F*33 AAW  gaj)) 1 1DAqBww+A.C-&&s++Cc8,, U !S3!S!Sc!S!STTT!&,llnnWWWW WWWWW5KKfQil 3444 S&)A,/0000 **%677  &)$68H8PQRR [  '+D BQBxr"querystrc ^|j|}|j||f||d|}|S)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of documents most similar to the query text with L2 distance in float. Lower score represents more similarity. r_ra)r) embed_queryr)r9rr]r_rarCr)rs r similarity_search_with_scorez"ScaNN.similarity_search_with_scoresV*N..u55 :t:          r"List[Document]c >|j||f||d|}d|DS)aReturn docs most similar to embedding vector. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the embedding. rcg|]\}}|Sr%r%rgro_s r ruz5ScaNN.similarity_search_by_vector..222Q222r")r)r9r)r]r_rarCdocs_and_scoress r similarity_search_by_vectorz!ScaNN.similarity_search_by_vectorsQ(F$E         32/2222r"c >|j||f||d|}d|DS)aReturn docs most similar to query. Args: query: Text to look up documents similar to. k: Number of Documents to return. Defaults to 4. filter: (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. fetch_k: (Optional[int]) Number of Documents to fetch before filtering. Defaults to 20. Returns: List of Documents most similar to the query. rcg|]\}}|Sr%r%rs r ruz+ScaNN.similarity_search..)rr")r)r9rr]r_rarCrs r similarity_searchzScaNN.similarity_searchsJ(<$; 1 #W  8>  32/2222r"List[List[float]]c `td}|dtj} |dd} t j|tj} |rt| } | |j | | } n| tj krA|j | dd } n@|j | dd } g} | d|D}t|D]5\}}|r||ni}| t!|| 6t#t|}t%|t%| kr/t't%|d t%| d t)t#t+|| }||| ||fd |i|S) Nr$r2r3rd dot_product squared_l2cNg|]"}ttj#Sr%)ruuiduuid4)rgrs r ruz ScaNN.__from..Qs&4443tz||$$444r") page_contentrmz ids provided for z, documents. Each document should have an id.r0)rrnrEUCLIDEAN_DISTANCErrvrwr!scann_ops_pybindcreate_searcherr~builderscore_brute_forcebuildryr}r dictlen ExceptionrrUvalues)clsr;r=r)r?rAr0rCr$r2r3rr* documentsrtextrm index_to_idr+s r __fromz ScaNN.__from+sGW%%"JJ !1!D  zz.$77 *BJ777  'v&&F  #*::6<PPEE $4$FFF*2261mLL&&((UWW*2261lKK&&((UWW  ;44e444C '' M MGAt'08y||bH   X4(KKK L L L L9S>>** {  s9~~ - -{##44s9~~444  $D[-?-?-A-A9)M)M$N$NOOs       &      r"c R||}|j|||f||d|S)aNConstruct ScaNN wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the ScaNN database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import ScaNN from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() scann = ScaNN.from_texts(texts, embeddings) rM)rN _ScaNN__from)rr;r)r?rArCr=s r from_textszScaNN.from_textsgsR4..u55 sz            r"List[Tuple[str, List[float]]]c Xd|D}d|D}|j|||f||d|S)aConstruct ScaNN wrapper from raw documents. This is a user friendly interface that: 1. Embeds documents. 2. Creates an in memory docstore 3. Initializes the ScaNN database This is intended to be a quick way to get started. Example: .. code-block:: python from langchain_community.vectorstores import ScaNN from langchain_community.embeddings import OpenAIEmbeddings embeddings = OpenAIEmbeddings() text_embeddings = embeddings.embed_documents(texts) text_embedding_pairs = list(zip(texts, text_embeddings)) scann = ScaNN.from_embeddings(text_embedding_pairs, embeddings) cg|] }|d S)rr%rgts r ruz)ScaNN.from_embeddings..s///!1///r"cg|] }|d S)rr%rs r ruz)ScaNN.from_embeddings..s444qad444r"rM)r)rrRr)r?rArCr;r=s r from_embeddingszScaNN.from_embeddingsse80////44O444 sz            r" folder_path index_nameNonect|}|d|z }|dd|jt |t |d|z d5}tj|j |j f|ddddS#1swxYwYdS)zSave ScaNN index, docstore, and index_to_docstore_id to disk. Args: folder_path: folder path to save index, docstore, and index_to_docstore_id to. {index_name}.scannrTexist_okparents{index_name}.pklwbN) rformatmkdirr* serializeropenpickledumpr+r,)r9rrpath scann_pathfs r save_localzScaNN.save_localsK  077:7NNN $555 S__---$+22j2III4 P P GTU K(ABA F F F G G G G G G G G G G G G G G G G G Gs"B==CC)allow_dangerous_deserializationrc |stdt|}|d|z }|ddt d}|jt|} t|d|z d5} tj | \} } d d d n #1swxYwY||| | | fi|S) aLoad ScaNN index, docstore, and index_to_docstore_id from disk. Args: folder_path: folder path to load index, docstore, and index_to_docstore_id from. embedding: Embeddings to use when generating queries index_name: for saving with a specific index file name allow_dangerous_deserialization: whether to allow deserialization of the data which involves loading a pickle file. Pickle files can be modified by malicious actors to deliver a malicious payload that results in execution of arbitrary code on your machine. aBThe de-serialization relies loading a pickle file. Pickle files can be modified to deliver a malicious payload that results in execution of arbitrary code on your machine.You will need to set `allow_dangerous_deserialization` to `True` to enable deserialization. If you do this, make sure that you trust the source of the data. For example, if you are loading a file that you created, and know that no one else has modified the file, then this is safe to do. Do not set this to `True` if you are loading a file from an untrusted source (e.g., some random site on the internet.).rrTrr$rrbN) rIrrrrr load_searcherrrrload) rrr)rrrCrrr$r*rr+r,s r load_localzScaNN.load_localsL./  "   K  077:7NNN $555W%%&44S__EE$+22j2III4 P P TU  $               s9eX/CNNvNNNs.CCCCallable[[float], float]c|j|jS|jtjkr|jS|jtjkr|jStd)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. NzJUnknown distance strategy, must be cosine, max_inner_product, or euclidean)r6r2rr~%_max_inner_product_relevance_score_fnr_euclidean_relevance_score_fnrI)r9s r _select_relevance_score_fnz ScaNN._select_relevance_score_fnse  + 73 3  !%5%G G G= =  #'7'J J J5 5  r"c  |dd |td|j|f|||d|}fd|D}  fd|D}|S)z?Return docs and their similarity scores on a scale from 0 to 1.rqNzLnormalize_score_fn must be provided to ScaNN constructor to normalize scores)r]r_rac0g|]\}}||fSr%r%)rgroscorer.s r ruzBScaNN._similarity_search_with_relevance_scores..,s;   1;eS$$U++ ,   r"c*g|]\}}|k ||fSr%r%)rgrorsrqs r ruzBScaNN._similarity_search_with_relevance_scores..0s7####C00j!000r")poprrIr) r9rr]r_rarCrdocs_and_rel_scoresr.rqs @@r (_similarity_search_with_relevance_scoresz.ScaNN._similarity_search_with_relevance_scoress!**%6==!<<>>  %9 <$;            ?N     &####':###  #"r")r)r r*rr+rr,r-r.r/r0r1r2rr3r4)NN) r;r<r=r>r?r@rArBrCrrrD) r;r<r?r@rArBrCrrrD) rRrSr?r@rArBrCrrrDrl)rArBrCrrrW)rZNr[) r)r\r]r^r_r`rar^rCrrrb) rrr]r^r_r`rar^rCrrrb) r)r\r]r^r_r`rar^rCrrr) rrr]r^r_r`rar^rCrrr)NNF)r;rDr=rr)r r?r@rArBr0r1rCrrr() r;rDr)r r?r@rArBrCrrr() rRrr)r r?r@rArBrCrrr()r*)rrrrrr) rrr)r rrrr1rCrrr()rr)__name__ __module__ __qualname____doc__rrr:rPrQrVrYrrrr classmethodrrrrrrrr%r"r r(r(!s.BF".>.Q&******2+/#' N N N N N$+/#' UUUUU0+/#' UUUUU8 P P P P P"+/ >>>>>F+/ D+/ 33333@+/ 333332 +/#'"9 9 9 9 [9 v +/#' ! ! ! ! [! F +/#' $ $ $ $ [$ LGGGGG$ " 3O 16 3O3O3O3O3O[3Oj8+/ "#"#"#"#"#"#"#r"r()rrrr)rr)# __future__rrrrpathlibrtypingrrrrr r r numpyrlangchain_core.documentsr langchain_core.embeddingsr langchain_core.utilsrlangchain_core.vectorstoresr!langchain_community.docstore.baserr&langchain_community.docstore.in_memoryr&langchain_community.vectorstores.utilsrr!r&r(r%r"r rsp"""""" GGGGGGGGGGGGGGGGGG------000000------333333DDDDDDDDCCCCCCCCCCCC    !!!!T#T#T#T#T#KT#T#T#T#T#r"