NglUddlmZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZddlZddlmZddlmZddlmZddlmZe rddl Z ed Z!e"ee!Z#d e$d <ed Z%e"ee%Z&d e$d<dZ'dZ(dZ)e)dzZ*gdZ+gdZ,ddigZ-ej.e/Z0djdZ1dkd!Z2dld#Z3dmdnd*Z4Gd+d,eZ5 dodpd9Z6 dqdrdCZ7dsdHZ8dIgdfdtdLZ9dudOZ:dvdQZ;dwdUZdzd`Z? d{d|ddZ@d}dfZAd~diZBdS)) annotationsN)deepcopy) TYPE_CHECKINGAnyCallableDictIterableListLiteralOptionalSizedTupleTypeUnionget_args)Document) Embeddings) VectorStore)maximal_marginal_relevance)L2IPzList[DISTANCE_METRICS]AVAILABLE_DISTANCE_METRICS) TileDBDense TileDBSparse FaissFlat FaissIVFFlatFlinngz List[ENGINES]AVAILABLE_ENGINES langchain ) _distanceidcontent)r#r%blobzMissing propertyxryx_namestry_namereturnNonect|trot|trZt|t|kr:t|d|d|dt|d|dt| dS)z Check that sizes of two variables are the same Args: x: Variable to compare y: Variable to compare x_name: Name for variable x y_name: Name for variable y z and z% expected to be equal length but len(z)=z and len(N) isinstancer len ValueError)r'r(r)r+s a/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/vectorstores/vdms.py_len_check_if_sizedr3>s!U 1e 4 4 Q3q669I9I A AF A A A A VV A A.4 A A8;A A A    FresultsList[Document]c4dt|DS)Ncg|]\}}|Sr9.0doc_s r2 z$_results_to_docs..Qs C C CFCC C C Cr4)_results_to_docs_and_scores)r5s r2_results_to_docsr@Ps C C9'BB C C CCr4List[Tuple[Document, float]]cg} |d\}}t|dkrd|dvrd|ddvr|ddd}|D]u}t|dd}|d}tD] }||vr||= d|D} |t || |fvn4#t $r'} td | Yd} ~ nd} ~ wwxYw|S) NrFindDescriptorentitiesr# r%c,i|]\}}|tv||Sr9)INVALID_METADATA_VALUE)r;mkeymvals r2 z/_results_to_docs_and_scores..es4"d#999$999r4) page_contentmetadataz2No results returned. Error while parsing results: ) r0roundINVALID_DOC_METADATA_KEYSitemsappendr Exceptionloggerwarning) r5 final_res responsesblobsresult_entitiesentdistance txt_contentsppropses r2r?r?Ts`IQ"1: 5  NNQ   IaL00il+;<<<'l+;(!% 0)!-! ! r4c|jSN)rlrs r2 embeddingszVDMS.embeddingss ~r4texts List[str]List[List[float]]ct|jtr|j|Sd}|dz }t |)Nz*Must provide `embedding` which is expectedz to be an Embeddings object)r/rlrembed_documentsr1)rrp_strs r2_embed_documentszVDMS._embed_documentssJ dnj 1 1 $>11%88 8@E 2 2EU## #r4pathskwargsrc |j)t|jdr|jjdd|i|Std)N embed_videorz:Must provide `embedding` which has attribute `embed_video`r9)rlhasattrrr1)rrrs r2 _embed_videozVDMS._embed_videosP > %'$.-*P*P %-4>-DDEDVDD DL r4urisc|j0t|jdr|j|Std)N embed_imagerz:Must provide `embedding` which has attribute `embed_image`)rlrrr1)rrs r2 _embed_imagezVDMS._embed_imagesI > %'$.-*P*P %>--4-88 8L r4text List[float]ct|jtr|j|St d)NzEMust provide `embedding` which is expected to be an Embeddings object)r/rlr embed_queryr1)rrs r2 _embed_queryzVDMS._embed_querysC dnj 1 1 >--d33 3. r4Callable[[float], float]c|j|jS|jdvrdStd|jd)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. N)ipl2c|Srr9)r's r2z1VDMS._select_relevance_score_fn..sQr4z=No supported normalization function for distance_strategy of z;.Consider providing relevance_score_fn to VDMS constructor.)r}rnlowerr1rs r2_select_relevance_score_fnzVDMS._select_relevance_score_fnsj  + 73 3  ! ' ' ) )\ 9 9; M-1-CMMM r4querykrbfetch_kfilterOptional[Dict[str, Any]]rAc |jd|d<|jd||||d|}g}|D]N\}} |j||| f$|||| fO|S)z?Return docs and their similarity scores on a scale from 0 to 1.NTnormalize_distance)rrrrr9)r}similarity_search_with_scorerP) rrrrrrdocs_and_scoresdocs_and_rel_scoresr<scores r2(_similarity_search_with_relevance_scoresz-VDMS._similarity_search_with_relevance_scoress  + 3+/F' (;$;       *,)  JC/7#**C<8888#**$::5AAB#"r4r metadatas1Optional[Union[List[None], List[Dict[str, Any]]]]idsOptional[List[str]]r ct||dd||n d|D}t||dd||n d|D}t||ddg}g}g}t||||D]e\} } } } ||| | | | \} }|?|| |||| f|||\}}|S)Nrrcg|]}dSrr9r;r=s r2r>zVDMS.add..-szVDMS.add..0&*L*L*L3tz||+<+<*L*L*Lr4rrLrldocumentr$)r3zip_VDMS__get_add_queryrP_VDMS__run_vdms_query)rrmrrrr all_queries all_blobs inserted_idsmetaembr<r$rr&responseresponse_arrays r2addzVDMS.add#s= E:w EEE!*!6II.s2!!!*(hrl!!!r4rCrz[!] Embedding with id (z) exists in DB;z#Therefore, skipped and not insertedz Skipped values are: )Nr% AddDescriptorr) _check_descriptor_exists_by_idrzrOprintupdatekeysrrPrembedding2bytes) rrmrLrlrr$r\ id_existsr skipped_valuepstrrr&s r2__get_add_queryzVDMS.__get_add_querys{ :$&EE2JE= or   Iu #!!.34D.E%/egg !!! EDDD==d >}>>???d{"  # LL " " " : % %'E)  5 5A222*11!444      y))    r4F unique_entityOptional[bool]deletionct|||}||g\}}t|dkr)t|dd}nt t }|S)N)rrr,)_find_property_entityrr0 _bytes2strsplitrDEFAULT_PROPERTIES)rrmrr find_queryr response_blobrs r2__get_propertieszVDMS.__get_propertiess + =8   #'"7"7 "E"E- }   ! !$.}Q/?$@$@$F$Fs$K$K ! !$,-?$@$@ !$$r4r List[Dict]rOptional[List]print_last_responseTuple[Any, Any]c|j||\}}t||}|r|j||fSr)rzr_check_valid_responser)rrrrrrr=s r2__run_vdms_queryzVDMS.__run_vdms_querysV $(<#5#5k9#M#M . !+x 8 8  / L , , . . .''r4 documentsc \t||ddt||dd||n d|D}t||dd||}g}t||||D]\}} } } d|ji} dd d gi} | d | g| d <t d |ddddd| | }||g\}}|||| | | \}}|0||g|g\}}|| ||||jdS)z Updates (find, delete, add) a collection based on id. If more than one collection returned with id, error occuers rr rNcg|]}dSrr9rs r2r>z!VDMS.__update..s>>,0III(=#!)#@)))% X #33K(LL !!! 1 0 JIr4TOptional[List[dict]] batch_sizeadd_pathc Vfd|D}|r#|r!t|D]\}} | ||d<n |rg}|D]} |d| i||n d|D}|} | d|D}n d|D}jd || |||d||S) aRun more images through the embeddings and add to the vectorstore. Images are added as embeddings (AddDescriptor) instead of separate entity (AddImage) within VDMS to leverage similarity search capability Args: uris: List of paths to the images to add to the vectorstore. metadatas: Optional list of metadatas associated with the images. ids: Optional list of unique IDs. batch_size (int): Number of concurrent requests to send to the server. add_path: Bool to add image path as metadata Returns: List of ids from adding images into the vectorstore. c<g|]}|S)) image_path) encode_image)r;urirs r2r>z#VDMS.add_images..Cs*GGG3T&&#&66GGGr4r#NcNg|]"}ttj#Sr9rrs r2r>z#VDMS.add_images..Ns&*K*K*K3tz||+<+<*K*K*Kr4rcg|]}iSr9r9rs r2r>z#VDMS.add_images..Ts******r4c,g|]}t|Sr9_validate_vdms_propertiesr;ms r2r>z#VDMS.add_images..V!III!2155IIIr4rrrrrr9) enumeraterPradd_from) rrrrrr r b64_textsmidxr%rs ` r2 add_imageszVDMS.add_images*s92HGGG$GGG  6  6&t__ 4 4 c03 $ -- 4  6I 6 6  ,!45555_cc*K*Kd*K*K*K&&D&11  **T***IIIIyIIII  !!        r4rc ,| d|D}|r#|r!t|D]\}} | ||d<n |rg}|D]} |d| i||n d|D}|jdd|i|} | d|D}|jd|| |||d||S) aRun videos through the embeddings and add to the vectorstore. Videos are added as embeddings (AddDescriptor) instead of separate entity (AddVideo) within VDMS to leverage similarity search capability Args: paths: List of paths to the videos to add to the vectorstore. metadatas: Optional list of text associated with the videos. metadatas: Optional list of metadatas associated with the videos. ids: Optional list of unique IDs. batch_size (int): Number of concurrent requests to send to the server. add_path: Bool to add video path as metadata Returns: List of ids from adding videos into the vectorstore. Ncg|]}dS)rr9rs r2r>z#VDMS.add_videos..}s'''AR'''r4 video_pathcNg|]"}ttj#Sr9rrs r2r>z#VDMS.add_videos..rr4rcg|]}iSr9r9rs r2r>z#VDMS.add_videos..++++++r4r.r9)r/rPrr0) rrrrrrr rr2pathrs r2 add_videoszVDMS.add_videosbs*4 ='''''E  7  7'.. 5 5 d04 $ -- 5  7I 7 7  ,!56666_cc*L*Le*L*L*L'T&==U=f==  ++U+++I  !!        r4 Iterable[str]c t|}| d|D}||}| d|D}n d|D}|jd|||||d|}|S)aRun more texts through the embeddings and add to the vectorstore. Args: texts: List of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. batch_size (int): Number of concurrent requests to send to the server. Returns: List of ids from adding the texts into the vectorstore. NcNg|]"}ttj#Sr9rrs r2r>z"VDMS.add_texts..&4443tz||$$444r4cg|]}iSr9r9rs r2r>z"VDMS.add_texts..r9r4c,g|]}t|Sr9r)r+s r2r>z"VDMS.add_texts..r-r4r.r9)rrr0)rrrrrrrrs r2 add_textszVDMS.add_textss(U  ;44e444C**511  ++U+++IIIIyIIII$t} !!      r4c ||j}g}tdt||D]} t | |zt|} || | } || | } || | } |r || | }||j| | || }||||j||j|S)Nr)rrrr) rrranger0minrextendrr)rrrrrrrrr start_idxend_idx batch_textsbatch_embedding_vectors batch_idsbatch_metadatasresults r2r0z VDMS.add_froms**4+@AA "$ q#e**j99 ( (I)j0#e**==G ' 12K&071B&C #Ig-.I ?"+Ig,=">XX%2!) F    ' ' ' '    !:t/I   r4Union[int, None]c@|jstd|jtvrtd|jt vrtd|jtd|||_n|j=t|jdr(t| d|_n|jt|jdst|jdr^t|jd r: |jj j j|_n,#t$rtd wxYwtd ||}t|d r|j|dS||_dS) Nz_VDMS client must be connected to a VDMS server.Please use VDMS_Client to establish a connectionz-distance_strategy must be either 'L2' or 'IP'z]engine must be either 'TileDBDense', 'TileDBSparse', 'FaissFlat', 'FaissIVFFlat', or 'Flinng'Must provide embedding functionrzThis is a sample sentence.rrmodelz>Embedding dimension needed. Please define embedding_dimensionsr)rz is_connectedr1rnrr{rrlrrr0rrQtoken_embedding embedding_dimrrrF)rrmrq current_propss r2r|zVDMS._check_required_inputss|((** E   !)C C CLMM M  (0A A A=  > !>?? ?  +';D $ $ ^ 'GDNM,R,R ''*!!">??((D $ $^ ' DNM 2 2(t~}55(t~w// ,<J,,"$X !T --o>> 40 1 1 B  & - -m < < < < <4AD & & &s D**Ec g}g}ddgd}td|dddddd| }|||||\}}|dddS)Nrr$)countrrCrrreturned)rrPr)rrmrrr5rrrs r2rWz VDMS.count s!# ! //       5!!!#'#8#8i#P#P .{+,Z88r4 base64_imagebytesc*tj|Sr)base64 b64decode)rrYs r2 decode_imagezVDMS.decode_image6s ---r4Optional[Dict]c H||n|j}||||S)zDelete by ID. These are the IDs in the vectorstore. Args: ids: List of ids to delete. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. N)rr)r _VDMS__delete)rrrmrrnames r2deletez VDMS.delete9s- #2"=4CX}}Ts }DDDr4setnamer5 normalize(Tuple[List[Dict[str, Any]], List, float]cd}d}t||||}||g|\} } |r*|| dvr | d|ddd}| | |fS)NrrC)rr5rrDrr#)rr) rrdrr5rremax_dist command_strrrrs r2get_k_candidateszVDMS.get_k_candidatesLs&        $(#8#8%)#L#L .  M 33{;/ ;B? LH11r4rirOptional[dict]query_embeddingOptional[List[float]]r!Tuple[List[Dict[str, Any]], List]c g} t|} | | | | ||||| |\} } } n|ddgi}n0d|vrdg|d<n%d|dvr|ddt||||}||g\} } || dvr7| d|ddkrd| d|dD}nggfS||||| |\} } } || dvs"|| dvr| d|ddkrggfSg}| d|dD]6}|d|vr||t ||krn7|| d|d<t || d|d<t ||krd }t ||rl| dtjfvrd n| } t| d|dD]6\}}|d | z |d <|d | d|d|d <7| | fS) N)rerr$rr5rrXcg|] }|d S)r$r9)r;rXs r2r>z0VDMS.get_descriptor_response..s)###"%CI###r4rDz4Returned items < k_neighbors; Try increasing fetch_kg?r#) rrPrjrrr0rnpinfr/)rrirdrrrr5rlrrr&rrrhrids_of_interest new_entitiesrXrent_idxs r2get_descriptor_responsezVDMS.get_descriptor_responsecs " //     T " " "  151F1FgyDV2G22 .Hnhh!D6*w&&#'&WV_,,&&t,,,$' E (,'<'K MPN K(4W=kJJ''r4r#ct|d5}|}tj|dcdddS#1swxYwYdS)Nrbzutf-8)openreadr\ b64encodedecode)rr#fr&s r2r$zVDMS.encode_images *d # # :q6688D#D))0099 : : : : : : : : : : : : : : : : : :s;AA Acls Type[VDMS]r6c r|d}||d|Dd|D||||S)aCreate a VDMS vectorstore from a list of documents. Args: collection_name (str): Name of the collection to create. documents (List[Document]): List of documents to add to vectorstore. embedding (Embeddings): Embedding function. Defaults to None. ids (Optional[List[str]]): List of document IDs. Defaults to None. batch_size (int): Number of concurrent requests to send to the server. Returns: VDMS: VDMS vectorstore. rhcg|] }|j Sr9rKr;r<s r2r>z'VDMS.from_documents..s9993#999r4cg|] }|j Sr9)rLrs r2r>z'VDMS.from_documents..s999s|999r4)rhrrrlrrrm) from_texts)rr rlrrrmrrhs r2from_documentszVDMS.from_documentssZ,#8,~~99y99999y999!+   r4c |d}||||} | d|D}| ||||| S)aHCreate a VDMS vectorstore from a raw documents. Args: texts (List[str]): List of texts to add to the collection. embedding (Embeddings): Embedding function. Defaults to None. metadatas (Optional[List[dict]]): List of metadatas. Defaults to None. ids (Optional[List[str]]): List of document IDs. Defaults to None. batch_size (int): Number of concurrent requests to send to the server. collection_name (str): Name of the collection to create. Returns: VDMS: VDMS vectorstore. rh)rmrlrhNcNg|]"}ttj#Sr9rrs r2r>z#VDMS.from_texts..r?r4)rrrr)rB) rrrlrrrrmrrhvdms_collections r2rzVDMS.from_textssx0#8,#+    ;44e444C!!! "   r4limitincludecg}g}ddi}|||d<d|vr||}||d<d|vrd|d <td |d|| } || |||\} } | | fS) aGets the collection. Get embeddings and their associated data from the data store. If no constraints provided returns all embeddings up to limit. Args: constraints: A dict used to filter results by. E.g. `{"color" : ["==", "red"], "price": [">", 4.00]}`. Optional. limit: The number of documents to return. Optional. include: A list of what to include in the results. Can contain `"embeddings"`, `"metadatas"`, `"documents"`. Ids are always included. Defaults to `["metadatas", "documents"]`. Optional. rWrNrrLrrTr&rC)rrr5)rrrPr) rrmrrrrrr5rrrrs r2getzVDMS.gets("$ ! #*B-  $GG   $($9$9/$J$J !3GFO 7 " ""GFO  #     5!!!#'#8#8i#P#P .''r4g? lambda_multfloatOptional[Dict[str, List]]c |jtdtj|s+t |jdr||}ntj|r3t |jdr||gd}nqtj|r3t |jdr||gd}nd |d }|d z }|d z }t|| ||||| } | SaRReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query (str): Query to look up. Text or path for image or video. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. NzBFor MMR search, you must specify an embedding function oncreation.rrrrrr(Could not generate embedding for query ''.9If using path for image or video, verify embedding model 6has callable functions 'embed_image' or 'embed_video'.)rr) rlr1osr:isfilerrrr'max_marginal_relevance_search_by_vector) rrrrrrrembedding_vector error_msgdocss r2max_marginal_relevance_searchz"VDMS.max_marginal_relevance_search2s[4 > !W  w~~e$$ ()O)O (#0077   W^^E " " (wt~}'M'M (#00ug0>>qA   W^^E " " (wt~}'M'M (#00w0??B  L5LLLI T TI Q QIY'' ';;  # <   r4c h ||g||gd}t|dddkrgSd|ddD}ttj|tj||| t |} fdt| D} | S) HReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. rr  distancesrquery_embeddings n_resultsrrrrcFg|]}tt|Sr9r_bytes2embeddingr;rMs r2r>z@VDMS.max_marginal_relevance_search_by_vector..639%f--..r4dtyperrc"g|] \}}|v | Sr9r9)r;ir mmr_selecteds r2r>z@VDMS.max_marginal_relevance_search_by_vector..s.   aqL7H7H7H7H7Hr4)query_collection_embeddingsr0rrrarrayfloat32r@r/ rrlrrrrrr5embedding_list candidatesselected_resultsrs @r2rz,VDMS.max_marginal_relevance_search_by_vectorhs422'[III 3   wqz!}   " "I=DQZ]N6"*555' L*'22J    ' 33    $ #r4c |jtdtj|s+t |jdr||}ntj|r3t |jdr||gd}nqtj|r3t |jdr||gd}nd |d }|d z }|d z }t|| ||||| } | Sr) rlr1rr:rrrrr2max_marginal_relevance_search_with_score_by_vector) rrrrrrrrlrrs r2(max_marginal_relevance_search_with_scorez-VDMS.max_marginal_relevance_search_with_scoresV4 > !W w~~e$$ ()O)O ())%00II W^^E " " (wt~}'M'M ())w)77:II W^^E " " (wt~}'M'M ()))88;IIL5LLLI T TI Q QIY'' 'FF  # G   r4c h ||g||gd}t|dddkrgSd|ddD}ttj|tj||| t |} fdt| D} | S) rrrrrcFg|]}tt|Sr9rrs r2r>zKVDMS.max_marginal_relevance_search_with_score_by_vector..rr4rrc,g|]\}\}}|v ||fSr9r9)r;rrsrs r2r>zKVDMS.max_marginal_relevance_search_with_score_by_vector..s6   $1fq!lARARAARARARr4)rr0rrrrrr?r/rs @r2rz7VDMS.max_marginal_relevance_search_with_score_by_vectors422'[III 3   wqz!}   " "I=DQZ]N6"*555' L5W==J    (1*(=(=    $ #r4rOptional[List[List[float]]]r!List[Tuple[Dict[str, Any], List]]c g} ||j}|| S|ddg} |d| vr |jd| vd}|D]9} |d||||||| \} } | | | g:| S)Nrrr)rr&rC)rrrr5rrl)rrrrwrP)rrrmrrrr5rr all_responsesrqembrrs r2rz VDMS.query_collection_embeddings s$&  ""3O  # **Y 66 ?{g552$/G % = =D'+'C'C %"#5 $(D ( ( $Hn  (N!; < < < <r4c >|j|f|||d|}d|DS)aRun similarity search with VDMS. Args: query (str): Query to look up. Text or path for image or video. k (int): Number of results to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List[Document]: List of documents most similar to the query text. )rrrcg|]\}}|Sr9r9r:s r2r>z*VDMS.similarity_search..Is222Q222r4)r)rrrrrrrs r2similarity_searchzVDMS.similarity_search3sJ&<$;    :@  32/2222r4c F|jd|g|||d|}t|S)aReturn docs most similar to embedding vector. Args: embedding (List[float]): Embedding to look up documents similar to. k (int): Number of Documents to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents most similar to the query vector. rrrrr9)rr@rrlrrrrr5s r2similarity_search_by_vectorz VDMS.similarity_search_by_vectorKsJ"3$2 '[       (((r4c |jtdtj|s+t |jdr||}ntj|r3t |jdr||gd}nqtj|r3t |jdr||gd}nd |d }|d z }|d z }t||j d|g|||d |}t|S)aERun similarity search with VDMS with distance. Args: query (str): Query to look up. Text or path for image or video. k (int): Number of results to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List[Tuple[Document, float]]: List of documents most similar to the query text and cosine distance in float for each. Lower score represents more similarity. NrPrrrrrrrrrrrr9) rlr1rr:rrrrrrr?) rrrrrrrlrr5s r2rz!VDMS.similarity_search_with_scorefsd* > !>?? ?7>>%(( ,WT^]-S-S ,/3/@/@/G/G&& ,74>=+Q+Q ,"&"3"3%"3"A"A!"D&& ,74>=+Q+Q ,"&"3"35'"3"B"B1"EPuPPP XX UU  +++6d6"1!2   G+7333r4c F|jd|g|||d|}t|S)a6 Return docs most similar to embedding vector and similarity score. Args: embedding (List[float]): Embedding to look up documents similar to. k (int): Number of Documents to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List[Tuple[Document, float]]: List of documents most similar to the query text. Lower score represents more similarity. rr9)rr?rs r2&similarity_search_with_score_by_vectorz+VDMS.similarity_search_with_score_by_vectorsJ03$2 '[      +7333r4 document_idrc4|||g|gS)zUpdate a document in the collection. Args: document_id (str): ID of the document to update. document (Document): Document to update. )update_documents)rrmrrs r2update_documentzVDMS.update_documents $$_{mhZPPPr4cd|D}d|D}||}||||||dS)zUpdate a document in the collection. Args: ids (List[str]): List of ids of the document to update. documents (List[Document]): List of documents to update. cg|] }|j Sr9rr;rs r2r>z)VDMS.update_documents..s@@@(%@@@r4c6g|]}t|jSr9)r*rLrs r2r>z)VDMS.update_documents..s1   =E %h&7 8 8   r4)rrr N)r _VDMS__update)rrmrr rrLrs r2rzVDMS.update_documentssA@i@@@  IR   **400   !      r4)rhrcrlrrrmr*rnrsrortrprurqrvr,r-)r,rr)rrr,r)rrrrr,r)rrr,r)rr*r,r)r,r) rr*rrbrrbrrrrr,rA)NN) rmr*rrrrrrrrr,r )rr)rmr*rortryrsr,r*)rmr*rrrrr,r)NNNN) rmr*rLrrlrrrr$rr,rFF)rmr*rrrrr,r)rrrrrrr,r r) rmr*rrr rrrrrr,r-)rmr*rr rrr,r-)rrrrrrrrbr rrrr,r)NNNrT)rrrrrrrrrrbr rrrr,r) rr<rrrrrrbrrr,r)rrrrrrrrrrbrrr,r)rmr*rqrNr,r-)rmr*r,rb)rYr*r,rZ)NNN) rrrmrrr_rrr,r)NNF) rdr*rrvr5rrrrerr,rf)rir*rdr*rrbrrbrrkr5rrlrmrrr,rn)r#r*r,r*)rrr r6rlrrrrrrbrmr*rrr,rk)rrrrrlrrrrrrrrbrmr*rrr,rk) rmr*rr_rrvrrr,r )rr*rrbrrbrrrrrrr,r6)rlrrrbrrbrrrrrrr,r6)rr*rrbrrbrrrrrrr,rA)rlrrrbrrbrrrrrrr,rA)rrrmrrrbrrbrrr5rrrrrr,r) rr*rrbrrbrrrrr,r6) rlrrrbrrbrrrrr,r6) rr*rrbrrbrrrrr,rA) rlrrrbrrbrrrrr,rA)rmr*rr*rrr,r-)rmr*rrr r6r,r-)3__name__ __module__ __qualname____doc__DEFAULT_COLLECTION_NAMErpropertyrrrrrr DEFAULT_KDEFAULT_FETCH_Krrr~rarrrrrDEFAULT_INSERT_BATCH_SIZEr3r;rBr0r|rWr^rcrjrwr$ classmethodrrrrrrrrrrrrrrr9r4r2rkrks$$T+/6.2%AE.2      8X$$$$4&+/ #####DHL#' H&#' 0'+37 +/+/+/+/+/`#'.2"& 6 6 6 6 6 v).#( %%%%%&%'.3 ( ( ( ( (&HL 9 9 9 9 9 vMMMM.+/#'3#' 66666v&**.#'#'66666v+/#'3 '''''\+/3 #####J6B6B6B6Bp9999,.... $()-&* EEEEE.-1$($) 222226%&&*,015#(J(J(J(J(J(X:::: +/#'36     [  D+/*.#'36&&&&[&V'+#(\ 0(0(0(0(0(j& ,0 44444r& ,0 5$5$5$5$5$t& ,0 22222n& ,0 5$5$5$5$5$r9=)-"&.2/3#('''''X&,0 333336&,0 )))))<&,0 ,4,4,4,4,4b&,0 44444B Q Q Q Q      r4rkrirdrrrrvr\rkrrrDict[str, Dict[str, Any]]c d|i} d|vr|r|| d<||| d<|tvr|| d<d|vr||| d<d|vr|t|| d<d|vr|tvr|| d <d|vr|tvr|| d <|| i} | S) NsetAddr_ref propertiesrFindrrr5)rGrb) rirdrrr\rrrr5entityrs r2rrs$W-F w v ***$|  0v !8 #K 0 0} 4J!J!J +} 0F!F!F#y & !E Lr4Frbnum_dimsroryr_rrDict[str, Any]c |dkrHtd||fDr-||d} ||| d<||| d<||| d<|difvr|| d<||| d<n6|d krd |i} |r|| d <| difvr| | d <| | | d <ntd||| i} | S)Nrc3K|]}|duV dSrr9)r;vars r2 z%_add_descriptorset..s7114111111r4)rb dimensionsroryrrrrrrrr5zUnknown command: )allr1) rirbrroryrr\rrrr5rrs r2rrs&(((S11$((#3111..("" "   %F8   %F8  ? F6N r " "#(F<  !F6N + + +  .#-F< tRj ( ($/F= !   'F9 :[::;;; & !E Lr4rmrr Tuple[Dict[str, Any], bytes]ct|dkrd|nd}d}i}d|d<d|d<d |i}d |d <||d <||d<t|}i}|||<||fS) Nrrr AddEntityrclassTr&rbzqueryable propertiestyper%)r0join _str2bytes)rmrall_properties_str querytyperr\ byte_datars r2_add_entity_with_blobr8s695H5H15L5L.111RTIF"F7OF6N#_5E*E&M)E) F<-..IEE) ) r4findrr cg}g}gd}||vr5tdd||dkr't |d}||n|dkr>t ||\}}||||n{|dkrct |d }||t ||\}}||||||fS) N)rrrz"[!] Invalid type. Choices are : {}rrT)rrr)r)rr1formatrrrPr) rmrrrrrchoicesrrs r2rrNsq  KH'''G7**=DDSXXgEVEVWWXXXv%%%oTJJJ5!!!!      & &0.QQy5!!! """"      ) )%oEEE5!!!1.QQy5!!! """   r4r&rZc2tj|d}|S)Nrr)rr frombuffer)r&rs r2rrqs -I . . .C Jr4in_bytesc*|Sr)r})rs r2rrvs ??  r4rrrcNttd|DS)Nc@g|]}|D]}|Sr9)r)r;qrs r2r>z(_get_cmds_from_query..{s->>>1QVVXX>>Q>>>>r4)rr)rs r2_get_cmds_from_queryr zs' >> >>>?? @ @@r4 List[dict]rct|}ttotfd|D}|S)Nc3|K|]6}|dvo'dd|vod|ddkV7dS)rrXNr9)r;cmdrs r2rz(_check_valid_response..ss33  x{ - (1+c* * - QK Z (1 ,333333r4)r r/rany)rrcmd_list valid_ress ` r2r r ~s`#K00H8T**s3333 33300I r4rhr$Tuple[bool, Any]cdd|gi}td||dgdd}|g}||\}}t||}||fS)Nr$rrCr)rrWrp)rrr ) rhrdr$rfindDescriptorrresr=rs r2rrsu $$K$"-- N ""K \\+ & &FC%k377I n $$r4rlrUnion[bytes, None]cbd}|*tj|d}|}|S)zConvert embedding to bytes.Nrr)rrrtobytes)rlr&rs r2rrs5 Dhy 222{{}} Kr4rrrcd}i}d|d<|r||d<i}d|d<d|d<d g|d <||d <i}|rd d g|d<d |g|d<||d<i}|||<|S)N FindEntityrruniqueTr&rrWr%rr5rrrrbrr9)rmrrrrr5rrs r2rrs IF"F7O)(x GGFOGG kGFOF9"$K-$(!9 K 1K'F=EE) Lr4in_strc6t|Sr)r*encode)rs r2rrs ::f  r4rLrci}|D],\}}t|ts||t|<-|Sr)rOr/rr*)rL new_metadatakeyrs r2r*r*sM#%Lnn&&++ U%&& +%*LS " r4) r'rr(rr)r*r+r*r,r-)r5rr,r6)r5rr,rA)r^r_)r`r*rarbr,rc)NNNNNNN)rir*rdr*rrrrvr\rkrrkrrvrrkr5rkr,r) NNNNNNFNN)rir*rbr*rrvrorryrrrvr\r_rr_rrrr_r5r_r,r)rmr*rr r,r) rmr*rr*rr rrvr,r )r&rZr,r)rrZr,r*)rrr,r)rr rrr,r)rhrcrdr*r$r*r,r)rlrr,rr)rmr*rrrrr,r)rr*r,rZ)rLrr,r)C __future__rr\loggingrrcopyrtypingrrrrr r r r r rrrrnumpyrrlangchain_core.documentsrlangchain_core.embeddingsrlangchain_core.vectorstoresr&langchain_community.vectorstores.utilsrrersrr__annotations__rtrrrrrrrNrG getLoggerrrRr3r@r?rirkrrrrrrr r rrrrr*r9r4r2r-s"""""""   ------000000333333MMMMMMKKK 6:T((CS:T:T5U5UUUUU  $(4(9(9#:#:::::% a-333<<<,dB7  8 $ $    $DDDD>(N N N N N ;N N N n*  !%"&"#####R#   "&"44444n0 ! ! ! ! !F AAAA%%%%&%*$8r4