NgWLddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl ZddlmZddlmZddlmZddlmZddlmZd Zd Zd Zd Zd ZdZdZGddeZ dS)) annotationsN)AnyDictListOptionalSequenceTupleType)Document) Embeddings)UpsertResponse) VectorStore)overrideHNSWCS langchainilc_textuniqueidceZdZeedddejdfdEfd ZdFdZedGdHdZ edIdZ e dJdKd Z edLd"Z dMd&Z dNdOd,Ze dJdPd-Ze dQdRd3Ze dQdSd4Zee dGdTd;ZeedUd=ZedVd>ZedFd?ZdWd@ZdXdDZxZS)Y ApertureDBN embeddingsr descriptor_setstr dimensions Optional[int]engine Optional[str]metric log_levelint propertiesOptional[Dict]kwargsrreturnNonec  ddlm} m} n#t$rtdwxYwt jdi|t jt|_ |j |||_ ||_ ||_ ||_||_||_|)|j dt%d ddlm} m} n)#t$r|j dwxYw| |_| |j|_ |jn)#t.$r|j dwxYw|dS)aCreate a vectorstore backed by ApertureDB A single ApertureDB instance can support many vectorstores, distinguished by 'descriptor_set' name. The descriptor set is created if it does not exist. Different descriptor sets can use different engines and metrics, be supplied by different embedding models, and have different dimensions. See ApertureDB documentation on `AddDescriptorSet` https://docs.aperturedata.io/query_language/Reference/descriptor_commands/desc_set_commands/AddDescriptorSet for more information on the engine and metric options. Args: embeddings (Embeddings): Embeddings object descriptor_set (str, optional): Descriptor set name. Defaults to "langchain". dimensions (Optional[int], optional): Number of dimensions of the embeddings. Defaults to None. engine (str, optional): Engine to use. Defaults to "HNSW" for new descriptorsets. metric (str, optional): Metric to use. Defaults to "CS" for new descriptorsets. log_level (int, optional): Logging level. Defaults to logging.WARN. rUtilscreate_connectorzMApertureDB is not installed. Please install it using 'pip install aperturedb'NzNo embedding function provided.zFailed to connect to ApertureDB)aperturedb.Utilsr)r* ImportErrorsuper__init__logging getLogger__name__loggersetLevelrembedding_functionrrrr"fatal ValueError exception connectionutilsstatus Exception_find_or_add_descriptor_set) selfrrrrrr r"r$r)r* __class__s g/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/vectorstores/aperturedb.pyr/zApertureDB.__init__sJ  @ @ @ @ @ @ @ @ @   +   ""6"""'11  Y''',",$  $   K  ? @ @ @>?? ?  @ @ @ @ @ @ @ @ @    K ! !+      +*,,U4?++   J          K ! !"C D D D   ((*****s  &C&C?'E&E'c|j} d|dddddidig}|j|\}}|js J|d|ddvr!t |dddnd}|dks Jd|dkr|dddd}|jd |d |d }t |dks Jd |j|d|_n<|j|dkr+|jd |jd|d|d}t |dks Jd|j |d|_ n<|j |dkr+|jd|j d|d|d} |j | |_ n0|j | kr%|jd|j d| d| D|_ dS|jd |d|j t|_|j t|_ |j ,t |jd|_ |j #d|j Dnd} |j||j |j|j | |jdd|jdd|jdt(dS)NFindDescriptorSetTall_properties) with_nameenginesmetricsrresultsentitiesrz+Multiple descriptor sets with the same namezDescriptor set z already exists_engineszOnly one engine is supportedzEngine mismatch: z != _metricszOnly one metric is supportedzMetric mismatch: _dimensionszDimensions mismatch: ci|]<\}}|t|ttd|=SN) startswithPROPERTY_PREFIXlen.0kvs r@ z:ApertureDB._find_or_add_descriptor_set..sSAq<<00#o&&(()1z does not exist. Creating ittestc*i|]\}}t|z|Sr+rPrRs r@rVz:ApertureDB._find_or_add_descriptor_set..s#LLLDAq1$aLLLrW)namedimrrr" _Descriptor _create_txn_DescriptorSet_name)rr9query last_query_okrQr3inforerrorrritemsr"ENGINEMETRICr5 embed_queryr:add_descriptorsetcreate_entity_indexUNIQUEID_PROPERTY) r>r find_ds_queryrb n_entitieserErFrr"s r@r=z&ApertureDB._find_or_add_descriptor_setls,E$!/##"& 0$7 &&   $$]331,,..11111QqT"5666 !()*5 6 6 6  Q M ??!()*5a8A K  N~NNN O O O mGw<<1$$$&D$$${"%aj  ** !!"Sdk"S"Swqz"S"STTT mGw<<1$$$&D$$${"%aj  ** !!"Sdk"S"Swqz"S"STTT=)J&",J.. !!MDOMMMMGGIIDOOO K  N.NNN   {"$ {"$ &"%d&=&I&I&&Q&Q"R"R?.MLDO4I4I4K4KLLLL  J ( (#O{{% )    J * *=- H H H J * *+;W E E E J * *=:K L L L L LrWidsOptional[List[str]]Optional[bool]c | Jdd|jtd|gidig}|j|\}}|S)zDelete documents from the vectorstore by id. Args: ids: List of ids to delete from the vectorstore. Returns: True if the deletion was successful, False otherwise Nzids must be providedDeleteDescriptorin)set constraints)rrkr:execute)r>rqr$raresult_s r@deletezApertureDB.deletesa 6#.$5c{#C%%  J&&u--  rW Sequence[str]List[Document]cdjtd|giddidig}j|\}}fd|dddgD}|S) zFind documents in the vectorstore by id. Args: ids: List of ids to find in the vectorstore. Returns: documents: List of Document objects found in the vectorstore. FindDescriptorrvrCT)rwrxrGc:g|]}|Sr+_descriptor_to_documentrSdr>s r@ z)ApertureDB.get_by_ids..s7     ( ( + +   rWrrH)rrkr:ryget)r>rqrarGr{docss` r@ get_by_idszApertureDB.get_by_idss!.$5c{#C 0$7##  Z''..     QZ 0155j"EE    rWrarTargsc||j Jd|j|}|j||g|Ri|S)aSearch for documents similar to the query using the vectorstore Args: query: Query string to search for. k: Number of results to return. Returns: List of Document objects ordered by decreasing similarity to the query. NzEmbedding function is not set)r5rhsimilarity_search_by_vector)r>rarTrr$ embeddings r@similarity_searchzApertureDB.similarity_searchsW&224S222+77>> /t/ 1NtNNNvNNNrWList[Tuple[Document, float]]cX|j|}|j|g|Ri|SrN)r5rh'_similarity_search_with_score_by_vector)r>rarr$rs r@similarity_search_with_scorez'ApertureDB.similarity_search_with_scores>+77>> ;t;IWWWWPVWWWrWrdictr ci}|D]>\}}|tr|||ttd<?|t}|t }t |||}|S)N page_contentmetadataid)rerOrPrQ TEXT_PROPERTYrkr )r>rrrTrUrrdocs r@rz"ApertureDB._descriptor_to_document sGGII 8 8DAq||O,, 8673//1123&'D8III rWFr List[float]vectorsboolcddlm}|j}tj}|j||djdtj|z dfd|DS)Nr DescriptorsT)rwvector k_neighbors distances"ApertureDB similarity search took  secondscJg|]}||df S) _distancerrs r@rzFApertureDB._similarity_search_with_score_by_vector..#s0WWWa--a00!K.AWWWrWaperturedb.Descriptorsrr9time find_similarrr3rc)r>rrTrr descriptors start_times` r@rz2ApertureDB._similarity_search_with_score_by_vectors 766666!k$/22 Y[[   #I1PT !     Sz1I S S S   XWWW;WWWWrWc ddlm}|j}tj}|j||jdtj|z dfd|DS)a3Returns the k most similar documents to the given embedding vector Args: embedding: The embedding vector to search for k: The number of similar documents to return Returns: List of Document objects ordered by decreasing similarity to the query. rr)rwrrrrc:g|]}|Sr+rrs r@rz:ApertureDB.similarity_search_by_vector..<'EEEA,,Q//EEErWr)r>rrTr$rrrs` r@rz&ApertureDB.similarity_search_by_vector%s 766666!k$/22 Y[[   #I1 !     Sz1I S S S   FEEEEEEErW?fetch_k lambda_multfloatc |jd||j|}|j||||fi|S)aReturns similar documents to the query that also have diversity This algorithm balances relevance and diversity in the search results. Args: query: Query string to search for. k: Number of results to return. fetch_k: Number of results to fetch. lambda_mult: Lambda multiplier for MMR. Returns: List of Document objects ordered by decreasing similarity/diversty. z)Max Marginal Relevance search for query: )r3rcr5rh'max_marginal_relevance_search_by_vector)r>rarTrrr$rs r@max_marginal_relevance_searchz(ApertureDB.max_marginal_relevance_search>sf, LULLMMM+77>> ;t; q';  28   rWc ddlm}|j}tj}|j||||jdtj|z dfd|DS)aReturns similar documents to the vector that also have diversity This algorithm balances relevance and diversity in the search results. Args: embedding: Embedding vector to search for. k: Number of results to return. fetch_k: Number of results to fetch. lambda_mult: Lambda multiplier for MMR. Returns: List of Document objects ordered by decreasing similarity/diversty. rr)rwrrrrz&ApertureDB similarity search mmr took rc:g|]}|Sr+rrs r@rzFApertureDB.max_marginal_relevance_search_by_vector..~rrW)rrr9rfind_similar_mmrrr3rc) r>rrTrrr$rrrs ` r@rz2ApertureDB.max_marginal_relevance_search_by_vectorZs, 766666!k$/22 Y[[ $$## %     WTY[[:5M W W W   FEEEEEEErWclsType[ApertureDB]texts List[str] metadatasOptional[List[dict]]c F|dd|i|}||||S)aLCreates a new vectorstore from a list of texts Args: texts: List of text strings embedding: Embeddings object as for constructing the vectorstore metadatas: Optional list of metadatas associated with the texts. kwargs: Additional arguments to pass to the constructor rr+) add_texts)rrrrr$stores r@ from_textszApertureDB.from_textss7"33y3F33 y))) rW documentsc D|dd|i|}|||S)aXCreates a new vectorstore from a list of documents Args: documents: List of Document objects embedding: Embeddings object as for constructing the vectorstore metadatas: Optional list of metadatas associated with the texts. kwargs: Additional arguments to pass to the constructor rr+) add_documents)rrrr$rs r@from_documentszApertureDB.from_documentss7 33y3F33 I&&& rWcjddlm}m}|}||}||dS)zDeletes a vectorstore and all its data from the database Args: descriptor_set: The name of the descriptor set to delete rr(N)r,r)r*remove_descriptorset)class_rr)r*dbr:s r@delete_vectorstorezApertureDB.delete_vectorstoresQ =<<<<<<<    b  "">22222rWcddlm}|}dddiddddig}||\}}|s J||dddS)zReturns a list of all vectorstores in the database Returns: List of descriptor sets with properties r)r*rBrCT)rGrErFrrH)r,r*rarb)rr*rraresponser{s r@list_vectorstoreszApertureDB.list_vectorstoress 655555    $ 0$7##"& &&   hhuoo !!!++8+++{./ ;;rWc Vd|vr|d}|r/t|t|krtdg}t||D]6\}}t |j|j|}||7n|}|j|fi|dS)aAdd or update documents in the vectorstore. Args: documents: Documents to add to the vectorstore. kwargs: Additional keyword arguments. if kwargs contains ids and documents contain ids, the ids in the kwargs will receive precedence. Returns: List of IDs of the added texts. Raises: ValueError: If the number of ids does not match the number of documents. rqzhThe number of ids must match the number of documents. Got {len(ids)} ids and {len(documents)} documents.r succeeded) poprQr7zipr rrappendupsert)r>rr$rq documents_id_document doc_with_ids r@rzApertureDB.add_documentss F??**U##C s3xx3y>>11 I J!$S)!4!4 / / X&!)!6%. !!+....  /#Jt{:0000==rWreSequence[Document]r c ddlm}d|D}|r||d|D}d|D}|j|}d|D}g} t ||||D]\} } } } d| D}| |t<| |t<d|j |d i}|g}tj | tj  g}| ||f||j}|| t" t%|g S) zInsert or update items Updating documents is dependent on the documents' `id` attribute. Args: items: List of Document objects to upsert Returns: UpsertResponse object with succeeded and failed r)ParallelLoadercJg|] }t|dr|j|j!Sr)hasattrr)rSitems r@rz%ApertureDB.upsert..s:$ $ $ t)<)<$ AEATDGATATATrWcg|] }|j Sr+)rrSrs r@rz%ApertureDB.upsert.. s333c!333rWcBg|]}t|dd|jniS)rN)getattrrrs r@rz%ApertureDB.upsert..s?   $CT::FCLLB   rWcg|]@}t|dr|j|jnttjASr)rrruuiduuid4rs r@rz%ApertureDB.upsert..sR   c4(( VSV-?CFFSEVEV   rWc*i|]\}}t|z|Sr+rZrRs r@rVz%ApertureDB.upsert..s#NNNTQ/A-qNNNrW AddDescriptor)rwr")dtype) batchsize)rfailed)aperturedb.ParallelLoaderrr|r5embed_documentsrrerrkrnparrayfloat32tobytesrr9ingest BATCHSIZEr )r>rer$r ids_to_deleterrrrqdatarrr unique_idr"commandrablobsloaders r@rzApertureDB.upserts =<<<<<$ $ %$ $ $   ' KK & & &33U333     ,<%>%>%>N4848484848484848rWr)! __future__rr0rrtypingrrrrrr r numpyrlangchain_core.documentsr langchain_core.embeddingsr langchain_core.indexing.baser langchain_core.vectorstoresrtyping_extensionsrrfrgrrrPrrkrr+rWr@r s6"""""" CCCCCCCCCCCCCCCCCC.-----000000777777333333&&&&&&     M8M8M8M8M8M8M8M8M8M8rW