NgsjddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZmZmZddlZddlmZddlmZddlmZmZmZddlmZmZmZm Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)ej*e+Z,ddZ-d dZ.d!d"dZ/d#dZ0d#dZ1edddGdd eeZ2dS)$) annotationsN) AnyCallableDictListLiteralMappingOptionalSequenceSetTupleUnioncast) deprecated) Embeddings)get_from_dict_or_envget_pydantic_field_namespre_init) BaseModel ConfigDictFieldmodel_validator) AsyncRetryingbefore_sleep_logretryretry_if_exception_typestop_after_attemptwait_exponential) is_openai_v1 embeddingsOpenAIEmbeddingsreturnCallable[[Any], Any]c ddl}tdt|jt d|j|jt|jj t|jj zt|jj zt|jj zt|jj zttt jS)NrT multiplierminmaxreraisestopwaitr before_sleep)openairr max_retriesrretry_min_secondsretry_max_secondsrerrorTimeoutAPIErrorAPIConnectionErrorRateLimitErrorServiceUnavailableErrorrloggerloggingWARNING)r r/s a/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/embeddings/openai.py_create_retry_decoratorr=,sMMM   6 7 7 ,,   $FL$8 9 9%fl&;<< =%fl&EFF G&fl&ABB C&fl&JKK  L &fgo>>   rc ddl}tdt|jt d|j|jt|jj t|jj zt|jj zt|jj zt|jj zttt jd fd }|S) NrTr%r&r*funcrr"cdfd }|S)Nargsrkwargsr"rcZK23d{V}|i|d{VcS6td)Nzthis is unreachable)AssertionError)rBrC_async_retryingr@s r< wrapped_fz7_async_retry_decorator..wrap..wrapped_fbsp) 3 3 3 3 3 3 3a!T42622222222222* !677 7s)rBrrCrr"r)r@rHrGs` r<wrapz$_async_retry_decorator..wrapas/ 8 8 8 8 8 8 8 r>)r@rr"r)r/rrr0rr1r2rr3r4r5r6r7r8rr9r:r;)r r/rJrGs @r<_async_retry_decoratorrKGsMMM#  6 7 7 ,,   $FL$8 9 9%fl&;<< =%fl&EFF G&fl&ABB C&fl&JKK  L &fgo>>N$ Kr>Fresponsedict skip_emptyboolctd|dDr |sddl}|jd|S)Nc3HK|]}t|ddkVdS) embeddingr%N)len).0ds r< z"_check_response..ns3 > >3q~  ! # > > > > > >r>datarz&OpenAI API returned an empty embedding)anyr/r3r5)rLrNr/s r<_check_responserYmsT > >Xf-= > > >>>NzN l##$LMMM Or>rCc trjjdi|St}|dfd }|di|S))Use tenacity to retry the embedding call.rCrr"cTjjdi|}t|jSN)rNrI)clientcreaterYrNrCrLr s r<_embed_with_retryz+embed_with_retry.._embed_with_retry{s4+:$+55f55xJ4IJJJJr>rIrCrr"r)rr^r_r=)r rCretry_decoratorras` r<embed_with_retryrdus~~2'z '11&111-j99OKKKKK_K   & &v & &&r>cKtrjjdi|d{VStdfd }|di|d{VS)r[NrCrr"cdKjjdi|d{V}t|jSr])r^acreaterYrNr`s r<_async_embed_with_retryz7async_embed_with_retry.._async_embed_with_retrysJ2*2<rIrb)r async_clientr_rK)r rCrhs` r<async_embed_with_retryrjs~~>3Z,3==f=========J''KKKKK('K)(22622 2 2 2 2 2 22r>z0.0.9z1.0z!langchain_openai.OpenAIEmbeddings)sinceremovalalternative_importc8eZdZUdZeddZded<eddZded<dZd ed <eZ d ed <edd Z d ed< eddZ d ed< dZ d ed<dZ d ed<dZded< eddZd ed< eddZd ed< eZded<dZded<d Zded!< d"Zded#< edd$Zd%ed&< dZded'<dZd(ed)< dZd ed*< d+Zd(ed,< ee-Zd.ed/< d+Zd(ed0< dZd1ed2<dZd3ed4<d5Z ded6< d7Z!ded8< dZ"d9ed:< e#dd;d<=Z$e%d>?e&dUdBZ'e(dVdDZ)e*dWdEZ+ddFdXdLZ,ddFdXdMZ- dYdZdOZ. dYdZdPZ/d[dSZ0d[dTZ1dS)\r!ajOpenAI embedding models. To use, you should have the ``openai`` python package installed, and the environment variable ``OPENAI_API_KEY`` set with your API key or pass it as a named parameter to the constructor. Example: .. code-block:: python from langchain_community.embeddings import OpenAIEmbeddings openai = OpenAIEmbeddings(openai_api_key="my-api-key") In order to use the library with Microsoft Azure endpoints, you need to set the OPENAI_API_TYPE, OPENAI_API_BASE, OPENAI_API_KEY and OPENAI_API_VERSION. The OPENAI_API_TYPE must be set to 'azure' and the others correspond to the properties of your endpoint. In addition, the deployment name must be passed as the model parameter. Example: .. code-block:: python import os os.environ["OPENAI_API_TYPE"] = "azure" os.environ["OPENAI_API_BASE"] = "https://Build extra kwargs from additional params that were passed in.rzFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsrall_required_field_namesr field_nameinvalid_model_kwargss r< build_extrazOpenAIEmbeddings.build_extras2$2..v,, ; ;JU"" !F*!F!F!FGGG!999 N*NNNN)3NNN %+JJz$:$:j!7DDUZZ\\RR  T2TTT  "'~ r>rc t|dd|d<|dptjd|d<t|ddd|d<t|d d d|d <|dd vrd }t|d d|d <nd}t|dd||d<|dp'tjdptjd|d< ddl}t r|dd vrt jd|d|d|d|d|d|d|d|dd}|ds|j d i|j |d<|ds|j d i|j |d<n>|ds |j |d<nn#t$rtdwxYw|S)!z?Validate that api key and python package exists in environment.r~OPENAI_API_KEYrxOPENAI_API_BASEryOPENAI_API_TYPE)rorz OPENAI_PROXYazureazure_adazureadz 2023-05-15rirvOPENAI_API_VERSIONr OPENAI_ORG_IDOPENAI_ORGANIZATIONrNzfIf you have openai>=1.0.0 installed and are using Azure, please use the `AzureOpenAIEmbeddings` class.rr0rrr)r}rrwrr0rrrr^riTCould not import openai python package. Please install it with `pip install openai`.rI)rosgetenvr(r/rrrrOpenAIr AsyncOpenAI Embedding ImportError)rrdefault_api_versionr/ client_paramss r<validate_environmentz%OpenAIEmbeddings.validate_environment s$8 $&6$ $  %++<$=%  B B  !%9    % % %  ! "6    " " " ~ # $(H H H". $'vl';T#B#BF< "$ ';  ' ( ( ( #$ ( ) 0y)) 0y.// $% !  MMM~~ +,0PPPMH &&67$*+@$A &'8 9%&78#)-#8'-.?'@%+O%<#)-#8 ! ! zz(++Q'4v}'E'E}'E'E'PF8$zz.11!-?V-?..'.. >*ZZ)) #)#3x  ?   ?  @ s F33G c Ztrd|ji|j}n|j|j|j|j|j|j|j|j d|j}|jdvr |j |d<|j r7 ddl }n#t$rtdwxYw|j |j d|_|S)Nrr)rrrrr}rapi_baseapi_typertrenginerr)httphttps)rrrrrrr~rrxryrvrsrzr/rproxy)self openai_argsr/s r<_invocation_paramsz#OpenAIEmbeddings._invocation_paramsms >> !($* J8I JKK#'#7<. $ 8 0 0#6  # K#'GGG(, H% !MMMM"%G!-!.   s 5A::B)rtexts List[str]r Optional[int]List[List[float]]c T g}g}|jp|j}|p|j}|js ddlm}n#t $rt dwxYw||} t|D]\} } | | d} tdt| |j D]S} | | | |j z}| |}|||| TnB ddl}n#t $rt dwxYw ||}nA#t"$r4t$d d }||}YnwxYwt|D]\} } |jd r| d d } | | |j|j}tdt||j D]<} ||| | |j z|| =|jr] ddlm}|tdt||}nL#t $r!tdt||}Yn"wxYwtdt||}g}|D]o} t9|fd|| | |zi|j}t=|t>s|}| d|dDpdtt|D}dtt|D}tt|D]~} |j!rt|| dkr#||| || ||| t|| dtt|D}tt|D]} || }t|dkrRt9|fddi|j}t=|t>s|}|ddd}ntEj#|d|| }|tDj$%|z &|| <|S)al Generate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. r AutoTokenizerzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`. pretrained_model_name_or_pathFadd_special_tokensNCould not import tiktoken python package. This is needed in order to for OpenAIEmbeddings. Please install it with `pip install tiktoken`.5Warning: model not found. Using cl100k_base encoding. cl100k_base001  textrr)tqdminputc3&K|] }|dV dSrRNrIrTrs r<rVz.&%O%Oa n%O%O%O%O%O%Or>rWcg|]}gSrIrIrTrFs r< z=OpenAIEmbeddings._get_len_safe_embeddings..+J+J+J1B+J+J+Jr>cg|]}gSrIrIrs r<rz=OpenAIEmbeddings._get_len_safe_embeddings../N/N/Nq/N/N/Nr>r%cg|]}gSrIrIrs r<rz=OpenAIEmbeddings._get_len_safe_embeddings..(G(G(G(G(G(Gr>rrRaxisweights)'rrrrr transformersrrfrom_pretrained enumerateencoderangerSr|decodeappendtiktokenencoding_for_modelKeyErrorr9warning get_encodingendswithreplacerrr tqdm.autorrdr isinstancerMextendrNnpaveragelinalgnormtolist)rrrrtokensindices model_name _chunk_sizer tokenizerir tokenizedj token_chunk chunk_textrencodingrrtokenr_iterbatched_embeddingsrLresultsnum_tokens_in_batchr _resultaverage_embeddedrs r<_get_len_safe_embeddingsz)OpenAIEmbeddings._get_len_safe_embeddingss&-;  3DO $9 & 6666666   !V &55.86I%U++ & &4%,,Te,LL q#i..$2KLL&&A"+AD4M0M,M"NK"+!1!1+!>!>JMM*---NN1%%%% & &    !E  8#66zBB 8 8 8VWWW%#0077 8%U++ & &4:&&u--3 <<c22D $($8'+'>(q#e**d.GHH&&AMM%A0I,I(I"JKKKNN1%%%%&  ! 7 ;******U1c&kk;??@@ ; ; ;aVk:: ;!S[[+66E02 P PA'Q[01)H h-- +#==??  % %%O%Ohv>N%O%O%O O O O O+J+Jc%jj8I8I+J+J+J/N/NE#e**> B B B B(G(GU3u::5F5F(G(G(G s5zz"" I IAajG7||q  #3$$$-$$ ""2D99?'7'<'<'>'>$*6215kB*W1>QRS>TUUU$ry~~g'>'>>FFHHJqMMs9+AD D#'D==;E;:E; -I;;(J&%J&c t Kg}g}|jp|j}|p|j}|js ddlm}n#t $rt dwxYw||} t|D]\} } | | d} tdt| |j D]S} | | | |j z}| |}|||| TnB ddl}n#t $rt dwxYw ||}nA#t"$r4t$d d }||}YnwxYwt|D]\} } |jd r| d d } | | |j|j}tdt||j D]<} ||| | |j z|| =g}|p|j}tdt||D]u} t3|fd|| | |zi|jd{V}t7|t8s|}|d|dDvdtt|D}dtt|D}tt|D]]} ||| || ||| t|| ^dtt|D}tt|D]} || }t|dkrXt3|fddi|jd{V}t7|t8s|}|ddd}nt=j|d|| }|t<j !|z "|| <|S)a Asynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. rrzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`.rFrNrrrrrrrrc3&K|] }|dV dSrrIrs r<rVz=OpenAIEmbeddings._aget_len_safe_embeddings..urr>rWcg|]}gSrIrIrs r<rz>OpenAIEmbeddings._aget_len_safe_embeddings..wrr>cg|]}gSrIrIrs r<rz>OpenAIEmbeddings._aget_len_safe_embeddings..xrr>cg|]}gSrIrIrs r<rz>OpenAIEmbeddings._aget_len_safe_embeddings..}rr>rrRr)#rrrrrrrrrrrrrSr|rrrrrr9rrrrrrrjrrrMrrrrrr)rrrrrrrrrr r rr r r rrrrrrrrLrrr rrrs r<_aget_len_safe_embeddingsz*OpenAIEmbeddings._aget_len_safe_embeddingss&-;  3DO $9 & 6666666   !V &55.86I%U++ & &4%,,Te,LL q#i..$2KLL&&A"+AD4M0M,M"NK"+!1!1+!>!>JMM*---NN1%%%% & &    !E  8#66zBB 8 8 8VWWW%#0077 8%U++ & &4:&&u--3 <<c22D $($8'+'>(q#e**d.GHH&&AMM%A0I,I(I"JKKKNN1%%%%&13 3DO q#f++{33 P PA3Q[01)H h-- +#==??  % %%O%Ohv>N%O%O%O O O O O+J+Jc%jj8I8I+J+J+J/N/NE#e**> B B B B(G(GU3u::5F5F(G(G(G s5zz"" I IAajG7||q  )?***-**$$$$$$ ""2D99?'7'<'<'>'>$*6215kB*W1>QRS>TUUU$ry~~g'>'>>FFHHJqMMs'-AD D%)D??;E=<E=rcdtt|j}|||S)aMCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. r)rrqrsrrrrrs r<embed_documentsz OpenAIEmbeddings.embed_documentss.c4?++,,U6,BBBr>ctKtt|j}|||d{VS)aSCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rN)rrqrsrr s r<aembed_documentsz!OpenAIEmbeddings.aembed_documentssDc4?++33E&3IIIIIIIIIr>r List[float]c:||gdS)zCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. r)r!)rrs r< embed_queryzOpenAIEmbeddings.embed_querys##TF++A..r>cNK||gd{V}|dS)zCall out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. Nr)r#)rrr s r< aembed_queryzOpenAIEmbeddings.aembed_querys9 00$88888888 !}r>)rrr"r)rrr"r)r"r)rrrrqrrr"r)r)rrrrr"r)rrqr"r$)2__name__ __module__ __qualname____doc__rr^__annotations__rirrrsrvrxryrzr|r~rsetrrrr0rrrrrrMrrNrrr1r2rr model_configr classmethodrrrpropertyrrrr!r#r&r(rIr>r<r!r!s( ''R%d333F3333dD999L9999)E)))) %J%%%%(-d-(P(P(PPPPPS%*U4z%J%J%JOJJJJ&*O))))"&L&&&& $$$$$8$)E$i$H$H$HNHHHHO).t>)R)R)RRRRRN7:suuO<<<#>#>L>>>>VJ!6:O::::7;M;;;;77$(K(((( :XBL_(###[$#2JJJXJXXHMQAAAAAALMQwwwwwwt=>CCCCC&=>JJJJJ$ / / / /      r>)r r!r"r#)r r!r"r)F)rLrMrNrOr"rM)r r!rCrr"r)3 __future__rr:rrtypingrrrrrr r r r r rrnumpyrlangchain_core._api.deprecationrlangchain_core.embeddingsrlangchain_core.utilsrrrpydanticrrrrtenacityrrrrrr langchain_community.utils.openair getLoggerr)r9r=rKrYrdrjr!rIr>r<r<s""""""                             666666000000 CBBBBBBBBBBB:99999  8 $ $6""""L ' ' ' ' 3 3 3 3   : rrrrry*rr  rrrr>