Ng9dZddlmZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl mZddl mZddl mZddl mZer dd lmZdd lmZGd d ZeZGd dZGddZeZdgZdS)aMake approximate assertions as "expectations" on test results. This module is designed to be used within test cases decorated with the `@test` decorator It allows you to log scores about a test case and optionally make assertions that log as "expectation" feedback to LangSmith. Example usage: from langsmith import expect, test @test def test_output_semantically_close(): response = oai_client.chat.completions.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Say hello!"}, ], ) response_txt = response.choices[0].message.content # Intended usage expect.embedding_distance( prediction=response_txt, reference="Hello!", ).to_be_less_than(0.9) # Score the test case matcher = expect.edit_distance( prediction=response_txt, reference="Hello!", ) # Apply an assertion and log 'expectation' feedback to LangSmith matcher.to_be_less_than(1) # You can also directly make assertions on values directly expect.value(response_txt).to_contain("Hello!") # Or using a custom check expect.value(response_txt).against(lambda x: "Hello" in x) # You can even use this for basic metric logging within tests expect.score(0.8) expect.score(0.7, key="similarity").to_be_greater_than(0.7) ) annotationsN) TYPE_CHECKINGAnyCallableLiteralOptionalUnionoverloadclient) run_helpers) run_trees)utils)EditDistanceConfig)EmbeddingConfigc"eZdZdZddZd dZdS) _NULL_SENTRYzA sentinel singleton class used to distinguish omitted keyword arguments from those passed in with the value None (which may have different behavior). returnLiteral[False]cdS)NFselfs M/var/www/html/ai-engine/env/lib/python3.11/site-packages/langsmith/_expect.py__bool__z_NULL_SENTRY.__bool__LsustrcdS)N NOT_GIVENrrs r__repr__z_NULL_SENTRY.__repr__Os{rN)rr)rr)__name__ __module__ __qualname____doc__rr rrrrrGsFrrcteZdZdZ d'd(d Zd)d*dZd+dZd,dZd,dZd-dZ d.d/d Z d,d!Z d0d"Z d1d#Z d2d&ZdS)3_Matcherz4A class for making assertions on expectation values.Nr Optional[ls_client.Client]keyrvaluer _executor,Optional[ls_utils.ContextThreadPoolExecutor]run_id Optional[str]c||_||_||_|ptjd|_t j}|r|jn||_ dS)N max_workers) _clientr(r)ls_utilsContextThreadPoolExecutorr*rhget_current_run_treetrace_id_run_id)rr r(r)r*r,rts r__init__z_Matcher.__init__YsX  "Wh&HUV&W&W&W  $ & &&(4r{{f rscoreintmessagerNonectjsO|jstj|_|j|jj|jd||dSdS)N expectation)r,r(r;comment) r3test_tracking_is_disabledr2r9get_cached_clientr*submitcreate_feedbackr8)rr;r=s r_submit_feedbackz_Matcher._submit_feedbackhsv133 < 6!355 N ! ! ,|! "       r conditionbool method_namec |s J||dd|jd|dS#t$r+}|dt||dd}~wwxYw)Nz Success: .)r=r)rFr(AssertionErrorrepr)rrGr=rIes r_assertz_Matcher._assertts  % %g % % %  ! !!-Q-Q-QK-Q-Q ! R R R R R     ! !!T!WW - - -  s.2 A'&A""A'floatc l||j|kd|jd|d|jddS)zAssert that the expectation value is less than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not less than the given value. Expected z to be less than , but got to_be_less_thanNrPr)r(rr)s rrUz_Matcher.to_be_less_than|sP J  P P P5 P PDJ P P      rc l||j|kd|jd|d|jddS)aAssert that the expectation value is greater than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not greater than the given value. rSz to be greater than rTto_be_greater_thanNrVrWs rrYz_Matcher.to_be_greater_thansP J  S S Se S Stz S S      r min_value max_valuec |||jcxko|kncd|jd|d|d|jddS)aJAssert that the expectation value is between the given min and max values. Args: min_value: The minimum value (exclusive). max_value: The maximum value (exclusive). Raises: AssertionError: If the expectation value is not between the given min and max. rSz to be between z and rT to_be_betweenNrV)rrZr[s rr]z_Matcher.to_be_betweens  . . . .Y . . . . % % % % % % %  % %       r precisionc |t|j|t||kd|jd|d|jddS)akAssert that the expectation value is approximately equal to the given value. Args: value: The value to compare against. precision: The number of decimal places to round to for comparison. Raises: AssertionError: If the rounded expectation value does not equal the rounded given value. rSz to be approximately rTto_be_approximatelyN)rProundr)r()rr)r_s rraz_Matcher.to_be_approximatelysd $*i ( (E%,C,C C T T Tu T T T T !     rc l||j|kd|jd|d|jddS)aAssert that the expectation value equals the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value does not exactly equal the given value. rSz to be equal to rTto_equalNrVrWs rrdz_Matcher.to_equalsP J%  O O O% O O4: O O      rcb||jdud|jd|jddS)zAssert that the expectation value is None. Raises: AssertionError: If the expectation value is not None. NrSz to be None, but got to_be_nonerVrs rrfz_Matcher.to_be_nonesH J$  C C Ctz C C      rcZ|||jvd|jd|dddS)zAssert that the expectation value contains the given value. Args: value: The value to check for containment. Raises: AssertionError: If the expectation value does not contain the given value. rSz to contain z, but it does not to_containNrVrWs rrhz_Matcher.to_containsI TZ  F F Fe F F F      rfuncrctj|}|||jd|d|jddS)zAssert the expectation value against a custom function. Args: func: A custom function that takes the expectation value as input. Raises: AssertionError: If the custom function returns False. z Assertion z failed for againstN)inspect signaturerPr)r()rrifunc_signatures rrkz_Matcher.againsts[!*400 D   ? ? ?TX ? ?      r)NN) r r'r(rr)rr*r+r,r-N)r;r<r=r-rr>)rGrHr=rrIrrr>)r)rQrr>)rZrQr[rQrr>)r^)r)rQr_r<rr>)rr>)r)rrr>)rirrr>)r!r"r#r$r:rFrPrUrYr]rardrfrhrkrrrr&r&Vs>>CG $ 5 5 5 5 5                  $     "                   rr&ceZdZdZddd'dZddd(dZddd)dZd*dZddddd+dZe d*dZ e d,d Z e dfd-d#Z d.d&Z dS)/_Expectz1A class for setting expectations on test results.Nr r r'c||_tjd|_t j|jjddS)Nr/r0T)wait)r2r3r4executoratexitregistershutdownrr s rr:z_Expect.__init__s>  :qIII  .T::::::rconfig predictionr referencerzOptional[EmbeddingConfig]rr&c &ddlm}|pi}|drdnd}||}|||}||jd}|d ||d |d |jd t |jd ||j S)aCompute the embedding distance between the prediction and reference. This logs the embedding distance to LangSmith and returns a `_Matcher` instance for making assertions on the distance value. By default, this uses the OpenAI API for computing embeddings. Args: prediction: The predicted string to compare. reference: The reference string to compare against. config: Optional configuration for the embedding distance evaluator. Supported options: - `encoder`: A custom encoder function to encode the list of input strings to embeddings. Defaults to the OpenAI API. - `metric`: The distance metric to use for comparison. Supported values: "cosine", "euclidean", "manhattan", "chebyshev", "hamming". Returns: A `_Matcher` instance for the embedding distance value. Examples: >>> expect.embedding_distance( ... prediction="hello", ... reference="hi", ... ).to_be_less_than(1.0) r)EmbeddingDistanceencodercustomopenairyr{r|)rmetricembedding_distanceUsing z , Metric: r; source_inforAr*) 'langsmith._internal._embedding_distancergetevaluatedistancerFr&r2rt) rr{r|rzr encoder_func evaluatorr;src_infos rrz_Expect.embedding_distancesF NMMMMM2#)::i#8#8Fxxh %%V444 ""jI"NN+y7IJJ  'PLPPI>> expect.edit_distance("hello", "helo").to_be_less_than(1) r) EditDistancerdamerau_levenshteinnormalize_scoreTryr)r normalize edit_distancerz , Normalize: rr)"langsmith._internal._edit_distancerrrrFr&r2rt) rr{r|rzrrrrr;rs rrz_Expect.edit_distance9s< DCCCCC2H%%>)>JJ0$77  L/// ""jI"NN$9==  'DFDDDD      L  m     rr)rc<t|jd||jS)a"Create a `_Matcher` instance for making assertions on the given value. Args: value: The value to make assertions on. Returns: A `_Matcher` instance for the given value. Examples: >>> expect.value(10).to_be_less_than(20) r)r)r&r2rtrWs rr)z _Expect.valuens gu NNNNrr;)r( source_run_idrAUnion[float, int]r(rOptional[ls_client.ID_TYPE]rAr-cv|||ddi||dt|j|||jS)aLog a numeric score to LangSmith. Args: score: The score value to log. key: The key to use for logging the score. Defaults to "score". Examples: >>> expect.score(0.8) # doctest: +ELLIPSIS >>> expect.score(0.8, key="similarity").to_be_greater_than(0.7) methodz expect.score)r;rrrAr)rFr&r2rt)rr;r(rrAs rr;z _Expect.score|sW(   (.9!."       c5DMJJJJrcdSrorrWs r__call__z_Expect.__call__s363rls_client.ClientcdSrorrxs rrz_Expect.__call__sCF3r Optional[Any]Union[_Expect, _Matcher]cbt|}|tur||S|S)Nr )rqrr))rr)r expecteds rrz_Expect.__call__s6 &)))  ! !>>%(( (rresultsdictctj}|r|jnd}tjsA|jst j|_|jj |jj f||d|dSdS)N)r,r() r5r6r7r3rBr2r9rCrtrDrE)rr(r current_runr,s rrFz_Expect._submit_feedbacks-// )4>%%$133 < 6!355 DM  , 5;  HO       r)r r')r{rr|rrzr}rr&)r{rr|rrzrrr&)r)rrr&) r;rr(rrrrAr-rr&)r rrrq)r r'r)rrr)r(rrr) r!r"r#r$r:rrr)r;r rrrFrrrrqrqs.;;?C;;;;;;-1 4 4 4 4 4 4 v04 3 3 3 3 3 3 j O O O O$59!% KKKKKKB666X6 FFFXF )-1     rrqexpect) r$ __future__rrurltypingrrrrrr r langsmithr ls_clientr r5rr9rr3rrrrrrr&rqr__all__rrrrs++Z#""""" *)))))''''''%%%%%%''''''HEEEEEEGGGGGG         LNN b b b b b b b b J{{{{{{{{|  *r