NgxdZddlmZddlZddlZddlZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZmZddlmZddlmZ ddlmZmZmZmZn#e$rddlmZmZmZmZYnwxYwddlZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%ej&e'Z(Gd d eZ)Gd dedZ*GddeZ+GddedZ,GddZ-ee+e,e.fZ/GddeZ0ee0e.fZ1Gdde-Z2d5dZ3dZ4d6d!Z5d7d%Z6Gd&d'Z7d8d)Z8d9d,Z9d:d.Z:d;d2Z;ee eej%eej$gee+e,ffe e ej%e ej$gee+e,fffZds++ r'r@T)precd|vs|d9t|ttfrtd||S)z$Check that the value is not numeric.r3NzJNumeric values should be provided in the 'score' field, not 'value'. Got: ) isinstanceintfloatloggerwarning)clsvvaluess r(check_value_non_numericz(EvaluationResult.check_value_non_numericis] & F7O$;!c5\** !!! r')r!r"r#r$r%r3rr4r5rdictr7r8r:r;r<r@rrKr&r'r(r1r1Is@ HHH@E0E8!G!!!!2!%J%%%%: 5666N66665=AOAAAA;59M9999659M9999!E    ) YwD!!!  "!   r'r1ceZdZUdZded<dS)EvaluationResultszqBatch evaluation results. This makes it easy for your evaluator to return multiple metrics at once. zList[EvaluationResult]resultsNr r&r'r(rNrNxs* $###!!r'rNc:eZdZdZe d d d Z d d d ZdS) RunEvaluatorzEvaluator interface class.NrunrexampleOptional[Example]return*Union[EvaluationResult, EvaluationResults]cdS)zEvaluate an example.Nr&selfrRrSs r( evaluate_runzRunEvaluator.evaluate_runsr'cnKtjd|j||d{VS)z#Evaluate an example asynchronously.N)asyncioget_running_looprun_in_executorrZrXs r( aevaluate_runzRunEvaluator.aevaluate_runsS-//?? $#S'         r'NrRrrSrTrUrV)r!r"r#r$rrZr_r&r'r(rQrQs^$$59####^# 6:       r'rQcJeZdZUdZded< ded< dZded< dZd ed <dS) ComparisonEvaluationResultzFeedback scores for the results of comparative evaluations. These are generated by functions that compare two or more runs, returning a ranking or other feedback. rr2z'Dict[Union[uuid.UUID, str], SCORE_TYPE]scoresNr9r:z6Optional[Union[str, Dict[Union[uuid.UUID, str], str]]]r4)r!r"r#r$r%r:r4r&r'r(rcrcsb  HHH@3333459M99996FJGJJJJ::r'rcceZdZdZ d"d#dZ d$d%dZd&dZd'dZed(dZ d"d)dZ d"d*fd Z d"d)dZ d+d!Z xZS),DynamicRunEvaluatoraA dynamic evaluator that wraps a function and transforms it into a `RunEvaluator`. This class is designed to be used with the `@run_evaluator` decorator, allowing functions that take a `Run` and an optional `Example` as arguments, and return an `EvaluationResult` or `EvaluationResults`, to be used as instances of `RunEvaluator`. Attributes: func (Callable): The function that is wrapped by this evaluator. NfuncXCallable[[Run, Optional[Example]], Union[_RUNNABLE_OUTPUT, Awaitable[_RUNNABLE_OUTPUT]]]afuncIOptional[Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]]ct|}|rt|}t||ddlm}|7||t |_t|dd|_tj |rJ|td||t |_t|dd|_dS|ttttt gt"f|t |_t|dd|_dS)zInitialize the DynamicRunEvaluator with a given function. Args: func (Callable): A function that takes a `Run` and an optional `Example` as arguments, and returns a dict or `ComparisonEvaluationResult`. r run_helpersNprocess_inputsr!rfFunc was provided as a coroutine function, but afunc was also provided. If providing both, func should be a regular function to avoid ambiguity.)_normalize_evaluator_funcr langsmithrmensure_traceable_serialize_inputsrigetattr_nameinspectiscoroutinefunction TypeErrorrrrr r_RUNNABLE_OUTPUTrgrYrgrirms r(__init__zDynamicRunEvaluator.__init__sV()..  5-e44Ed D))))))  $55&76DJ! 4IJJDJ  &t , , J 3 %55%66DJ!z3HIIDJJJ#44XsHW$568HHI4PP05DI!z3HIIDJJJr'FresultUnion[EvaluationResult, dict]r: uuid.UUID allow_no_keyboolrUr1c\ttrjs|_S stddvr |r |jd<t fddDrtdtdid|iS#t $r}td|d}~wwxYw) NziExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got empty result: r2c3 K|]}|vV dSr`r&).0kr}s r( z@DynamicRunEvaluator._coerce_evaluation_result..s'JJq1F?JJJJJJr')r3rr4zrExpected an EvaluationResult object, or dict with a metric 'key' and optional 'score' or categorical 'value'; got r:z[Expected an EvaluationResult object, or dict with a metric 'key' and optional 'score'; got r&)rCr1r: ValueErrorrvallr)rYr}r:res ` r(_coerce_evaluation_resultz-DynamicRunEvaluator._coerce_evaluation_results6 f. / / ' 5'4$M   OFLOOF""|" $ u JJJJ,IJJJJJ  XOUXX$QQ &P&PQQ Q   =4:==  sA B B+B&&B+rOUnion[dict, EvaluationResults]rVcd|vr8|}fd|dD|d<tdi|Stt|dS)NrOc>g|]}|S))r:)r)rrrYr:s r( zBDynamicRunEvaluator._coerce_evaluation_results..s<..q .NNr'T)r:rr&)copyrNrrrL)rYrOr:cps` ` r(_coerce_evaluation_resultsz.DynamicRunEvaluator._coerce_evaluation_resultss   B +ByM%**r** *-- w  }4.   r'MUnion[EvaluationResult, EvaluationResults, dict, str, int, bool, float, list]ct|tr|js||_|St|}|||Sr`)rCr1r:_format_evaluator_resultr)rYr}r:s r(_format_resultz"DynamicRunEvaluator._format_resultsR f. / / ' 5'4$M)&11..v}EEEr'c"t|dSzCheck if the evaluator function is asynchronous. Returns: bool: True if the evaluator function is asynchronous, False otherwise. rihasattrrYs r(is_asynczDynamicRunEvaluator.is_async'tW%%%r'rRrrSrTct|ds_tj}|rt d||||Stj}d|j i}t|ddrt|j |d<| ||||d}|||S) aEvaluate a run using the wrapped function. This method directly invokes the wrapped function with the provided arguments. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: Union[EvaluationResult, EvaluationResults]: The result of the evaluation. rgtCannot call `evaluate_run` on an async run evaluator from within an running event loop. Use `aevaluate_run` instead.r; session_idN experimentrun_idmetadatalangsmith_extra)rr\get_event_loop is_running RuntimeErrorrun_until_completer_uuiduuid4idrurrrgr)rYrRrS running_loopr:rr}s r(rZz DynamicRunEvaluator.evaluate_run0stV$$ Y"133L&&(( Y"R $66t7I7I#w7W7WXXX $3SV#< 3 d + + 9%(%8%8H\ "  '4(KK   ""6=999r'cnKt|ds(t||d{VStj}d|ji}t |ddrt|j|d<| ||||dd{V}| ||S)aEvaluate a run asynchronously using the wrapped async function. This method directly invokes the wrapped async function with the provided arguments. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: Union[EvaluationResult, EvaluationResults]: The result of the evaluation. riNr;rrrr) rsuperr_rrrrurrrir)rYrRrSr:rr} __class__s r(r_z!DynamicRunEvaluator.aevaluate_runRstW%% =..sG<<<<<<<< < $3SV#< 3 d + + 9%(%8%8H\ "zz  '4(KK"         ""6=999r'c.|||S)aMake the evaluator callable, allowing it to be used like a function. This method enables the evaluator instance to be called directly, forwarding the call to `evaluate_run`. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: Union[EvaluationResult, EvaluationResults]: The result of the evaluation. )rZrXs r(__call__zDynamicRunEvaluator.__call__ms  g...r'rcd|jdS))Represent the DynamicRunEvaluator object.zrvrs r(__repr__zDynamicRunEvaluator.__repr__~s4tz4444r'r`)rgrhrirj)F)r}r~r:rrrrUr1)rOrr:rrUrV)r}rr:rrUrVrUrra)rRrrSrTrUr)r!r"r#r$r|rrrpropertyrrZr_rr __classcell__)rs@r(rfrfs+, 0J0J0J0J0Jl# <    " F F F F&&&X&6: : : : : :D:::::::86://///"55555555r'rfrgrhc t|S)zmCreate a run evaluator from a function. Decorator that transforms a function into a `RunEvaluator`. )rfrgs r( run_evaluatorrs t $ $$r'i'objrc~t|}t|tkr|dtdz dz}|S)Nz...))reprlen_MAXSIZE)rss r( _maxsize_reprrs: S A 1vv n1 n  & Hr'inputsrLrUct|d}t|d}||dS)NrRrS)rRrS)rget)r run_truncatedexample_truncateds r(rtrtsD!&**U"3"344M%fjj&;&;<< -> ? ??r'ceZdZdZ dddZedd Z dddZ dddZ dddZ d dZ e d!dZ d"dZ dS)#DynamicComparisonRunEvaluatorz4Compare predictions (as traces) from 2 or more runs.NrgfCallable[[Sequence[Run], Optional[Example]], Union[_COMPARISON_OUTPUT, Awaitable[_COMPARISON_OUTPUT]]]riUOptional[Callable[[Sequence[Run], Optional[Example]], Awaitable[_COMPARISON_OUTPUT]]]ct|}|rt|}t||ddlm}|7||t |_t|dd|_tj |rJ|td||t |_t|dd|_dS|ttttt t"gt$f|t |_t|dd|_dS)zInitialize the DynamicRunEvaluator with a given function. Args: func (Callable): A function that takes a `Run` and an optional `Example` as arguments, and returns an `EvaluationResult` or `EvaluationResults`. rrlNrnr!rfrp)$_normalize_comparison_evaluator_funcrrrrmrsrtrirurvrwrxryrrr rr r_COMPARISON_OUTPUTrgr{s r(r|z&DynamicComparisonRunEvaluator.__init__sj(4D99  @8??Ed D))))))  $55&76DJ! 4IJJDJ  &t , , J 3 %55%66DJ!z3HIIDJJJ#44!#(9:*,  15  DI!z3HIIDJJJr'rUrc"t|dSrrrs r(rz&DynamicComparisonRunEvaluator.is_asyncrr'runs Sequence[Run]rSrTrcct|ds_tj}|rt d||||Stj}| |}| ||||d}| |||S)zCompare runs to score preferences. Args: runs: A list of runs to compare. example: An optional example to be used in the evaluation. rgrrtagsr) rr\rrrr acompare_runsrr _get_tagsrg_format_results)rYrrSrr:rr}s r( compare_runsz*DynamicComparisonRunEvaluator.compare_runsstV$$ "133L&&(( "R $66&&tW55 ~~d##  '4dCC   ##FM4@@@r'cKt|ds|||Stj}||}|||||dd{V}||||S)aEvaluate a run asynchronously using the wrapped async function. This method directly invokes the wrapped async function with the provided arguments. Args: runs (Run): The runs to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: ComparisonEvaluationResult: The result of the evaluation. rirrN)rrrrrrir)rYrrSr:rr}s r(rz+DynamicComparisonRunEvaluator.acompare_runss tW%% 4$$T733 3 ~~d##zz  '4dCC"         ##FM4@@@r'c.|||S)aMake the evaluator callable, allowing it to be used like a function. This method enables the evaluator instance to be called directly, forwarding the call to `evaluate_run`. Args: run (Run): The run to be evaluated. example (Optional[Example]): An optional example to be used in the evaluation. Returns: ComparisonEvaluationResult: The result of the evaluation. )r)rYrrSs r(rz&DynamicComparisonRunEvaluator.__call__s  w///r'rcd|jdS)rz>>>>r' List[str]cg}|D]g}|dt|jzt|ddr*|dt|jzh|S)zExtract tags from runs.zrun:rNz experiment:)appendrrrur)rrrRs r(rz'DynamicComparisonRunEvaluator._get_tags3su A AC KKSV, - - -sL$// A MC,?,??@@@ r'r}-Union[dict, list, ComparisonEvaluationResult]r:rct|tr|js||_|St|tr$dt ||D|j|d}n8t|t rd|vr |j|d<nd|}t| tdid|i|S#t$r}td||d}~wwxYw)Nc$i|] \}}|j|Sr&)r)rrRr3s r( zADynamicComparisonRunEvaluator._format_results..Js MMMZS%365MMMr')rdr2r:r2zXExpected 'dict', 'list' or 'ComparisonEvaluationResult' result object. Received: result=r:zExpected a dictionary with a 'key' and dictionary of scores mappingrun IDs to numeric scores, or ComparisonEvaluationResult object, got r&) rCrcr:listziprvrLrr)rYr}r:rmsgrs r(rz-DynamicComparisonRunEvaluator._format_results>s: f8 9 9 "' 5'4$M  % % "MM3tV;L;LMMMz!.FF  % % "F"" $ u /%+// S// ! -"M> 6J6J6J6J6Jp&&&X&AEAAAAA@AEAAAAA:AE00000"????\""""""r'rrc t|S)z.Create a comaprison evaluator from a function.)rrs r(comparison_evaluatorrcs ) . ..r'r|Union[Callable[[Run, Optional[Example]], _RUNNABLE_OUTPUT], Callable[[Run, Optional[Example]], Awaitable[_RUNNABLE_OUTPUT]]]cJdtj}d|jDr.t fdDs(t dkrdd}t |t fdDrdd gkrStjr5dfd }tdrtdn|j |_ |Sdfd }tdrtdn|j |_ |S)NrRrSroutputsreference_outputscBg|]\}}|j|j|jfv|Sr&kindPOSITIONAL_OR_KEYWORDPOSITIONAL_ONLYrpnameps r(rz-_normalize_evaluator_func..u> E1 6a-q/@A A A  A A Ar'c3 K|]}|vV dSr`r&rrsupported_argss r(rz,_normalize_evaluator_func..{(EEE'EEEEEEr'kInvalid evaluator function. Must have at least one positional argument. Supported positional arguments are . Please see https://docs.smith.langchain.com/evaluation/how_to_guides/evaluation/evaluate_llm_application#use-custom-evaluatorsc3 K|]}|vV dSr`r&rs r(rz,_normalize_evaluator_func..9$)r'rRrSrrTrUrzcK|||r|jni|jpi|r |jpinidfdD}|d{VS)Nrc3(K|] }|V dSr`r&rargarg_maps r(rz>_normalize_evaluator_func..awrapper..'@@ @@@@@@r'rrrRrSargsr rgpositional_argss @r(awrapperz+_normalize_evaluator_func..awrappers&07?gnnR"{0bBI)Q)>Br A@@@@@@!T4[(((((((r'r!rcr|||r|jni|jpi|r |jpinidfdD}|S)Nrc3(K|] }|V dSr`r&rs r(rz=_normalize_evaluator_func..wrapper..r r'r r s @r(wrapperz*_normalize_evaluator_func..wrappersh&07?gnnR"{0bBI)Q)>Br A@@@@@@tT{"r')rRrrSrTrUrz)rRrrSrrUrz rw signature parametersitemsrrrrxrrur!rgsigrrrrrs` @@r(rqrqms RN  D ! !C,,..O : EEEE_EEE E E:  A % % G.rr'c3 K|]}|vV dSr`r&rs r(rz7_normalize_comparison_evaluator_func..rr'rrrc3 K|]}|vV dSr`r&rs r(rz7_normalize_comparison_evaluator_func..rr'rrSrrTrUrcK|||r|jnid|D|r |jpinidfdD}|d{VS)Nc g|] }|jpi Sr&rrrRs r(rzJ_normalize_comparison_evaluator_func..awrapper..BBBc 1rBBBr'rc3(K|] }|V dSr`r&rs r(rzI_normalize_comparison_evaluator_func..awrapper..r r'r rrSr r rgrs @r(rz6_normalize_comparison_evaluator_func..awrappers!&07?gnnRBBTBBBBI)Q)>Br A@@@@@@!T4[(((((((r'r!rcx|||r|jnid|D|r |jpinidfdD}|S)Nc g|] }|jpi Sr&r!r"s r(rzI_normalize_comparison_evaluator_func..wrapper..r#r'rc3(K|] }|V dSr`r&rs r(rzH_normalize_comparison_evaluator_func..wrapper..r r'r r%s @r(rz5_normalize_comparison_evaluator_func..wrapperso &07?gnnRBBTBBBBI)Q)>Br A@@@@@@tT{"r')rrrSrTrUr)rrrSrrUrrrs` @@r(rrs SN  D ! !C,,..O : EEEE_EEE E E:  A % % G. s,771:a&&777777r'z8Expected a list of dicts or EvaluationResults. Received .rOrzZExpected a dict, str, bool, int, float, list, EvaluationResult, or EvaluationResults. Got ) rCrrErDrrrrrL)r}s r(rrs&4,-- 6"   D;A D D    FD ! ! 7777777 T6TTT V$ FC  6" FD ! !   /&, / /    Mr'SUMMARY_EVALUATOR_Tcdtj}d|jDr.t fdDs3t dkr dd}r |ddz }t |t fdDrd d gkrSdfd }tdrtdn|j |_ |S)NrexamplesrrrcBg|]\}}|j|j|jfv|Sr&rrs r(rz0_normalize_summary_evaluator..,rr'c3 K|]}|vV dSr`r&rs r(rz/_normalize_summary_evaluator..2rr'rrr.z Received positional arguments c3 K|]}|vV dSr`r&rs r(rz/_normalize_summary_evaluator..>rr'rr2Sequence[schemas.Run]Sequence[schemas.Example]rUrVc||d|Dd|Dd|DdfdD}|}t|tr|St|S)Ncg|] }|j Sr&)rrrSs r(rzA_normalize_summary_evaluator..wrapper..JsBBBg7>BBBr'c g|] }|jpi Sr&r!r"s r(rzA_normalize_summary_evaluator..wrapper..Ks>>>#CK-2>>>r'c g|] }|jpi Sr&r!r:s r(rzA_normalize_summary_evaluator..wrapper..Ls%T%T%Tgo&;%T%T%Tr'r1c3(K|] }|V dSr`r&rs r(rz@_normalize_summary_evaluator..wrapper..Ns'<.wrapperDs$BBBBB>>>>>%T%T8%T%T%T G=<<)s|SN  D ! !C,,..O & EEEE_EEE E E&  A % % Nr$ __future__rr\rwrabcrtypingrrrrr r r r r rtyping_extensionsrrrr pydantic.v1rrrr ImportErrorpydanticlogging functoolsrlangsmith.schemasrrrr getLoggerr!rFrr*r1rNrQrLrzrcrrfrrrrtrrrqrrr/r>r&r'r(rJsEE""""""                         ('''''  BBBBBBBBBBBB  8 $ $'''''y''' 6 6 6 6 6Ye 6 6 6 6,,,,,y,,,^""""" """"        $)+