NgdZddlmZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZdd lmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l"m&Z'dd l"m(Z)ddl*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6ddl4m7Z8ddl9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKddlLmMZMmNZNmOZOddlPmQZRdd lSm&ZTddlSmUZUmVZVe rddlWZXejYeZZ[eegeeIe#ffeee\gefe#eIfZ]eegeeIe#ffefZ^Gdde_Z`Gd d!e\ZaGd"d#e\Zb ddd+Zcdd.ZdGd/d0eGZedd2Zfdd8Zgdd;Zhdd=ZiddEZjddKZkddMZlddOZmddYZndd[Zodd]Zpddddd^ddgZqddddhddkZrddlddoZsddddhddpZtddddhddrZuddlddsZv ddd{ZwGd|d}eGd~ZxejyGddZzddZ{ddZ|dZ}dddddd~ddddZ~dddddd~ddddZdZee_edde~_dS)z>Utilities for running language models or Chains over datasets.) annotationsN)datetimetimezone) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnioncast)warn_deprecated) Callbacks)BaseLanguageModel) BaseMessagemessages_from_dict) ChatResult LLMResult)RunnableRunnableConfigRunnableLambdaconfig)utils)EvaluatorCallbackHandlerwait_for_all_evaluators)LangChainTracer)Client) get_git_infoget_langchain_env_var_metadata)EvaluationResult RunEvaluator) run_evaluator) as_runnableis_traceable_function)DatasetDataTypeExampleRun TracerSession)LangSmithError) HTTPError) TypedDict)Chain)load_evaluator) EvaluatorTypePairwiseStringEvaluatorStringEvaluator) evaluation)name_generationprogressceZdZdZdS)InputFormatErrorz(Raised when the input format is invalid.N)__name__ __module__ __qualname____doc__c/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain/smith/evaluation/runner_utils.pyr8r8Ns2222r>r8c"eZdZdZddZddZdS) TestResultz1A dictionary of the results of a single test run.return pd.DataFramec|}d|jD}|d|dS)zReturn quantiles for the feedback scores. This method calculates and prints the quantiles for the feedback scores across all feedback keys. Returns: A DataFrame containing the quantiles for each feedback key. cg|]G}|ds.|ds|dvs|dE|HS)inputs.outputs.>inputoutput reference) startswith).0cols r? z5TestResult.get_aggregate_feedback..esp   ~~i(( ~~j))  )))~~k*** *))r>all)include)axis) to_dataframecolumnsdescribedrop)selfdfto_drops r?get_aggregate_feedbackz!TestResult.get_aggregate_feedbackXs^     z   {{5{))..wQ.???r>c  ddl}n"#t$r}td|d}~wwxYwg}g}|dD]g\}}|d}|d}t |t rd|D} n |i} nd|i} id|d D| } d |vr^t |d t r8| d |d Dn |d | d <| id |D|d |d|dd|| ||i|j||S)z#Convert the results to a dataframe.rNzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.resultsfeedbackrIc i|] \}}d|| S)rGr=rLkvs r? z+TestResult.to_dataframe..s$HHH1.Q..!HHHr>c i|] \}}d|| S)rFr=r_s r?rbz+TestResult.to_dataframe..s$HHH1=Q==!HHHr>rHrJc i|] \}}d|| S)z reference.r=r_s r?rbz+TestResult.to_dataframe..s'UUUA)a))1UUUr>c.i|]}d|j|jS)z feedback.)keyscore)rLfs r?rbz+TestResult.to_dataframe..s'FFF*15**AGFFFr>Errorexecution_timerun_id)errorrjrk)index) pandas ImportErroritemsget isinstancedictupdateappend DataFrame) rWpdeindicesrecords example_idresultr]output_rIrs r?rSzTestResult.to_dataframeos/         @   "&y/"7"7"9"9 ' ' Jj)Hjj**G'4(( -HH HHH"G,HHw0E0E0G0GHHHAf$$f[14889HHUU 9L9R9R9T9TUUU&,K%8AkN HHFFXFFF#ZZ00&,-=&>$jj22     NN1    NN: & & & &r|G73333s &!&N)rBrC)r9r:r;r<rZrSr=r>r?rArAUsF;;@@@@.,4,4,4,4,4,4r>rAc,eZdZdZd fd Zd d ZxZS) EvalErrorz"Your architecture raised an error.ri BaseExceptionkwargsrrBNonec >tjdd|i|dS)Nrir=)super__init__)rWrir __class__s r?rzEvalError.__init__s+//u//////r>namestrcV ||S#t$rtd|dwxYw)Nz%'EvalError' object has no attribute '')KeyErrorAttributeError)rWrs r? __getattr__zEvalError.__getattr__sK R:  R R R !P!P!P!PQQ Q Rs ()rirrrrBr)rrrBr)r9r:r;r<rr __classcell__)rs@r?rrs_,,000000RRRRRRRRr>r llm_or_chain_factoryMODEL_OR_CHAIN_FACTORY dataset_namerrBMCFc  t|trD|jj}|j*jjj}t d|d|d|dfdSt|t r|St|tr| fdSt|r>t|r'ttt| fdS |}nk#t$r^tt|}tj|}t d |d t%| fd cYSwxYwtt|t|t r|Sttt|r'ttt| fd St|tsfd SS|S)zForgive the user if they pass in a chain without memory instead of a chain factory. It's a common mistake. Raise a more helpful error message as well.Na$Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called. This will safegaurd against information leakage between dataset examples. For example: def chain_constructor(): new_memory = z(...) return z*(memory=new_memory, ...) run_on_dataset("z", chain_constructor, ...)cSNr=)chainsr?z(_wrap_in_chain_factory..sur>cSrr=)lcfsr?rz(_wrap_in_chain_factory..ssr>cSrr= runnable_sr?rz(_wrap_in_chain_factory..9r>zWrapping function z as RunnableLambda.cSrr=)wrappedsr?rz(_wrap_in_chain_factory..s7r>cSrr=rsr?rz(_wrap_in_chain_factory..rr>c"tSr)r) constructorsr?rz(_wrap_in_chain_factory..s>+66r>)rrr/rr9memory ValueErrorrrcallabler&r%rr TypeErrorinspect signatureloggerinfor) rr chain_class memory_class_model user_funcsigrrrrrs @@@@@r?_wrap_in_chain_factoryrsO &..3$o.  & 2 <1:L L%1 L L* L L$0 L L L   }}} (*; < < ## (( 3 3"{{{ & ' ' !5 6 6 %#D3G$H$HIII$$$$ $ #))++FF # # #X';<z_get_prompt..s=> > #$Jq#  > > > > > > r>z,Expected list of strings for 'prompts', got rQc3@K|]}t|tVdSrrrs r?rz_get_prompt.. s,.S.Saz!S/A/A.S.S.S.S.S.Sr>z)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got rz)LLM Run expects single prompt input. Got z prompts.) r8rrrtyper9listrOlennextitervalues)rrprompt_s r? _get_promptrs ><===G6&*C00 "6)**366 (#$ f  &+T22 #> > (.y(9> > > ; ;  #;VI.//8;; # V  tFMMOO,,-- gs # # YiGG  & & Y3.S.S7.S.S.S+S+S YGG"#Wv#W#WXX X LF L L    7||qqz OG O O O   r>ceZdZUdZded<dS)ChatModelInputzVInput for a chat model. Parameters: messages: List of chat messages. zList[BaseMessage]messagesNr9r:r;r<__annotations__r=r>r?rrs*  r>rrsc&|std|}d|vr|d|d<nDt|dkr1t t ||d<d|vrv|d}t|trtd|Dr|g}t|dkrt|d|d<ntd|Std|) zGet Chat Messages from inputs. Args: inputs: The input dictionary. Returns: A list of chat messages. Raises: InputFormatError: If the input format is invalid. rrrHrQc3@K|]}t|tVdSr)rrrsrs r?rz _get_messages..9s=2 2 $%Jq$  2 2 2 2 2 2 r>rzGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got ) r8copypoprrrrrrrrOr)r input_copy raw_messagess r? _get_messagesr%sK ><===JV(nnZ88 7 V  "4 #8#899 7*!'* lD ) ) *c2 2 )52 2 2 / /  *)>L |   ! !"4\!_"E"EJw  ",  $! $ $   r> first_exampler) input_mapperOptional[Callable[[Dict], Any]]rc|rz||j}t|tsQt|trt d|Ds%t d|dt |ddSdS t|jdS#t $r? t|jYdS#t $rt d|jdwxYwwxYw)Nc3@K|]}t|tVdSrrrrrLmsgs r?rz>_validate_example_inputs_for_language_model..Us,IISJsK00IIIIIIr>zWhen using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages. Got: of type .zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.) rrrrrrOr8rrr)rr prompt_inputs r?+_validate_example_inputs_for_language_modelrMs`#|M$899 ,,,  |T * * IILIIIII #G'GG26l1C1CGGG        , - - - - -    m2333333#   &G*1GGG  s>B CB66#CCrr/cZ|r||j}t|j|}t |t s#t d|dt|d|r,t d|jd|dS|j}t|j|}t|dkrt|jdkrdS|r,t d|jd|dS) zexamplec t|trt||dS|}t|trt |||dSt|t rt d|dSdS)z9Validate that the example inputs are valid for the model.zSkipping input validation for N)rrrrr/rrrdebug)rrrrs r?_validate_example_inputsrs &(9::C3G\JJJJJ$$&& eU # # C .w| L L L L L x ( ( C LLA%AA B B B B B C Cr>examples List[Example]r4"Optional[smith_eval.RunEvalConfig] data_typer(Optional[List[RunEvaluator]]c P|rt|trd\}}d}nHd}|}t|tr|jnd}t|tr|jnd}t ||||djrt|djnd||}nd}|S)z > R&0 #J HHH((**E-7u-E-EO))4J/9%/G/GQ%++TK-   )1!)< FD!$ % % %$      r>rsmith_eval.RunEvalConfigrOptional[List[str]] Optional[str]cd}|jr/|j}|r%||vr!td|d|dnQ|rt|dkr |d}n3|1t|dkrtd|d|S)Nz Input key z% not in chain's specified input keys '. Evaluation behavior may be undefined.rQrz#Chain expects multiple input keys: z, Evaluator is likely to fail. Evaluation behavior may be undefined. Specify an input_key in the RunEvalConfig to avoid this warning.) input_keyrwarningr)rrrs r?_determine_input_keyrsI  $  ):55 NNSYSS)SSS     J1,,qM  C OOa$7$7 P* P P P   r>rcd}|jr/|j}|r%||vr!td|d|dnQ|rt|dkr |d}n3|1t|dkrtd|d|S)NzPrediction key z& not in chain's specified output keys rrQrz$Chain expects multiple output keys: zl, Evaluation behavior may be undefined. Specify a prediction_key in the RunEvalConfig to avoid this warning.)prediction_keyrrr)rrrs r?_determine_prediction_keyrsN  .  ><< NNU.UU +UUU     [))Q..$Q  S%5%5%9%9 ;; ; ; ;   r>example_outputsc|jr#|j}|r||vrtd|d|n-|r)t|dkrt|d}nd}|S)NzReference key z! not in Dataset example outputs: rQr) reference_keyrrr)rrrs r?_determine_reference_keyrs ,  }OCC777%477  S11Q66_--a0  r> eval_configYUnion[smith_eval_config.SINGLE_EVAL_CONFIG_TYPE, smith_eval_config.CUSTOM_EVALUATOR_TYPE]eval_llmOptional[BaseLanguageModel]rrrrr#c t|tr|St|ttfr=t|tst|}t ||}|j} nt|t jrld|i|} t |j fi| }|j j} t|t j r|j p|}|j p|}|j p|}n=t|rt|St!dt#|t|t$rG|jr|t!d| d|dt(j||||||| g} n;t|t.rt1d| d t1d| d | S) N)rrzUnknown evaluator type: zPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r)rrrtagszRun evaluator for z is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead? See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)rrr#r1rr0valuesmith_eval_config EvalConfig get_kwargsevaluator_typeSingleKeyEvalConfigrrrrrun_evaluator_decrrr3requires_reference smith_evalStringRunEvaluatorChainfrom_run_and_data_typer2NotImplementedError) r r rrrrrr evaluator_ eval_type_tagrr$s r?_construct_run_evaluatorrsC+|,,+ s344I+}55 5' 44K#KX>>> #) K!2!= > > I>[%;%;%=%=>#K$>II&II #28 k#4#H I I G#-:I(7I>N'5FM +  I ---GD4E4EGGHHH*o..  ( ]-BJ&3JJ7FJJJ  #:QQ   )'R   J 7 8 8  ! Q Q Q Q   " C C C C    r>2Tuple[Optional[str], Optional[str], Optional[str]]clt||}t||}t||}|||fSr)rrr)rrrrrrrs r? _get_keysr Is> %VZ88I.v{CCN,V_EEM nm 33r>List[RunEvaluator]c g}d\}}} |js%|jr4td|jDrt||||\}}} |jD]2} t | |j|||| ||} || 3|jpg} | D]} t| tr|| -t| tr9|tj | ||||| {t| r#|t| td| d|S)z Load run evaluators from a configuration. Args: config: Configuration for the run evaluators. Returns: A list of run evaluators. NNNc8g|]}t|tSr=)rrr3)rLrxs r?rNz(_load_run_evaluators..js"RRRAA//RRRr>)rrrzUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.) evaluatorscustom_evaluatorsanyr rr rurrr#r3rrrrrr)rrrrrrrrrrr r$r&custom_evaluators r?rrUs"N/?,I~}    RR9QRRR S S 4= J _4 4 0 >=( - - 0  O          m,,,,06B- & 5 5   ! !"2 3 3 3 3 (/ : :   ! !2II$'#1"/ J    & ' '   ! !"34D"E"E F F F F>1A>>>  r>r callbacksrmetadatarrrr*rr+Optional[Dict[str, Any]]Union[str, BaseMessage]c JK|||}t|ts.t|trJtd|Dr1||t ||pg|pid{VSt d|d t|}||t ||pg|pid{V}nJ#t $r=t|} |jdi| dt ||pg|piid{V}YnwxYw|S) aAsynchronously run the language model. Args: llm: The language model to run. inputs: The input dictionary. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map inputs to the expected format. Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Nc3@K|]}t|tVdSrrrs r?rz_arun_llm..,OOSJsK00OOOOOOr>r*rr+rz%Input mapper returned invalid format 3 Expected a single string or list of chat messages.rr=) rrrrrOainvokerr8rr) rrrr*rr+prompt_or_messagesr llm_output llm_inputss r? _arun_llmr7s0)\&11 )3 / / ,d33 OOr% #G&GGG   ((F8; %'djb8>r9D99333333JJ     &v..J*s{    %'djb8>r   JJJ  sACAD D rrr+Union[Chain, Runnable]Union[dict, str]cK||n ||}t|trt|trzt|dkrg|jr`t t |}||t||pg|pid{V}n3t|pg||pi} ||| d{V}|S)z%Run a chain asynchronously on inputs.NrQr1rrr*r+) rrr/rsrrrrrr3r rrr*rrr+inputs_valrIrunnable_configs r? _arun_chainrAs4%,ff,,v2F2FG5%  F w % %F LLA     4(())**}} !#$*"x~2%        )y8>r   }}W_}EEEEEEEE Mr>)rr'Union[dict, str, LLMResult, ChatResult]c JKt|trdnd}d} t|tr@t||j|d|d||dd{V}nI|}t ||j|d|d||dd{V}|}nc#t $rV}t|d|j d |jd t|t| }Yd}~nd}~wwxYw|S) aAsynchronously run the Chain or language model. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map the input to the expected format. Returns: A list of outputs. LLMr/Nrr*r+r) failed for example with inputs  ri) rrrr7rrqrA Exceptionrridreprr) rrrr chain_or_llmr|rIrrxs r?_arun_llm_or_chainrMs(02CDDQ'F$ *,= > >  )$F^ -)J// !!!FF)(**E&F^ -)J// F $$$    ">  a     ### $ MsB C D  A DD c |||}t|ts.t|trEtd|Dr,||t ||pg|pi}nt d|d t|}||t ||pg|pi}nA#t $r4t|} |jd i| dt ||pii}YnwxYw|S) a Run the language model on the example. Args: llm: The language model to run. inputs: The input dictionary. callbacks: The callbacks to use during the run. tags: Optional tags to add to the run. input_mapper: function to map to the inputs dictionary from an Example Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Nc3@K|]}t|tVdSrrrs r?rz_run_llm..Or0r>r1rz'Input mapper returned invalid format: r2r)r*r+r=) rrrrrOinvokerr8rr) rrr*rrr+r4r5 llm_promptsr6s r?_run_llmrR1s2)\&11 )3 / / ,d33 OOr3=33JJ#G&GGG  %f--K%'djb8>r$JJ     &v..J#% HNPRSSSJJJ  s:C ;D  D Union[Dict, str]c||n ||}t|trt|trtt|dkra|jrZt t |}||t||pg|pi}n-t|pg||pi} ||| }|S)zRun a chain on inputs.NrQr1rr<) rrr/rsrrrrrrPrr=s r? _run_chainrUos%,ff,,v2F2FG5%  ? w % %? LLA     4(())** !#$*"x~2  )y8>r   go>> Mr>c Bt|trdnd}d} t|tr:t||j|d|d||d}nC|}t ||j|d|d||d}|}nm#t $r`}t|j} t |d|j d |jd | d | t| }Yd}~nd}~wwxYw|S) a Run the Chain or language model synchronously. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. Returns: Union[List[dict], List[str], List[LLMResult], List[ChatResult]]: The outputs of the model or chain. rDr/Nr*rr+r8rErFz Error Type: z , Message: rH) rrrrRrrqrUrIrr9rrrJr) rrrrrLr|rIrrx error_types r?_run_llm_or_chainrXs*02CDDQ'F$ *,= > > "${#F^)J// FF)(**E{#F^)J// F $$$!WW%  8 8 8 8"> 8 8' 8 845 8 8   ###$ MsBB22 D.s!GGGbG2>GGGr>gitr\r)reference_dataset_id project_extrar+zalready exists z+ run_on_dataset( ... project_name="z - z", # Update since z already exists ) z Test project z/ already exists. Please use a different name: z/compare?selectedSessions=z)View the evaluation results for project 'z' at: z View all tests for Dataset z at: T)flush)r read_datasetr list_examplesrJrmax isoformatr create_projectr-r,ruuiduuid4urlprint)rYrrrZr[rr\ wrapped_modeldatasetrrcmax_modified_atinferred_versiongit_infoprojectrxuid example_msgcomparison_urls r?_prepare_eval_runr{sz++?NNM!!|!<>   " x    /?*+'' !(,0864..b% (   z> 2    CFF * *Gjll %(r?r}r}s<??....####r>r}F)totalceZdZUdZded<ded<ded<ded <d ed <d Zd ed<d8dZd9dZd:dZd;dZ dd?d7Z d S)@_DatasetRunContainerz3A container to help manage the state of a eval run.rrYr+rwrrrrrzList[RunnableConfig]configsNz6Optional[List[smith_eval_config.BATCH_EVALUATOR_LIKE]]batch_evaluators batch_resultsrall_eval_resultsDict[str, _RowResult]rBrsc li}t|j|D]\}}tt|t |ji}|j|dg|d|dd|t |j<t|tr#|j |t |jd<n||t |jd<|j r"|j |t |jd<|S)Nr]rjrk)rHr]rjrkrirIrJ) ziprrr}rqrrJrrrrrir)rWrrr\rrI row_results r?_merge_test_outputsz(_DatasetRunContainer._merge_test_outputss "4=-@@ H HOGVj*:*>*>s7:PR*S*STTJ &NN:r::",..1A"B"B$..22 ((GC OO $ &),, <4:LGJ(115;GJ(2 H8?GJ(5r>runsDict[str, Run] List[dict]c \|j}|sgSfd|jD}g}tj5}|D]} |||j}t |t r|}|tt||j |j j fi|d|j jd#t$r7}t dt%|d|Yd}~d}~wwxYw dddn #1swxYwY|S)NcDg|]}t|jSr=)rrJ)rLrrs r?rNz>_DatasetRunContainer._run_batch_evaluators..1s&HHHwT#gj//*HHHr>)rk project_idzError running batch evaluator z: )rr concurrentfuturesThreadPoolExecutorrrr"rsrursubmitrYcreate_feedbackrwrJrIrrlrK) rWrr% runs_listaggregate_feedbackexecutor evaluatorr|rxs ` r?_run_batch_evaluatorsz*_DatasetRunContainer._run_batch_evaluators-s*  IHHHH$-HHH    2 2 4 4 '   &Yy$-@@F!&*:;;/!'&--d4.@.@AAA#HO 3  $#'-List[Union[dict, str, LLMResult, ChatResult]]rAcTtdt|\}}d}|jr/td||}|||}t|jj ||S)Nz#Waiting for evaluators to complete.zRunning session evaluators.)rZr\aggregate_metrics) rrrrrrrrArwr)rWrrrrr\s r?_collect_test_resultsz*_DatasetRunContainer._collect_test_resultscs  9:::!!!%)%:%:%<%<"(!   F KK5 6 6 6!%!;!;H!E!E **=:JKK*0    r>Fverboseboolc ||}|rf |}t|nA#t$r4}tdt |Yd}~nd}~wwxYw |j|j j tj tjnA#t$r4}tdt |Yd}~nd}~wwxYw|S)Nz$Failed to print aggregate feedback: )rzFailed to close project: )rrZ_display_aggregate_resultsrIrrrKrYupdate_projectrwrJrnowrutc)rWrrr\ agg_feedbackrxs r?finishz_DatasetRunContainer.finishus2,,];;  O O&==?? *<8888 O O O MDGGMMNNNNNNNN O @ K & & (,x|*D*D '     @ @ @ LL>T!WW>> ? ? ? ? ? ? ? ? @s.#= A;*A66A;?AC D *C;;DrrrrrZrr4rrrrrconcurrency_levelintr[r, revision_idr\Optional[Union[datetime, str]]c |ptj}| r| si} | d| it|||| | \} } }pgjdpiD] \}}d|d|!djdi| r| d<t|} t| ||| j p tj t|d| |tjt!|fd|D}|| |||r|jnd S) Nr)r[rr\rezgit:=r\rc g|]J}ttj|jt pg|jdgKS))rZrYr{r)r%rYr{max_concurrency)r*rrr+)rrrrJr) rLrrYr progress_barrwr run_metadatars r?rNz0_DatasetRunContainer.prepare..s   (' #%,\%#*: -#1#7R%#*:()  !  1%#      r>)rYrwrrrrr)r5 random_namertr{r+rqrprurrrr(kvrr6ProgressBarCallbackrr)clsrYrrrZr4rrrr[rr\rrrsrr`rarrrwrrs ` ` ` @@@@r?preparez_DatasetRunContainer.prepares$D'B'D'D  B# &#%  # #]K$@ A A A4E   -+5 5 5 1 wzr%))%006B==?? ( (DAq KKq1 ' ' ' ')7+;)rrrrrBrs)rrrBr)rBr)rrrBrA)F)rrrrrBrA)NNNrNNN)rYrrrrrrZrr4rrrrrrrr[r,rrr\rrBr) r9r:r;r<rrrrrrr classmethodrr=r>r?rr s==NNN!!!!OSSSSS,""""2GGGG:    $":>$(8E E E E [E E E r>rrc ddlm}|}|duodtt|vS#t$rYdSwxYw)Nr) get_ipythonzmqshellF)IPythonrrrro)rress r?_is_jupyter_environmentrsm''''''kmm{}}D(IZ3tCyy>>-II uus9< A  A aggregate_resultsrCctr)ddlm}m}||d||dS|dd}t dt |dS)Nr)HTMLdisplayz

Experiment Results:

c |dS)Nz.2fr=)xs r?rz,_display_aggregate_results..s a::r>right) float_formatjustifyz Experiment Results:)rIPython.displayrr to_stringrq)rrrformatted_strings r?rrs   11111111344555!""""",66--w7   &''' r>aThe input_mapper argument is deprecated and will be removed in a future release. Please add a RunnableLambda to your chain to map inputs to the expected format instead. Example: def construct_chain(): my_chain = ... input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x} return input_mapper | my_chain run_on_dataset(..., llm_or_chain_factory=construct_chain) (See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r)r4r\rrZr[rrOptional[Client]rrrrrrrc K| dd} | rtdtd| !td} | dd} | rtddd| r(tdd | d d |p t }t|||||| | ||| | } tj | j d dgttjt| j| | j| j Rd{V}| ||S)Nr0.0.305Tmessagependingrr0.1.9qThe tags argument is deprecated and will be removed in a future release. Please specify project_metadata instead.PThe following arguments are deprecated and will be removed in a future release: rrremovalr[rr\rrrrr)rr_INPUT_MAPPER_DEP_WARNINGr!rqrrrrrunnable_utilsgather_with_concurrencyrmap functoolspartialrMrrrr)rYrrr4r\rrZr[rrrrr containerrs r?arun_on_datasetrs::nd33LT +DdSSSS466::=II ::fd # #D   U        {{}}         vxxF$,, )'-  I)@!  !233   "%.%<)                  M   M7  ; ;;r>c N| ddrtdtd| dd} | rtddd| !td} | r(tdd | d d |p t }t|||||| ||| | |d kr(fdtj j D} ntj j d 5} t| t!jt$jj j } dddn #1swxYwY| |S)NrrTrrrrrrrrrrcFg|]\}}t||jS)r)rXrr)rLrrrrs r?rNz"run_on_dataset..lsM     %.%<)       r>rr)rrrr!rqrrrrrrrr@get_executor_for_configrrrrrXrrr)rYrrr4r\rrZr[rrrrrrrrs @@r?run_on_datasetr9sM::nd33LT +DdSSSS ::fd # #D   U     466::=II     {{}}         vxxF$,, )'-  IA     $'y'99;L#M#M    4Y5Fq5I J J h  %)-6-D%1 &%  M                  M7  ; ;;s+AFF  F a1 Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to add to the project. Useful for storing information the test variant. (prompt version, model version, etc.) client: LangSmith client to use to access the dataset and to log feedback and run traces. verbose: Whether to print progress. tags: Tags to add to each run in the project. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see :func:`arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain_openai import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = smith_eval.RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", smith_eval.RunEvalConfig.Criteria("helpfulness"), smith_eval.RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = smith_eval.RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) zrun_on_dataset(zawait arun_on_dataset()r)rrrrrBr)rrrBr)rrrBrs)rr)rrrBr)rr)rr/rrrBr)rr)rrrrrBr) rrrrr4rrr(rBr)rrrrrBr)rrrrrBr)rrrrrBr)r r r r rrrr(rrrrrrrrrBr#) rrrrrrrrrBr)rrrrrr(rrrrrrrBr!)rrrrrrr*rrrr+r,rBr-)rr9rrr*rrrrrr+r,rBr:) rr)rrrrrrrBrB)rrrrr*rrrrrr+r,rBr-)rr9rrr*rrrrrr+r,rBrSr#)rYrrrrrrZrr[r,rrr\r]rBr^)rBr)rrCrBr)rYrrrrrr4rr\rrrrZrr[r,rrrrrrrBr)r< __future__rconcurrent.futuresr dataclassesrrloggingrnrrtypingrrrr r r r r rlangchain_core._apir langchain_core.callbacks.managerrlangchain_core.language_modelsrlangchain_core.messagesrrlangchain_core.outputsrrlangchain_core.runnablesrrrrr@rr!langchain_core.tracers.evaluationrr langchain_core.tracers.langchainrlangsmith.clientr langsmith.envr r!langsmith.evaluationr"r#r$rlangsmith.run_helpersr%r&langsmith.schemasr'r(r)r*r+langsmith.utilsr,requestsr-typing_extensionsr.langchain.chains.baser/langchain.evaluation.loadingr0langchain.evaluation.schemar1r2r3langchain.smithr4rlangchain.smith.evaluationrr5r6rnrw getLoggerr9rrsrrrIr8rArrrrrrrrrrrrrr rr7rArMrRrUrXr{r} dataclassrrrrrr_RUN_ON_DATASET_DOCSTRINGreplacer=r>r?rs DD"""""" ''''''''                      0/////666666<<<<<<CCCCCCCC88888888MMMMMMMMMM>>>>>><<<<<<=<<<<<######FFFFFFFFEDDDDDDDLLLLLLLLLLLLLL******''''''''''''777777 544444BBBBBB@@@@@@@@  8 $ $ Ruh' '( dVS[    HRuh// 02C CD33333y333F4F4F4F4F4F4F4F4R R R R R R R R R': : : : : z0 0 0 0 f     Y   $ $ $ $ PB""""JCCCC(<0.$@@@@F 4 4 4 4====N!%48)-======J!%48)-J59 333333|!%48)-;;;;;;F!%48)-J59 555555z26 $6::5:5:5:5:5z%         D      :,6:6:"&15!%><><><><><>