Ng7*dZddlmZddlZddlZddlmZmZmZm Z m Z ddl m Z ddl mZddlmZddlmZdd lmZdd lmZmZmZdd lmZmZdd lmZddZddZGddeeeZ GddeeeZ!Gdde!Z"dS)z-LLM Chains for evaluating question answering.) annotationsN)AnyListOptionalSequenceTuple) Callbacks)BaseLanguageModel)PromptTemplate) ConfigDict)LLMChain)CONTEXT_PROMPT COT_PROMPTPROMPT) LLMEvalChainStringEvaluator)RUN_KEYtextstrreturnOptional[Tuple[str, int]]ctjd|tj}|rZ|ddkrdS|ddkrdS |dt ddtj }|dkrdS|dkrdS|d t ddtj }|dkrdS|dkrdSn#t$rYnwxYwdS) Nzgrade:\s*(correct|incorrect)CORRECT)rr INCORRECT)rrr) researchstrip IGNORECASEgroupuppersplit translater maketransstring punctuation IndexError)rmatch first_word last_words ^/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain/evaluation/qa/eval_chain.py _get_scorer.s I5tzz||R] S SE " ;;q>>   ! !Y . .< [[^^ ! ! # #{ 2 2!> JJLL    # - -cmmBFDV.W.W X X       * *<     ; . .!> JJLL UWWR  Ys}}RV-?@@ A A  ??   ) )< __  + - -!>.      4s&A;F? F?'A;F?$F?? G  G dictcn|}t|}|d\}}n|\}}|||dS)zParse the output text. Args: text (str): The output text to parse. Returns: Any: The parsed output. N)NN) reasoningvaluescore)r r.)rr1 parsed_scoresr2r3s r-_parse_string_eval_outputr51sP Iy))M! uu$ u  ceZdZUdZdZded<edZed.d Z e d/d Z e d.d Z e d.d Z e d0d1dZ d2d dd3d"Zd4d$Zd d d d%d&d5d,Zd d d d%d&d5d-Zd S)6 QAEvalChainz,LLM Chain for evaluating question answering.resultsr output_keyignoreextrarboolcdSNFclss r-is_lc_serializablezQAEvalChain.is_lc_serializablePur6cdS)N correctnessrAselfs r-evaluation_namezQAEvalChain.evaluation_nameTs}r6cdSNTrArHs r-requires_referencezQAEvalChain.requires_referenceXtr6cdSrLrArHs r-requires_inputzQAEvalChain.requires_input\rNr6Nllmr promptOptional[PromptTemplate]kwargsrc |pt}hd}|t|jkrtd|d|j|d||d|S)aLoad QA Eval Chain from LLM. Args: llm (BaseLanguageModel): the base language model to use. prompt (PromptTemplate): A prompt template containing the input_variables: 'input', 'answer' and 'result' that will be used as the prompt for evaluation. Defaults to PROMPT. **kwargs: additional keyword arguments. Returns: QAEvalChain: the loaded QA eval chain. >queryanswerresultInput variables should be , but got rQrRrA)rsetinput_variables ValueError)rCrQrRrTexpected_input_varss r-from_llmzQAEvalChain.from_llm`s,!6;;; #f&<"="= = =4-@44!144 s4s644V444r6rVrWrX callbacksexamplesSequence[dict] predictions question_key answer_keyprediction_keyrbr List[dict]ctfdt|D}|||S)5Evaluate question answering examples and predictions.cRg|]#\}}|||d$S)rVrWrXrA).0iexamplergrhrerfs r- z(QAEvalChain.evaluate..sR   7 !.!*-%a.8     r6ra enumerateapply)rIrcrerfrgrhrbinputss ```` r-evaluatezQAEvalChain.evaluate_       (11    zz&Iz666r6r/cvt||j}t|vr|t|t<|SNr5r:rrIrX parsed_results r-_prepare_outputzQAEvalChain._prepare_output51&2IJJ f  %+G_M' "r6F referenceinputrbinclude_run_info predictionr Optional[str]rrc P||||d||}||S)aEvaluate Chain or LLM output, based on optional input and label. Args: prediction (str): the LLM or chain prediction to evaluate. reference (Optional[str], optional): the reference label to evaluate against. input (Optional[str], optional): the input to consider during evaluation callbacks (Callbacks, optional): the callbacks to use for tracing. include_run_info (bool, optional): whether to include run info in the returned results. **kwargs: additional keyword arguments, including callbacks, tags, etc. Returns: dict: The evaluation results containing the score or value. rmrbrr}rIrrrrbrrTrXs r-_evaluate_stringszQAEvalChain._evaluate_stringssK0#$    -   ##F+++r6ctK||||d||d{V}||S)Nrmrurbracallr}rs r-_aevaluate_stringszQAEvalChain._aevaluate_stringssgzz"i:NN-"         ##F+++r6rr>rrry)rQr rRrSrTrrr8rm)rcrdrerdrfrrgrrhrrbr rrirXr/rr/rrrrrrrbr rr>rTrrr/)__name__ __module__ __qualname____doc__r:__annotations__r model_config classmethodrDpropertyrJrMrPr`rvr}rrrAr6r-r8r8Gs66J:L[XXX,05555[5D$"& 7 $777777,$(##!&!,!,!,!,!,!,N$(##!&,,,,,,,,r6r8ceZdZdZed.dZed.dZed.dZe dZ ed/d Z ed0dZ e d1d2dZ d3ddd4d"Zd5d$Zdddd%d&d6d,Zdddd%d&d6d-ZdS)7ContextQAEvalChainz3LLM Chain for evaluating QA w/o GT based on contextrr>cdSr@rArBs r-rDz%ContextQAEvalChain.is_lc_serializablerEr6cdS)z.Whether the chain requires a reference string.TrArHs r-rMz%ContextQAEvalChain.requires_reference tr6cdS)z+Whether the chain requires an input string.TrArHs r-rPz!ContextQAEvalChain.requires_inputrr6r;r<rRr Nonecrhd}|t|jkrtd|d|jdS)N>rVrXcontextrYrZ)r\r]r^)rCrRr_s r-_validate_input_varsz'ContextQAEvalChain._validate_input_varss`<<< #f&<"="= = =4-@44!144  > =r6rcdS)NzContextual AccuracyrArHs r-rJz"ContextQAEvalChain.evaluation_names$$r6NrQr rSrTrc T|pt}|||d||d|S)aLoad QA Eval Chain from LLM. Args: llm (BaseLanguageModel): the base language model to use. prompt (PromptTemplate): A prompt template containing the input_variables: 'query', 'context' and 'result' that will be used as the prompt for evaluation. Defaults to PROMPT. **kwargs: additional keyword arguments. Returns: ContextQAEvalChain: the loaded QA eval chain. r[rA)rrrCrQrRrTs r-r`zContextQAEvalChain.from_llms?,)>   (((s4s644V444r6rVrrXrarcrirerf context_keyrhrbr ctfdt|D}|||S)rkcRg|]#\}}|||d$S)rVrrXrA)rnrorprrhrerfs r-rqz/ContextQAEvalChain.evaluate..sR   7 !.";/%a.8     r6rarr)rIrcrerfrrhrbrus ```` r-rvzContextQAEvalChain.evaluate rwr6r/cvt||j}t|vr|t|t<|Sryrzr{s r-r}z"ContextQAEvalChain._prepare_output"r~r6Frrrrrrc P||||d||}||S)Nrrrrs r-rz$ContextQAEvalChain._evaluate_strings(sK$$    -   ##F+++r6ctK||||d||d{V}||S)Nrrrrs r-rz%ContextQAEvalChain._aevaluate_strings=sgzz"yJOO-"         ##F+++r6r)rRr rrrry)rQr rRrSrTrrrr)rcrirerirfrrrrhrrbr rrirr)rrrrrrDrrMrPr rrrJr`rvr}rrrAr6r-rrs==[XX:L[%%%X%,05555[5:$$& 7 $777777,$(##!&,,,,,,2$(##!&,,,,,,,,r6rc^eZdZdZeddZeddZe dddZdS)CotQAEvalChainz=LLM Chain for evaluating QA using chain of thought reasoning.rr>cdSr@rArBs r-rDz!CotQAEvalChain.is_lc_serializableRrEr6rcdS)NzCOT Contextual AccuracyrArHs r-rJzCotQAEvalChain.evaluation_nameVs((r6NrQr rRrSrTrc T|pt}|||d||d|S)zLoad QA Eval Chain from LLM.r[rA)rrrs r-r`zCotQAEvalChain.from_llmZs?%:   (((s4s644V444r6rrry)rQr rRrSrTrrr) rrrrrrDrrJr`rAr6r-rrOsGG[)))X),0 5 5 5 5[ 5 5 5r6r)rrrr)rrrr/)#r __future__rrr'typingrrrrr langchain_core.callbacks.managerr langchain_core.language_modelsr langchain_core.promptsr pydanticr langchain.chains.llmr #langchain.evaluation.qa.eval_promptrrrlangchain.evaluation.schemarrlangchain.schemarr.r5r8rrrAr6r-rs33"""""" 77777777777777666666<<<<<<111111))))))RRRRRRRRRREEEEEEEE$$$$$$:,F,F,F,F,F,(O\F,F,F,R|,|,|,|,|,?L|,|,|,~55555'55555r6