Ng9ddlZddlZddlmZmZmZmZmZddlm Z m Z ddl m Z ddl mZddlmZGdde ZdS) N)AnyDictListOptionalcast) AgentAction AgentFinish)BaseCallbackHandler) LLMResult)parsec 6eZdZUdZdZeed<edZeed<dZeed<dZ eed < d-d ed e ed e ede edd f fd Z de ee fdeede dd fdZdede dd fdZdede dd fdZdede dd fdZde ee fde ee fde dd fdZde ee fde dd fdZdede dd fdZde ee fd ede dd fd!Zd"ede de fd#Z d.d$e d%e ed&e ede dd f d'Zdede dd fd(Zd)ede dd fd*Zd+ede dd fd,ZxZ S)/ArgillaCallbackHandleraCallback Handler that logs into Argilla. Args: dataset_name: name of the `FeedbackDataset` in Argilla. Note that it must exist in advance. If you need help on how to create a `FeedbackDataset` in Argilla, please visit https://docs.argilla.io/en/latest/tutorials_and_integrations/integrations/use_argilla_callback_in_langchain.html. workspace_name: name of the workspace in Argilla where the specified `FeedbackDataset` lives in. Defaults to `None`, which means that the default workspace will be used. api_url: URL of the Argilla Server that we want to use, and where the `FeedbackDataset` lives in. Defaults to `None`, which means that either `ARGILLA_API_URL` environment variable or the default will be used. api_key: API Key to connect to the Argilla Server. Defaults to `None`, which means that either `ARGILLA_API_KEY` environment variable or the default will be used. Raises: ImportError: if the `argilla` package is not installed. ConnectionError: if the connection to Argilla fails. FileNotFoundError: if the `FeedbackDataset` retrieval from Argilla fails. Examples: >>> from langchain_community.llms import OpenAI >>> from langchain_community.callbacks import ArgillaCallbackHandler >>> argilla_callback = ArgillaCallbackHandler( ... dataset_name="my-dataset", ... workspace_name="my-workspace", ... api_url="http://localhost:6900", ... api_key="argilla.apikey", ... ) >>> llm = OpenAI( ... temperature=0, ... callbacks=[argilla_callback], ... verbose=True, ... openai_api_key="API_KEY_HERE", ... ) >>> llm.generate([ ... "What is the best NLP-annotation tool out there? (no bias at all)", ... ]) "Argilla, no doubt about it." z%https://github.com/argilla-io/argillaREPO_URLz/issues ISSUES_URLzphttps://docs.argilla.io/en/latest/tutorials_and_integrations/integrations/use_argilla_callback_in_langchain.htmlBLOG_URLzhttp://localhost:6900DEFAULT_API_URLN dataset_nameworkspace_nameapi_urlapi_keyreturnct ddl}|j|_n#t $rt dwxYwt |jt dkrt d|jd|8tjd$tj d|j d |j }|ftjd Rt |jt d krd nd |_ tj d|j d|j } | ||n.#t$r!}td|d|jd|d}~wwxYw||_|p||_ i}t |jt dkr'tj d|jdt(ddi}|jjd)|j|jd||_nF#t$r9}t1d|d|jd|jd|jd|jd |d}~wwxYwdd g}|d!|jjDkrBt7d"|jd|jd#|d$d%|jjDd&|jd' i|_tj d(|jddS)*a)Initializes the `ArgillaCallbackHandler`. Args: dataset_name: name of the `FeedbackDataset` in Argilla. Note that it must exist in advance. If you need help on how to create a `FeedbackDataset` in Argilla, please visit https://docs.argilla.io/en/latest/tutorials_and_integrations/integrations/use_argilla_callback_in_langchain.html. workspace_name: name of the workspace in Argilla where the specified `FeedbackDataset` lives in. Defaults to `None`, which means that the default workspace will be used. api_url: URL of the Argilla Server that we want to use, and where the `FeedbackDataset` lives in. Defaults to `None`, which means that either `ARGILLA_API_URL` environment variable or the default will be used. api_key: API Key to connect to the Argilla Server. Defaults to `None`, which means that either `ARGILLA_API_KEY` environment variable or the default will be used. Raises: ImportError: if the `argilla` package is not installed. ConnectionError: if the connection to Argilla fails. FileNotFoundError: if the `FeedbackDataset` retrieval from Argilla fails. rNzTo use the Argilla callback manager you need to have the `argilla` Python package installed. Please install it with `pip install argilla`z1.8.0z#The installed `argilla` version is z} but `ArgillaCallbackHandler` requires at least version 1.8.0. Please upgrade `argilla` with `pip install --upgrade argilla`.ARGILLA_API_URLz[Since `api_url` is None, and the env var `ARGILLA_API_URL` is not set, it will default to `z6`, which is the default API URL in Argilla Quickstart.ARGILLA_API_KEYz1.11.0z admin.apikeyz owner.apikeyz[Since `api_key` is None, and the env var `ARGILLA_API_KEY` is not set, it will default to `z6`, which is the default API key in Argilla Quickstart.)rrz.Could not connect to Argilla with exception: 'z'. Please check your `api_key` and `api_url`, and make sure that the Argilla server is up and running. If the problem persists please report it to z as an `integration` issue.1.14.0zYou have Argilla z., but Argilla 1.14.0 or higher is recommended. with_recordsF)name workspacez@`FeedbackDataset` retrieval from Argilla failed with exception `z+`. Please check that the dataset with name=z in the workspace=zz exists in advance. If you need help on how to create a `langchain`-compatible `FeedbackDataset` in Argilla, please visit z.. If the problem persists please report it to promptresponsecg|] }|j Sr.0fields j/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/callbacks/argilla_callback.py z3ArgillaCallbackHandler.__init__..sLLLu LLLz`FeedbackDataset` with name=z] had fields that are not supported yet for the`langchain` integration. Supported fields are: z/, and the current `FeedbackDataset` fields are cg|] }|j Sr"r#r$s r'r(z3ArgillaCallbackHandler.__init__..sAnAnAnQV%*AnAnAnr)zl. For more information on how to create a `langchain`-compatible `FeedbackDataset` in Argilla, please visit .zThe `ArgillaCallbackHandler` is currently in beta and is subject to change based on updates to `langchain`. Please report any issues to r")super__init__argilla __version__ARGILLA_VERSION ImportErrorr osgetenvwarningswarnrDEFAULT_API_KEYinit ExceptionConnectionErrorrr get_workspacer UserWarningFeedbackDataset from_argilladatasetFileNotFoundErrorrfields ValueErrorprompts) selfrrrrrge extra_argssupported_fields __class__s r'r-zArgillaCallbackHandler.__init__=sb<   #%>D    Y   % & &w 7 7Jd6JJJJ  ?ry):;;C M>151E>>>   *G ?ry):;;C-..x@@#  M>151E>>>   *G  GGGWG 5 5 5 5   !TTT(,TTT    ),B0@0@0B0B JT)**U8__<< .(<... -e4 :2-:&-DLL    #USTUU=A=NUU"1UU+/- UU )- UUU  %j1 LL 8KLLL L LPt/@PP&PPBRPPBoAnZ^ZfZmAnAnAnPP @D} PPP .0  AOAAA     s:4A1E E4E//E4A2H I4I  I serializedrBkwargsc t|jt|dp|d|idS)z.Save the prompts in memory when an LLM starts. parent_run_idrun_idN)rBupdatestr)rCrIrBrJs r' on_llm_startz#ArgillaCallbackHandler.on_llm_starts< S!8!LF8.sN# !&,(2(=(=(?(?##r)recordsr) rBrOzip generationsr> add_recordspopr r0push_to_argilla)rCr rJrBr^rs @r' on_llm_endz!ArgillaCallbackHandler.on_llm_ends / "  F,s6(#3445#&w0D#E#E   FK L $ $'2 %     VH-../// % & &x 8 8 L ( ( * * * * * 9 8r)errorc dS)z%Do nothing when LLM outputs an error.Nr"rCrcrJs r' on_llm_errorz#ArgillaCallbackHandler.on_llm_errorrTr)inputsc d|vrc|jt|dp|dt|dtr|dn|dgidSdS)aIf the key `input` is in `inputs`, then save it in `self.prompts` using either the `parent_run_id` or the `run_id` as the key. This is done so that we don't log the same input prompt twice, once when the LLM starts and once when the chain starts. inputrLrMN)rBrNrO isinstancelist)rCrIrgrJs r'on_chain_startz%ArgillaCallbackHandler.on_chain_starts f   L  /C6(3CDD%fWot<</w$Wo.        r)outputsc dtfdt|dt|dfDsdSjt|dp@t t jt|dg}|D]\}}t|tr4j dt||DNj dd || d igt|djvr-jt|dt|djvr-jt|dtjtd krj dSdS) zIf either the `parent_run_id` or the `run_id` is in `self.prompts`, then log the outputs to Argilla, and pop the run from `self.prompts`. The behavior differs if the output is a list or not. c3*K|] }|jvVdS)N)rB)r%keyrCs r' z6ArgillaCallbackHandler.on_chain_end.. s>   4<       r)rLrMNcRg|]$\}}d||ddi%S)r@rXrW)rY)r%routputs r'r(z7ArgillaCallbackHandler.on_chain_end..sS+FF %*0,26N,@,@,B,B''r)r[r@ rWr)anyrOrBgetrritemsrjrkr>r_r]joinrYr`r r0ra)rCrmrJrBchain_output_keychain_output_vals` r' on_chain_endz#ArgillaCallbackHandler.on_chain_endsS     F?344c&:J6K6KL       F ((VO-D)E)EFF $ $,""3vh'7#8#8"==K K 3:--//   . .*D11  ((/2';K.L.L)     ((%*-((7*;*;,<,B,B,D,D'')     vo& ' '4< 7 7 L  S!899 : : : vh DL 0 0 L  S!122 3 3 3 % & &x 8 8 L ( ( * * * * * 9 8r)c dS)z+Do nothing when LLM chain outputs an error.Nr"res r'on_chain_errorz%ArgillaCallbackHandler.on_chain_error8rTr) input_strc dS)zDo nothing when tool starts.Nr")rCrIr~rJs r' on_tool_startz$ArgillaCallbackHandler.on_tool_start<s  r)actionc dS)z.Do nothing when agent takes a specific action.Nr")rCrrJs r'on_agent_actionz&ArgillaCallbackHandler.on_agent_actionErTr)rsobservation_prefix llm_prefixc dS)zDo nothing when tool ends.Nr")rCrsrrrJs r' on_tool_endz"ArgillaCallbackHandler.on_tool_endIs  r)c dS)z&Do nothing when tool outputs an error.Nr"res r' on_tool_errorz$ArgillaCallbackHandler.on_tool_errorSrTr)rXc dSz Do nothingNr")rCrXrJs r'on_textzArgillaCallbackHandler.on_textWrTr)finishc dSrr")rCrrJs r'on_agent_finishz&ArgillaCallbackHandler.on_agent_finish[rTr))NNN)NN)!__name__ __module__ __qualname____doc__rrO__annotations__rrrrr-rrrrPrSr rb BaseExceptionrfrlr{r}rrrrrrr r __classcell__)rHs@r'rr s))Vrs 22222222222222::::::::888888,,,,,,######R R R R R 0R R R R R r)