Ng76 d dl Z d dlZd dlmZ d dlmZ d dlmZmZmZm Z m
Z
m Z d dl m
Z
d dlmZmZ d dlmZ d dlmZmZ d d lmZmZ G d
d e Zd e eef d
e
eef fdZ G d de Z dS ) N)datetime)Enum)AnyDictListOptionalTupleUnion)UUID) AgentAction AgentFinish)BaseCallbackHandler) BaseMessage ChatMessage)
Generation LLMResultc e Zd ZdZdZdZdS )LabelStudioModez Label Studio mode enumerator.promptchatN)__name__
__module__ __qualname____doc__PROMPTCHAT n/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/callbacks/labelstudio_callback.pyr r s ''
F DDDr r modereturnc t j j dt j j di}t | t
rt | } || j | fS )zGet default Label Studio configs for the given mode.
Parameters:
mode: Label Studio mode ("prompt" or "chat")
Returns: Tuple of Label Studio config and mode
a
a
)r r valuer
isinstancestr)r _default_label_configss r get_default_label_configsr' s^ $ ' * " % -' R $ % t$$ !$* -t 33r c e Zd ZU dZdZeed< dddedej fde e de e de e
ded e e d
e eef f fd
Z d ed
e
e
e ddfdZdeeef de
e deddfdZdedefdZdddddeeef de
e
e d ede e de e
e de eeef dedefdZd ededdfd Zd ededdfd Zd ededdfd!Zdeeef d"eeef deddfd#Z d$eeef deddfd%Z d ededdfd&Z deeef d'ededdfd(Z d)e dedefd*Z! d4d+ed,e e d-e e deddf
d.Z"d ededdfd/Z#d0ededdfd1Z$d2e%deddfd3Z& xZ'S )5LabelStudioCallbackHandlera Label Studio callback handler.
Provides the ability to send predictions to Label Studio
for human evaluation, feedback and annotation.
Parameters:
api_key: Label Studio API key
url: Label Studio URL
project_id: Label Studio project ID
project_name: Label Studio project name
project_config: Label Studio project config (XML)
mode: Label Studio mode ("prompt" or "chat")
Examples:
>>> from langchain_community.llms import OpenAI
>>> from langchain_community.callbacks import LabelStudioCallbackHandler
>>> handler = LabelStudioCallbackHandler(
... api_key='',
... url='http://localhost:8080',
... project_name='LangChain-%Y-%m-%d',
... mode='prompt'
... )
>>> llm = OpenAI(callbacks=[handler])
>>> llm.invoke('Tell me a story about a dog.')
zLangChain-%Y-%m-%dDEFAULT_PROJECT_NAMENapi_keyurl
project_id project_nameproject_configr c t dd l}n+# t $ r t d| j j d w xY w|s`t
j d r"t t
j d }n*t d| j j d| j j d || _
|sZt
j d rt
j d }n1t j d|j
d | j j d
|j
}|| _ i | _ | | j | j
| _ || _ |r|| _ d | _ n t+ | \ | _ | _ |pt
j d | _ | j 2| j t1 | j | _ nt5 j | j }| j |
} | r | d | _ | j j | _ n7| j || j | _ | j j | _ | j j | _ d\ | _! | _" | _# | _$ | j % D ]_\ }
} | d dk rN|
| _! | d d | _" | d d d | _# | d d d | _$ n`| j! s:d| j d} | j tL j' k r| dz
} n| dz
} t | d S )Nr z
You're using z in your code, but you don't have the LabelStudio SDK Python package installed or upgraded to the latest version. Please run `pip install -U label-studio-sdk` before using this callback.LABEL_STUDIO_API_KEYz in your code, Label Studio API key is not provided. Please provide Label Studio API key: go to the Label Studio instance, navigate to Account & Settings -> Access Token and copy the key. Use the key as a parameter for the callback: zr(label_studio_api_key='', ...) or set the environment variable LABEL_STUDIO_API_KEY=LABEL_STUDIO_URLz5Label Studio URL is not provided, using default URL: z8If you want to provide your own URL, use the parameter: zj(label_studio_url='', ...) or set the environment variable LABEL_STUDIO_URL=)r, r+ LABEL_STUDIO_PROJECT_ID)title)r4 label_config)NNNNtypeTextAreato_nameinputsr# zLabel Studio project "zI" does not have a TextArea tag. Please add a TextArea tag to the project.z
HINT: go to project Settings -> Labeling Interface -> Browse Templates and select "Generative AI -> Supervised Language Model Fine-tuning" template.z
HINT: go to project Settings -> Labeling Interface -> Browse Templates and check available templates under "Generative AI" section.)(super__init__label_studio_sdk ImportError __class__r osgetenvr%
ValueErrorr+ warningswarnLABEL_STUDIO_DEFAULT_URLr, payloadClient ls_clientr. r/ r r' r- get_projectint
ls_projectr todaystrftime get_projectsidcreate_projectparsed_label_config from_namer8 r#
input_typeitemsr r )selfr+ r, r- r. r/ r ls
project_titleexisting_projectstag_nametag_info
error_messager> s r r; z#LabelStudioCallbackHandler.__init__j s ) ) ) ) ) 0 7 0 0 0
y/00
bi(> ? ?@@ YDN$; Y Y ~.
Y Y Y
2y+,,
2i 233
X*,*EX X ~.X X X 1 )+ tx FF ( M"0D DII-Ft-L-L *D $L 2K(L(L ? & "n 8 8T_9M9M N NDOO $N , , 5 5d6G H HM $ ; ;- ; P P
5"3A"6"&/"4"&."?"?'d6I #@ # # #'/"4#'?#F E
A dj$/ #'":"@"@"B"B Hh :--!) ' 21 5 %h / 27 ;
"*8"4Q"7"? . ~ ,=): = = =
yO 222 G
/
]++ +) , ,s ( (Arun_id generationsr! c
g }| j | d }| j | d di d }t || D ]F\ }}| | j |d|i| j | j ddd |D id g|d
gd G| j | d S ) Npromptskwargsinvocation_params
model_namer[ textareatextc g | ] }|j
S r )rc ).0gs r
zFLabelStudioCallbackHandler.add_prompts_generations.. s 6R6R6R!qv6R6R6Rr )rQ r8 r6 r# )result
model_version)data predictions) rE getzipappendr# rQ r8 rJ import_tasks)rT r[ r\ tasksr^ ri r
generations r add_prompts_generationsz2LabelStudioCallbackHandler.add_prompts_generations s
,v&y1 L *
S$b
)
)
S
#&g{";"; FJ LL
F & 26 /3|,6.46R6Rz6R6R6R-S !" !"' .;
$
*
$$U+++++r
serializedr^ r_ c | j dk r t d| j d| j d t |d }||d| j |< dS ).Save the prompts in memory when an LLM starts.Text
Label Studio project "" has an input type . To make it work with the mode="chat", the input type should be .
Read more here https://labelstud.io/tags/textr[ )r^ r_ N)rR rA r. r% rE )rT rs r^ r_ r[ s r on_llm_startz'LabelStudioCallbackHandler.on_llm_start s ?f $ $ A4+< A A&*oA A A
VH%&&+2f E E V r messagec R t |t r|j S |j j S )z Get the role of the message.)r$ r roler> r )rT rz s r _get_message_rolez,LabelStudioCallbackHandler._get_message_role s( g{ + + .< $- -r )
parent_run_idtagsmetadatamessagesr~ r r c > | j dk r t d| j d| j d g }|D ]N} g }
| D ]2} |
| | | j d 3| |
O||||||d| j t | < dS )ru
Paragraphsrw rx z>. To make it work with the mode="chat", the input type should be .
Read more here https://labelstud.io/tags/paragraphs)r| content)r^ r r r[ r~ r_ N)rR rA r. rn r} r rE r% ) rT rs r r[ r~ r r r_ r^ message_listdialogrz s r on_chat_model_startz.LabelStudioCallbackHandler.on_chat_model_start s ?l * * G4+< G G&*oG G G
$ # #LF'
$ 6 6w ? ?#*?
NN6 " " " " *
%
%
S[[!!!r tokenc dS )z)Do nothing when a new token is generated.Nr )rT r r_ s r on_llm_new_tokenz+LabelStudioCallbackHandler.on_llm_new_tokenG r responsec t |d }| ||j | j | dS )z>Create a new Label Studio task for each prompt and generation.r[ N)r% rr r\ rE pop)rT r r_ r[ s r
on_llm_endz%LabelStudioCallbackHandler.on_llm_endK sM VH%&&
$$VX-ABBB
r errorc dS )z%Do nothing when LLM outputs an error.Nr rT r r_ s r on_llm_errorz'LabelStudioCallbackHandler.on_llm_errorU r r r9 c d S Nr )rT rs r9 r_ s r on_chain_startz)LabelStudioCallbackHandler.on_chain_startY s
r outputsc d S r r )rT r r_ s r on_chain_endz'LabelStudioCallbackHandler.on_chain_end^ s r c dS )z+Do nothing when LLM chain outputs an error.Nr r s r on_chain_errorz)LabelStudioCallbackHandler.on_chain_errora r r input_strc dS )z Do nothing when tool starts.Nr )rT rs r r_ s r
on_tool_startz(LabelStudioCallbackHandler.on_tool_starte s
r actionc dS )z.Do nothing when agent takes a specific action.Nr )rT r r_ s r on_agent_actionz*LabelStudioCallbackHandler.on_agent_actionn r r outputobservation_prefix
llm_prefixc dS )zDo nothing when tool ends.Nr )rT r r r r_ s r on_tool_endz&LabelStudioCallbackHandler.on_tool_endr s
r c dS )z&Do nothing when tool outputs an error.Nr r s r
on_tool_errorz(LabelStudioCallbackHandler.on_tool_error| r r rc c dS z
Do nothingNr )rT rc r_ s r on_textz"LabelStudioCallbackHandler.on_text r r finishc dS r r )rT r r_ s r on_agent_finishz*LabelStudioCallbackHandler.on_agent_finish r r )NN)(r r r r r* r% __annotations__r r r rI r
r; r r rr r r ry r r} r r r r r
BaseExceptionr r r r r r r r r r r
r
__classcell__)r> s @r r) r) N s} 2 !5#444 "& !$( 0(,,;,Bv, v,# v, c]v, SM v,
v, !
v, C ()v, v, v, v, v, v,p!,!,(,T*-=(>!,
!, !, !, !,FFcNF cF F
F F F F$. . . . . . )-$(-1'
'
'
cN'
tK()'
'
~
'
tCy!'
4S>*'
'
'
'
'
'
R
c
S
T
!9 ! ! ! ! ! !
-
3
4
sCx.
26sCx.
LO
DcN
c
d
M
S
T
cN
k
S
S
-1$(
%SM
SM
=
C
D
C
3
4
k
S
T
r r) ) r? rB r enumr typingr r r r r r
uuidr langchain_core.agentsr r
langchain_core.callbacksr langchain_core.messagesr r langchain_core.outputsr r r r% r' r) r r r r sy : : : : : : : : : : : : : : : : : : : : : : : : 8 8 8 8 8 8 < < < < < < < < 8 8 8 8 8 8 8 8 d 64
_$
%64
3 64 64 64 64rx
x
x
x
x
!4 x
x
x
x
x
r