
    NgY                         d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZ d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dlZd d	lmZmZmZmZmZ d
ZdefdZ	 dde
e         de
e         defdZ dede!fdZ"dede!fdZ# G d dee          Z$dS )    N)deepcopy)Path)AnyCallableDictListOptionalSequence)AgentActionAgentFinish)BaseCallbackHandler)
Generation	LLMResultguard_import)BaseMetadataCallbackHandlerflatten_dictimport_pandasimport_spacyimport_textstatzlangchain-modelreturnc                       t          d          S )z:Import comet_ml and raise an error if it is not installed.comet_mlr        k/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/callbacks/comet_ml_callback.pyimport_comet_mlr      s    
###r   	workspaceproject_namec                 P    t                      }|                    | |          }|S )N)r   r   )r   
Experiment)r   r   r   
experiments       r   _get_experimentr#      s7       H$$! %  J
 r   textc                    t                      }i d|                    |           d|                    |           d|                    |           d|                    |           d|                    |           d|                    |           d|                    |           d|                    |           d	|	                    |           d
|
                    |           d|                    |           d|                    |           d|                    |           d|                    |           d|                    |           d|                    |           }|S )Nflesch_reading_easeflesch_kincaid_grade
smog_indexcoleman_liau_indexautomated_readability_indexdale_chall_readability_scoredifficult_wordslinsear_write_formulagunning_fogtext_standardfernandez_huertaszigriszt_pazosgutierrez_polinicrawfordgulpease_indexosman)r   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   )r$   textstattext_complexity_metricss      r   _fetch_text_complexity_metricsr8   )   s     Hx;;DAA = =d C C 	h))$// 	h99$??	
 	&x'K'KD'Q'Q 	'(M(Md(S(S 	833D99 	 !?!?!E!E 	x++D11 	//55 	H55d;; 	833D99 	H55d;; 	H%%d++ 	(11$77  	%%!$ #"r   metricsc                     t                      }|                    |           }|                                }|                                S N)r   	DataFramedescribeto_dict)r9   pd
metrics_dfmetrics_summarys       r   (_summarize_metrics_for_generated_outputsrB   @   s?    	Bg&&J ))++O""$$$r   c                   H    e Zd ZdZ	 	 	 	 	 	 	 	 	 dBdee         dee         dee         d	ee         d
ee         deee                  dedee	         deddf fdZ
defdZdeeef         dee         deddfdZdededdfdZdededdfdZdededdfdZdeeef         deeef         deddfdZdeeef         deddfdZdededdfd Zdeeef         d!ededdfd"Zd#ededdfd$Zdededdfd%Zd&ededdfd'Zd(ededdfd)Zd*ededefd+Zd&edefd,Z d-e!d.e"d/e"defd0Z#	 	 	 	 	 	 	 	 	 	 	 dCd2edee         dee         dee         d	ee         d
ee         deee                  dedee	         d(ed3eddfd4Z$d5ed6ed7e"ddfd8Z%d2eddfd9Z&dDd2ee         ddfd:Z'd;ee         d7e"ddfd<Z(d=eddfd>Z)	 	 	 	 	 	 	 	 dEdee         dee         dee         d	ee         d
ee         deee                  dedee	         ddfd?Z*dDd2edefd@Z+dDd2edefdAZ, xZ-S )FCometCallbackHandlera  Callback Handler that logs to Comet.

    Parameters:
        job_type (str): The type of comet_ml task such as "inference",
            "testing" or "qc"
        project_name (str): The comet_ml project name
        tags (list): Tags to add to the task
        task_name (str): Name of the comet_ml task
        visualize (bool): Whether to visualize the run.
        complexity_metrics (bool): Whether to log complexity metrics
        stream_logs (bool): Whether to stream callback actions to Comet

    This handler will utilize the associated callback method and formats
    the input of each callback function with metadata regarding the state of LLM run,
    and adds the response to the list of records for both the {method}_records and
    action. It then logs the response to Comet.
    	inferenceNFT	task_typer   r   tagsnamevisualizationscomplexity_metricscustom_metricsstream_logsr   c
                    t                      | _        t                                                       || _        || _        || _        || _        || _        || _	        || _
        |	| _        t          j                    | _        t          ||          | _        | j                            dd           |r| j                            |           || _        | j        r| j                            | j                   d}
| j        j                            |
           g | _        g | _        || _	        | j        r*t3                      }|                    d          | _        dS d| _        dS )zInitialize callback handler.zCreated from	langchainzThe comet_ml callback is currently in beta and is subject to change based on updates to `langchain`. Please report any issues to https://github.com/comet-ml/issue-tracking/issues with the tag `langchain`.en_core_web_smN)r   r   super__init__rF   r   r   rG   rI   rJ   rK   rL   tempfileTemporaryDirectorytemp_dirr#   r"   	log_otheradd_tagsrH   set_nameLOGGERwarningcallback_columnsaction_recordsr   loadnlp)selfrF   r   r   rG   rH   rI   rJ   rK   rL   rY   spacy	__class__s               r   rQ   zCometCallbackHandler.__init__[   sY    ())""(	,"4,& 355))\BB!!.+>>> 	+O$$T***	9 	0O$$TY/// 	 	$$W---&($&"4 	 NNEzz"233DHHHDHHHr   c                 $    d | j         D             S )Nc                     i | ]}|d S r;   r   ).0ks     r   
<dictcomp>z3CometCallbackHandler._init_resp.<locals>.<dictcomp>   s    777A4777r   )rZ   )r^   s    r   
_init_respzCometCallbackHandler._init_resp   s    77!67777r   
serializedpromptskwargsc                 .   | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |                                 }|                    ddi           |                    t          |                     |                    |                                            |D ]m}t          |          }||d<   | j        	                    |           | j
        	                    |           | j        r|                     ||| j                    ndS )zRun when LLM starts.   actionon_llm_startrh   N)step
llm_startsstartsrf   updater   get_custom_callback_metar   on_llm_start_recordsappendr[   rL   _log_stream)r^   rg   rh   ri   metadatapromptprompt_resps          r   rm   z!CometCallbackHandler.on_llm_start   s    			Q		1q??$$>2333Z001115577888 	> 	>F"8,,K%+K	"%,,[999&&{333 >  49===	> 	>r   tokenc                     | xj         dz  c_         | xj        dz  c_        |                                 }|                    d|d           |                    |                                            | j                            |           dS )z#Run when LLM generates a new token.rk   on_llm_new_token)rl   ry   N)rn   llm_streamsrf   rq   rr   r[   rt   )r^   ry   ri   resps       r   r{   z%CometCallbackHandler.on_llm_new_token   s    		Q		A  1EBBCCCD1133444""4(((((r   responsec                    | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |                                 }|                    ddi           |                    t          |j        pi                      |                    |                                            g }g }t          |j	                  D ]B\  }}t          |          D ]+\  }}	|	j
        }
t          |          }|                    t          |	                                                     |                     |
          }|r*|                    |           |                    |           |                     |	||          }|r*|                    |           |                    |           | j        r|                     |
|| j                    | j                            |           | j                            |           -D|                     || j                    |                     || j                    dS )zRun when LLM ends running.rk   rl   
on_llm_end)rn   N)rn   llm_endsendsrf   rq   r   
llm_outputrr   	enumerategenerationsr$   r   dict_get_complexity_metricsrt   _get_custom_metricsrL   ru   r[   on_llm_end_records_log_text_metrics)r^   r~   ri   rv   output_complexity_metricsoutput_custom_metrics
prompt_idxr   gen_idx
generationr$   generation_resprJ   rK   s                 r   r   zCometCallbackHandler.on_llm_end   sL   		Q				Q		??$$<0111X%8%>B??@@@5577888$&! "'01E'F'F 	@ 	@#J'0'='= @ @#!"*8"4"4&&|JOO4E4E'F'FGGG%)%A%A$%G%G"% ?-445GHHH#**+=>>>!%!9!9
G" " " ;)00@@@#**>:::# @$$T8TY???#**?;;;'..????-@0 	8tyIII449EEEEEr   errorc                 F    | xj         dz  c_         | xj        dz  c_        dS )zRun when LLM errors.rk   Nrn   errorsr^   r   ri   s      r   on_llm_errorz!CometCallbackHandler.on_llm_error   %    		Q		qr   inputsc                    | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |                                 }|                    ddi           |                    t          |                     |                    |                                            |                                D ]\  }}t          |t                    rdt          |          }| j        r|                     ||| j                    |                    ||i           | j                            |           ~| j        j                            d| d           dS )zRun when chain starts running.rk   rl   on_chain_startz1Unexpected data format provided! Input Value for  will not be loggedN)rn   chain_startsrp   rf   rq   r   rr   items
isinstancestrr   rL   ru   r[   rt   r   rX   rY   )r^   rg   r   ri   r}   chain_input_keychain_input_val
input_resps           r   r   z#CometCallbackHandler.on_chain_start   sq    			Q		Qq  X/0111L,,---D113344406 	 	,O_/3// %d^^
# G$$_dDIFFF!!?O"DEEE#**:6666 $,,L'6L L L   	 	r   outputsc                 v   | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |                                 }|                    ddi           |                    |                                            |                                D ]\  }}t          |t                    rdt          |          }| j
        r|                     ||| j                    |                    ||i           | j                            |           ~| j        j                            d| d           dS )zRun when chain ends running.rk   rl   on_chain_endz2Unexpected data format provided! Output Value for r   N)rn   
chain_endsr   rf   rq   rr   r   r   r   r   rL   ru   r[   rt   r   rX   rY   )r^   r   ri   r}   chain_output_keychain_output_valoutput_resps          r   r   z!CometCallbackHandler.on_chain_end   sS   		Q		1		Q		  X~.///D113344429--// 	 	..*C00 
&tnn# H$$%5tTYGGG""$46F#GHHH#**;7777$,,N(8N N N   	 	r   c                 F    | xj         dz  c_         | xj        dz  c_        dS )zRun when chain errors.rk   Nr   r   s      r   on_chain_errorz#CometCallbackHandler.on_chain_error  r   r   	input_strc                    | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |                                 }|                    ddi           |                    t          |                     |                    |                                            | j        r|                     ||| j                    |                    d|i           | j	        
                    |           dS )zRun when tool starts running.rk   rl   on_tool_startr   N)rn   tool_startsrp   rf   rq   r   rr   rL   ru   r[   rt   )r^   rg   r   ri   r}   s        r   r   z"CometCallbackHandler.on_tool_start  s     			Q		Aq  X/000L,,---D1133444 	9Ydi888[),---""4(((((r   outputc                    t          |          }| xj        dz  c_        | xj        dz  c_        | xj        dz  c_        |                                 }|                    ddi           |                    |                                            | j        r|                     ||| j                   |                    d|i           | j	        
                    |           dS )zRun when tool ends running.rk   rl   on_tool_endr   N)r   rn   	tool_endsr   rf   rq   rr   rL   ru   r[   rt   )r^   r   ri   r}   s       r   r   z CometCallbackHandler.on_tool_end+  s    V		Q		!		Q		  X}-...D1133444 	6VT49555Xv&'''""4(((((r   c                 F    | xj         dz  c_         | xj        dz  c_        dS )zRun when tool errors.rk   Nr   r   s      r   on_tool_errorz"CometCallbackHandler.on_tool_error;  r   r   r$   c                    | xj         dz  c_         | xj        dz  c_        |                                 }|                    ddi           |                    |                                            | j        r|                     ||| j                    |                    d|i           | j                            |           dS )z+
        Run when agent is ending.
        rk   rl   on_textr$   N)	rn   text_ctrrf   rq   rr   rL   ru   r[   rt   )r^   r$   ri   r}   s       r   r   zCometCallbackHandler.on_text@  s     			Q		  Xy)***D1133444 	4T4333VTN###""4(((((r   finishc                    | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |                                 }|j        d         }|j        }|                    d|d           |                    |                                            | j        r| 	                    ||| j                    |                    d|i           | j
                            |           dS )zRun when agent ends running.rk   r   on_agent_finish)rl   logN)rn   
agent_endsr   rf   return_valuesr   rq   rr   rL   ru   r[   rt   )r^   r   ri   r}   r   r   s         r   r   z$CometCallbackHandler.on_agent_finishP  s    		Q		1		Q		  %h/j0==>>>D1133444 	6VT49555Xv&'''""4(((((r   rl   c                    | xj         dz  c_         | xj        dz  c_        | xj        dz  c_        |j        }t	          |j                  }|j        }|                                 }|                    d||d           |                    | 	                                           | j
        r|                     ||| j                    |                    d|i           | j                            |           dS )zRun on agent action.rk   on_agent_action)rl   r   tool
tool_inputN)rn   r   rp   r   r   r   r   rf   rq   rr   rL   ru   r[   rt   )r^   rl   ri   r   r   r   r}   s          r   r   z$CometCallbackHandler.on_agent_actionb  s    		Q		Aq{*++
j  0dKKLLLD1133444 	:Zty999\:.///""4(((((r   c                 `    i }| j         r$t          |          }|                    |           |S )zCompute text complexity metrics using textstat.

        Parameters:
            text (str): The text to analyze.

        Returns:
            (dict): A dictionary containing the complexity metrics.
        )rJ   r8   rq   )r^   r$   r}   r7   s       r   r   z,CometCallbackHandler._get_complexity_metricsu  s:     " 	1&DT&J&J#KK/000r   r   r   r   c                 p    i }| j         r,|                      |||          }|                    |           |S )aW  Compute Custom Metrics for an LLM Generated Output

        Args:
            generation (LLMResult): Output generation from an LLM
            prompt_idx (int): List index of the input prompt
            gen_idx (int): List index of the generated output

        Returns:
            dict: A dictionary containing the custom metrics.
        )rK   rq   )r^   r   r   r   r}   rK   s         r   r   z(CometCallbackHandler._get_custom_metrics  sB      	(!00ZQQNKK'''r   comet-langchain-demolangchain_assetresetc           
      @   |                      |           |rK	 |                     |           n4# t          $ r' | j        j                            ddddi           Y nw xY w|
r| j                                         |r|                     ||||||||	           dS dS )aU  Flush the tracker and setup the session.

        Everything after this will be a new table.

        Args:
            name: Name of the performed session so far so it is identifiable
            langchain_asset: The langchain asset to save.
            finish: Whether to finish the run.

            Returns:
                None
        z&Failed to export agent or LLM to CometTshow_tracebackexc_infoextraN)	_log_session
_log_model	Exceptionr   rX   r   r"   end_reset)r^   r   rF   r   r   rG   rH   rI   rJ   rK   r   r   s               r   flush_trackerz"CometCallbackHandler.flush_tracker  s    4 	/*** 	0000   $**<!+T2 +       	"O!!! 
	KK"	 	 	 	 	
	 
	s   / .A A rw   rv   rn   c                 @    | j                             |||           d S )N)rv   rn   )r"   log_text)r^   rw   rv   rn   s       r   ru   z CometCallbackHandler._log_stream  s%      ( FFFFFr   c                    |                      |          }| j                            |d           t          | j        j        d          }| j        r| j        nt          }	 t          |d          r?|                    |           | j        	                    |t          |                     d S d S # t          t          t          f$ r}t          |d          r>|                    |           | j        	                    |t          |                     n4| j        j                            | d|j        j                    Y d }~d S Y d }~d S d }~ww xY w)Nmodel)prefixz
model.jsonsave
save_agentz$ Could not save Langchain Asset for )_get_llm_parametersr"   log_parametersr   rT   rH   LANGCHAIN_MODEL_NAMEhasattrr   	log_modelr   
ValueErrorAttributeErrorNotImplementedErrorr   r   rX   r   r`   __name__)r^   r   model_parameterslangchain_asset_path
model_namees         r   r   zCometCallbackHandler._log_model  s   33ODD&&'7&HHH#DM$6EE"&)ETYY1E
	// Q$$%9:::))*c:N6O6OPPPPPQ Q N,?@ 		 		 		55 **+?@@@))*c:N6O6OPPPP$** @ @*4=@ @         QPPPPP		s   "AB3 3E
A<EEc                 N   	 |                      |          }| j                            d|           n4# t          $ r' | j        j                            ddddi           Y nw xY w	 dt          t          j	                  i}| j        
                    | j        d|           n4# t          $ r' | j        j                            ddddi           Y nw xY w	 |                     |           d S # t          $ r( | j        j                            d	dddi           Y d S w xY w)
Nzlangchain-llm-session.csvz#Failed to log session data to CometTr   r   langchain_versionzlangchain-action_records.json)rv   z%Failed to log visualizations to Comet)"_create_session_analysis_dataframer"   	log_tabler   r   rX   rY   r   langchain_community__version__log_asset_datar[   _log_visualizations)r^   r   llm_session_dfrv   s       r   r   z!CometCallbackHandler._log_session  s   		!DD_UUNO%%&A>RRRR 	 	 	M ((5'. )     		+S1D1P-Q-QRHO**#%Dx +      	 	 	M ((5'. )     		$$^44444 	 	 	M ((7'. )      	s3   03 .A$#A$(=B& &.CCC2 2.D$#D$r9   c                     |sd S t          |          }|                                D ]"\  }}| j                            |||           #d S )N)r   rn   )rB   r   r"   log_metrics)r^   r9   rn   rA   keyvalues         r   r   z&CometCallbackHandler._log_text_metrics
  si     	FB7KK)//11 	F 	FJCO''c'EEEE	F 	Fr   
session_dfc           
      d   | j         r| j        sd S t                      }|d                                         }|d                                         }t	          t          ||                    D ]\  }\  }}|                     |          }t          |j                  }	| j         D ]}
	 |j        	                    |	|
ddidd          }| j
                            |d|
 d| d	d
|i|           L# t          $ r.}| j        j                            |dddi           Y d }~d }~ww xY wd S )Nrh   r$   compactTF)styleoptionsjupyterpagezlangchain-viz--z.htmlrw   )rH   rv   rn   r   r   )rI   r]   r   tolistr   ziplistsentsdisplacyrenderr"   r   r   r   rX   rY   )r^   r   r_   rh   r   idxrw   r   docsentence_spansvisualizationhtmlr   s                r   r   z(CometCallbackHandler._log_visualizations  s   # 	 	FY'..00V$++--%.s7G/D/D%E%E 	 	!C!&&((6""C!#)__N!%!4   >00&+!*D 1 %! 1  D O22HmHHcHHH"*F!3 	 3     !   M(00D1A40H 1        ( 	s   +AC44
D,>$D''D,c	           
      H   |r|n| j         }	|r|n| j        }
|r|n| j        }|r|n| j        }|r|n| j        }|r|n| j        }|r|n| j        }|r|n| j        }|                     |	|
||||||           | 	                                 t          j                    | _        d S )N)rF   r   r   rG   rH   rI   rJ   rK   )rF   r   r   rG   rH   rI   rJ   rK   rQ   reset_callback_metarR   rS   rT   )r^   rF   r   r   rG   rH   rI   rJ   rK   
_task_type
_workspace_project_name_tags_name_visualizations_complexity_metrics_custom_metricss                    r   r   zCometCallbackHandler._reset5  s     #,?YY
"+?YY
(4K$:K+$)+$),:S..@S"4Q$:Q 	 -;S..@S  &*2* 	 		
 		
 		
 	  """ 355r   c                    t                      }|                     |          }|                    dd          }|                    | j                  }|j        |j                            |                                       d          }|                    | j	                  }|
                    ||ddddg          }|S )Nnrk   T)drop
_llm_start_llm_end)
left_indexright_indexsuffixes)r   r   getr<   rs   locindexrepeatreset_indexr   merge)r^   r   r?   llm_parametersnum_generations_per_promptllm_start_records_dfllm_end_records_dfr   s           r   r   z7CometCallbackHandler._create_session_analysis_dataframeY  s    __11/BB%3%7%7Q%?%?"!||D,EFF37 &--.HII 

+4+
 
  	  \\$*ABB "J/ " 
 
 r   c                 x   |si S 	 t          |d          r$|j        j        j                                        }nmt          |d          r|j        j                                        }n>t          |d          r|j                                        }n|                                }n# t
          $ r i cY S w xY w|S )Nagent	llm_chainllm)r   r$  r%  r&  r   r   )r^   r   r  s      r   r   z(CometCallbackHandler._get_llm_parametersp  s     	I
	00 8!0!6!@!D!I!I!K!K+66 8!0!:!>!C!C!E!E%00 8!0!4!9!9!;!;!0!5!5!7!7 	 	 	III	 s   B!B( (B76B7)	rE   NNNNNFNT)NrE   Nr   NNNFNFFr;   )NNNNNNFN).r   
__module____qualname____doc__r	   r   r
   r   boolr   rQ   r   rf   r   rm   r{   r   r   BaseExceptionr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   intr   r   ru   r   r   r   r   r   r   r   __classcell__)r`   s   @r   rD   rD   H   s{        ( $/#'&*#'".2#(-1 2 2C=2 C=2 sm	2
 x 2 sm2 !c+2 !2 !*2 2 
2 2 2 2 2 2h8D 8 8 8 8>sCx.>379>HK>	> > > >,	)c 	)S 	)T 	) 	) 	) 	)(F9 (F (F (F (F (F (FT- 3 4    
sCx.26sCx.LO	   6DcN c d    .M S T    
)sCx.)58)DG)	) ) ) )$)# ) ) ) ) ) ) = C D    
)C )3 )4 ) ) ) ) )k )S )T ) ) ) )$)k )S )S ) ) ) )&C D     $25@C	   .  $#.#'&<#'".2#(-13 33 C=3 C=	3
 sm3 x 3 sm3 !c+3 !3 !*3 3 3 
3 3 3 3jG# G GS GT G G G G# $    ,   HSM  T        DF$ Fs Ft F F F F!c !d ! ! ! !J $(#'&*#'".2#(-1"6 "6C="6 C="6 sm	"6
 x "6 sm"6 !c+"6 !"6 !*"6 
"6 "6 "6 "6H # QU    . 3 $        r   rD   )NN)%rR   copyr   pathlibr   typingr   r   r   r   r	   r
   langchain_core.agentsr   r   langchain_core.callbacksr   langchain_core.outputsr   r   langchain_core.utilsr   r   #langchain_community.callbacks.utilsr   r   r   r   r   r   r   r   r#   r   r8   rB   rD   r   r   r   <module>r6     s                @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ : : : : : : : : 8 8 8 8 8 8 8 8 8 8 8 8 8 8 - - - - - -                  ) $ $ $ $ $ DH
 
}
3;C=

 
 
 
# # # # # #.%h %4 % % % %w w w w w68K w w w w wr   