Ng(1ddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZe rddlmZejeZGd d e ZdS) ) annotationsN) defaultdict) nullcontext) TYPE_CHECKING)SentenceEvaluator)paraphrase_mining)SentenceTransformerc`eZdZdZ d*d+fd Z d,d-d(Zed)ZxZS).ParaphraseMiningEvaluatora Given a large set of sentences, this evaluator performs paraphrase (duplicate) mining and identifies the pairs with the highest similarity. It compare the extracted paraphrase pairs with a set of gold labels and computes the F1 score. Example: :: from datasets import load_dataset from sentence_transformers.SentenceTransformer import SentenceTransformer from sentence_transformers.evaluation import ParaphraseMiningEvaluator # Load a model model = SentenceTransformer('all-mpnet-base-v2') # Load the Quora Duplicates Mining dataset questions_dataset = load_dataset("sentence-transformers/quora-duplicates-mining", "questions", split="dev") duplicates_dataset = load_dataset("sentence-transformers/quora-duplicates-mining", "duplicates", split="dev") # Create a mapping from qid to question & a list of duplicates (qid1, qid2) qid_to_questions = dict(zip(questions_dataset["qid"], questions_dataset["question"])) duplicates = list(zip(duplicates_dataset["qid1"], duplicates_dataset["qid2"])) # Initialize the paraphrase mining evaluator paraphrase_mining_evaluator = ParaphraseMiningEvaluator( sentences_map=qid_to_questions, duplicates_list=duplicates, name="quora-duplicates-dev", ) results = paraphrase_mining_evaluator(model) ''' Paraphrase Mining Evaluation of the model on the quora-duplicates-dev dataset: Number of candidate pairs: 250564 Average Precision: 56.51 Optimal threshold: 0.8325 Precision: 52.76 Recall: 59.19 F1: 55.79 ''' print(paraphrase_mining_evaluator.primary_metric) # => "quora-duplicates-dev_average_precision" print(results[paraphrase_mining_evaluator.primary_metric]) # => 0.5650940787776353 NF順 dT sentences_mapdict[str, str]duplicates_listlist[tuple[str, str]]duplicates_dictdict[str, dict[str, bool]]add_transitive_closureboolquery_chunk_sizeintcorpus_chunk_size max_pairstop_kshow_progress_bar batch_sizenamestr write_csv truncate_dim int | Nonec tg|_g|_|D]9\}}|j||j|:| |_| |_| |_||_ ||_ ||_ ||_ | |_ ||ntd|_|0|D]-\}}||vr$||vr d|j||<d|j||<.|r||j|_t#}|jD]q}|j|D]a}||vr[||vrW|j||s|j||r1|t't)||gbrt+||_| rd| z} d| zdz|_gd|_| |_d|_dS) a Initializes the ParaphraseMiningEvaluator. Args: sentences_map (Dict[str, str]): A dictionary that maps sentence-ids to sentences. For example, sentences_map[id] => sentence. duplicates_list (List[Tuple[str, str]], optional): A list with id pairs [(id1, id2), (id1, id5)] that identifies the duplicates / paraphrases in the sentences_map. Defaults to None. duplicates_dict (Dict[str, Dict[str, bool]], optional): A default dictionary mapping [id1][id2] to true if id1 and id2 are duplicates. Must be symmetric, i.e., if [id1][id2] => True, then [id2][id1] => True. Defaults to None. add_transitive_closure (bool, optional): If true, it adds a transitive closure, i.e. if dup[a][b] and dup[b][c], then dup[a][c]. Defaults to False. query_chunk_size (int, optional): To identify the paraphrases, the cosine-similarity between all sentence-pairs will be computed. As this might require a lot of memory, we perform a batched computation. query_chunk_size sentences will be compared against up to corpus_chunk_size sentences. In the default setting, 5000 sentences will be grouped together and compared up-to against 100k other sentences. Defaults to 5000. corpus_chunk_size (int, optional): The corpus will be batched, to reduce the memory requirement. Defaults to 100000. max_pairs (int, optional): We will only extract up to max_pairs potential paraphrase candidates. Defaults to 500000. top_k (int, optional): For each query, we extract the top_k most similar pairs and add it to a sorted list. I.e., for one sentence we cannot find more than top_k paraphrases. Defaults to 100. show_progress_bar (bool, optional): Output a progress bar. Defaults to False. batch_size (int, optional): Batch size for computing sentence embeddings. Defaults to 16. name (str, optional): Name of the experiment. Defaults to "". write_csv (bool, optional): Write results to CSV file. Defaults to True. truncate_dim (Optional[int], optional): The dimension to truncate sentence embeddings to. `None` uses the model's current truncation dimension. Defaults to None. Nc*ttS)N)rrv/var/www/html/ai-engine/env/lib/python3.11/site-packages/sentence_transformers/evaluation/ParaphraseMiningEvaluator.pyz4ParaphraseMiningEvaluator.__init__..sbmnrbsbsr)T_paraphrase_mining_evaluationz _results.csv)epochsteps precisionrecallf1 thresholdaverage_precisionr4)super__init__ sentencesidsitemsappendr!rr rrrrr$r duplicatesrsetaddtuplesortedlentotal_num_duplicatescsv_file csv_headersr#primary_metric)selfrrrrrrrrrr r!r#r$idsentenceid1id2positive_key_pairskey1key2 __class__s r*r6z"ParaphraseMiningEvaluator.__init__As=^ )//11  LB N ! !( + + + HOOB     !2$ 0!2" (-<-H//kZsZsNtNt  &+ 5 5S-''C=,@,@04DOC(-04DOC(- " K"99$/JJDO UUO H HD- H HM)) --.t4.8<8Md8S.'**5t 1E1E+F+FGGG  H%((:$;$;!  :D;dB^S lll"1r)modelr output_pathr.r/returndict[str, float]c |dkr|dkrd|}n d|d|d}nd}|j|d|jdz }td |jd |d |jt n||j5t ||j|j|j |j |j |j |j }dddn #1swxYwYtd tt|zd x}}d } d x} x} } d } t!t|D]}||\}}}|j|}|j|}|dz }|j||s|j||rt|dz }||z }||jz }d|z|z||zz }| |z } || krG|} |} |} ||d |t)|dzt|dz d zdz } | |jz } td| dzdtd| dtd| dzdtd| dzdtd| dzdd||jrt,j||j}t,j|svt7|ddd5}t9j|}||j|||| | | | | gdddn #1swxYwYn[t7|ddd5}t9j|}|||| | | | | gdddn #1swxYwY| | | | | d}| ||j}|!|||S)NrNz after epoch z in epoch z after z stepsrz (truncated to )z1Paraphrase Mining Evaluation of the model on the z dataset:zNumber of candidate pairs: rzAverage Precision: rz.2fzOptimal threshold: z.4fz Precision: zRecall: zF1:  wzutf-8)newlinemodeencodinga)r4r2r0r1r3)"r$loggerinfor!rtruncate_sentence_embeddingsrr7rr rrrrr"r@ranger8r;rAminr#ospathjoinrBisfileopencsvwriterwriterowrCprefix_name_to_metrics store_metrics_in_model_card_data)rErOrPr.r/out_txt pairs_list n_extract n_correctr3best_f1 best_recallbest_precisionr4idxscoreijrHrIr0r1r2csv_pathfrimetricss r*__call__z"ParaphraseMiningEvaluator.__call__sZ B;;{{1%11BuBBUBBBG   ( =):=== =G e ee[beeefff#/7[]]]U=_=_`d`q=r=r  *&%&   J                 1CJ4H4HHIII!"! I 1222+Z)) l lC$S/KE1a(1+C(1+C NIs#C( lDOC,@,E lQ % 1 "T%>>]V+y6/AB!Y.!<< G%.N"(K!+C!3jS1WcR\oo`aNaAbAb6cde6f!fjk kI-0II G*;c*AGGGHHH 9)999::: <.3"6<<<=== 6{S0666777 07S=0000111  "t~ "w||K??H7>>(++ x(BS7KKKxq Z]]FOOD$4555OOUE>;PWYbdu$vwwwxxxxxxxxxxxxxxx (BS7KKKxq Z]]FOOUE>;PWYbdu$vwwwxxxxxxxxxxxxxxx "3'!"   --gtyAA --eW===s7 :CCCA M))M-0M-1OO  O c6t}t|D]g}||vr_t}||t||}t |dkr\|d}||vr0|||||t |dk\t|}tt |dz D]}t|dzt |D]f}d|||||<d|||||<||||||gi|S)NrrVT)r<listkeysr=r@popextendra)graph nodes_visitedr]connected_subgraph_nodesneighbor_nodes_queuenodervrws r*rz0ParaphraseMiningEvaluator.add_transitive_closures ejjll## G GA %%+.55((,,Q///(,E!H~~$.//!33/33A66D#;;;044T:::,33E$K@@@ .//!33,00H+I+I(s#;<rs""""""  ######"""""" PPPPPP888888NMMMMMM  8 $ $}}}}} 1}}}}}r)