
    Ng1;                        d dl mZ d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
Zd dlmZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZ erd d
lmZ  ej        e          Z G d de          ZdS )    )annotationsN)nullcontext)TYPE_CHECKINGLiteral)average_precision_score)paired_cosine_distancespaired_euclidean_distancespaired_manhattan_distances)SentenceEvaluator)InputExample)SimilarityFunction)SentenceTransformerc                       e Zd ZdZ	 	 	 	 	 	 d+d, fdZd-dZed.d            Z	 d/d0d%Zd1d'Z	e
d2d)            Ze
d2d*            Z xZS )3BinaryClassificationEvaluatora  
    Evaluate a model based on the similarity of the embeddings by calculating the accuracy of identifying similar and
    dissimilar sentences.
    The metrics are the cosine similarity, dot score, Euclidean and Manhattan distance
    The returned score is the accuracy with a specified metric.

    The results are written in a CSV. If a CSV already exists, then values are appended.

    The labels need to be 0 for dissimilar pairs and 1 for similar pairs.

    Args:
        sentences1 (List[str]): The first column of sentences.
        sentences2 (List[str]): The second column of sentences.
        labels (List[int]): labels[i] is the label for the pair (sentences1[i], sentences2[i]). Must be 0 or 1.
        name (str, optional): Name for the output. Defaults to "".
        batch_size (int, optional): Batch size used to compute embeddings. Defaults to 32.
        show_progress_bar (bool, optional): If true, prints a progress bar. Defaults to False.
        write_csv (bool, optional): Write results to a CSV file. Defaults to True.
        truncate_dim (Optional[int], optional): The dimension to truncate sentence embeddings to. `None` uses the model's current truncation dimension. Defaults to None.
        similarity_fn_names (Optional[List[Literal["cosine", "dot", "euclidean", "manhattan"]]], optional): The similarity functions to use. If not specified, defaults to the ``similarity_fn_name`` attribute of the model. Defaults to None.

    Example:
        ::

            from sentence_transformers import SentenceTransformer
            from sentence_transformers.evaluation import BinaryClassificationEvaluator
            from datasets import load_dataset

            # Load a model
            model = SentenceTransformer('all-mpnet-base-v2')

            # Load a dataset with two text columns and a class label column (https://huggingface.co/datasets/sentence-transformers/quora-duplicates)
            eval_dataset = load_dataset("sentence-transformers/quora-duplicates", "pair-class", split="train[-1000:]")

            # Initialize the evaluator
            binary_acc_evaluator = BinaryClassificationEvaluator(
                sentences1=eval_dataset["sentence1"],
                sentences2=eval_dataset["sentence2"],
                labels=eval_dataset["label"],
                name="quora_duplicates_dev",
            )
            results = binary_acc_evaluator(model)
            '''
            Binary Accuracy Evaluation of the model on the quora-duplicates-dev dataset:
            Accuracy with Cosine-Similarity:           81.60    (Threshold: 0.8352)
            F1 with Cosine-Similarity:                 75.27    (Threshold: 0.7715)
            Precision with Cosine-Similarity:          65.81
            Recall with Cosine-Similarity:             87.89
            Average Precision with Cosine-Similarity:  76.03
            '''
            print(binary_acc_evaluator.primary_metric)
            # => "quora_duplicates_dev_cosine_ap"
            print(results[binary_acc_evaluator.primary_metric])
            # => 0.760277070888393
         FTN
sentences1	list[str]
sentences2labels	list[int]namestr
batch_sizeintshow_progress_barbool	write_csvtruncate_dim
int | Nonesimilarity_fn_names?list[Literal['cosine', 'dot', 'euclidean', 'manhattan']] | Nonec
                   t                                                       || _        || _        || _        || _        |	pg | _        t          | j                  t          | j                  k    sJ t          | j                  t          | j                  k    sJ |D ]}
|
dk    s|
dk    sJ || _        || _	        || _
        |Nt                                          t          j        k    p&t                                          t          j        k    }|| _        d|rd|z   ndz   dz   | _        ddg| _        |                     | j                   d S )	Nr       binary_classification_evaluation_r   z_results.csvepochsteps)super__init__r   r   r   r   r!   lenr   r   r   loggergetEffectiveLevelloggingINFODEBUGr   csv_filecsv_headers_append_csv_headers)selfr   r   r   r   r   r   r   r   r!   label	__class__s              z/var/www/html/ai-engine/env/lib/python3.11/site-packages/sentence_transformers/evaluation/BinaryClassificationEvaluator.pyr*   z&BinaryClassificationEvaluator.__init__P   sX    	$$(#6#<" 4?##s4?';';;;;;4?##s4;'7'77777 	, 	,EA::!"	$$((**gl:if>V>V>X>X\c\i>i  "3:D>XcDjjVXY\jj#W-  !9:::::    returnNonec                `    g d}|D ]&}|D ]!}| j                             | d|            "'d S )N)accuracyaccuracy_thresholdf1	precisionrecallf1_thresholdapr&   )r2   append)r4   r!   metricsvms        r7   r3   z1BinaryClassificationEvaluator._append_csv_headersv   sf    
 
 
 % 	4 	4A 4 4 ''1

q

33334	4 	4r8   exampleslist[InputExample]c                    g }g }g }|D ]\}|                     |j        d                    |                     |j        d                    |                     |j                   ] | |||fi |S )Nr   r$   )rC   textsr5   )clsrG   kwargsr   r   scoresexamples          r7   from_input_examplesz1BinaryClassificationEvaluator.from_input_examples   s    

 	) 	)GgmA.///gmA.///MM'-((((s:z6<<V<<<r8   modelr   output_pathr'   r(   dict[str, float]c                T   |dk    r|dk    rd| }nd| d| d}nd}| j         |d| j          d	z  }t                              d
| j         d| d           | j        s'|j        g| _        |                     | j                   |                     |          ||g}| j        D ]A}d|v r;|	                    dd          \  }}	|
                    |         |	                    B|| j        rt          j                            || j                  }
t          j                            |
          sot#          |
ddd          5 }t%          j        |          }|                    | j                   |                    |           ddd           n# 1 swxY w Y   nTt#          |
ddd          5 }t%          j        |          }|                    |           ddd           n# 1 swxY w Y   d                                 D             }t-          | j                  dk    r/|                    fdd         D                        d| _        n| j        d          d| _        |                     || j                  }|                     ||           |S )a  
        Compute the evaluation metrics for the given model.

        Args:
            model (SentenceTransformer): The model to evaluate.
            output_path (str, optional): Path to save the evaluation results CSV file. Defaults to None.
            epoch (int, optional): The epoch number. Defaults to -1.
            steps (int, optional): The number of steps. Defaults to -1.

        Returns:
            Dict[str, float]: A dictionary containing the evaluation metrics.
        rP   z after epoch z
 in epoch z after z stepsr   Nz (truncated to )z/Binary Accuracy Evaluation of the model on the z dataset:r&   r$   )maxsplitwzutf-8)newlinemodeencodingac                X    i | ]'\  }}|                                 D ]\  }}| d | |(S )r&   )items).0
short_namevaluesmetricvalues        r7   
<dictcomp>z:BinaryClassificationEvaluator.__call__.<locals>.<dictcomp>   s_     
 
 
"
F!'
 
  $$F$$e
 
 
 
r8   c                R    i | ]"d  t          fdD                       #S )max_c              3  4   K   | ]}|                  V  d S N )r_   r`   rb   rM   s     r7   	<genexpr>zDBinaryClassificationEvaluator.__call__.<locals>.<dictcomp>.<genexpr>   s-      (](]

);F)C(](](](](](]r8   )max)r_   rb   rM   s    @r7   rd   z:BinaryClassificationEvaluator.__call__.<locals>.<dictcomp>   sS        $6OOS(](](](](]V\(](](]%]%]  r8   cosinemax_apr   _ap)r   r,   infor   r!   similarity_fn_namer3   compute_metricesr2   splitrC   r   ospathjoinr1   isfileopencsvwriterwriterowr^   r+   updateprimary_metricprefix_name_to_metrics store_metrics_in_model_card_data)r4   rQ   rR   r'   r(   out_txtfile_output_dataheader_namesim_fctrb   csv_pathfry   rD   rM   s                 @r7   __call__z&BinaryClassificationEvaluator.__call__   sq    B;;{{1%11BuBBUBBBG(=):====GcdiccY`cccddd' 	?(-(@'AD$$$T%=>>>&&u--!5>+ 	A 	AKk!!"-"3"3C!"3"D"D ''w(?@@@"t~"w||K??H7>>(++ 6(BS7KKK 6q Z]]FOOD$4555OO$45556 6 6 6 6 6 6 6 6 6 6 6 6 6 6
 (BS7KKK 6q Z]]FOO$45556 6 6 6 6 6 6 6 6 6 6 6 6 6 6
 
&,llnn
 
 

 t'((1,,NN   "("2     #+D%)%=a%@"E"E"ED--gtyAA--eW===s%   AFF!$F!<*G22G69G6dict[str, dict[str, float]]c                *   | j         t                      n|                    | j                   5  	 t          t	          | j        | j        z                       }|                    || j        | j	        d          }d t          ||          D             fd| j        D             }fd| j        D             }nx# t          $ rk |                    | j        | j        z   | j        | j	        d          }|d t          | j                           }|t          | j                  d          }Y nw xY wd d d            n# 1 swxY w Y   t          j        j        d dddt          j        j        d	 d
ddt          j        j        d dddt          j        j        d dddi}t'          j        | j                  }i }| j        D ]F}	||	         }
 |
d         ||          }|
d         }|
d         }|                     |||          \  }}|                     |||          \  }}}}t3          |||rdndz            }t4                              d| d|dz  dd|dd           t4                              d| d|dz  dd|dd           t4                              d| d|dz  d           t4                              d | d!|dz  d           t4                              d"| d#|dz  dd$           |||||||d%||	<   H|S )&NT)r   r   convert_to_numpyc                    i | ]\  }}||	S ri   ri   )r_   sentembs      r7   rd   zBBinaryClassificationEvaluator.compute_metrices.<locals>.<dictcomp>   s    RRR)$D#RRRr8   c                     g | ]
}|         S ri   ri   r_   r   emb_dicts     r7   
<listcomp>zBBinaryClassificationEvaluator.compute_metrices.<locals>.<listcomp>       JJJ$x~JJJr8   c                     g | ]
}|         S ri   ri   r   s     r7   r   zBBinaryClassificationEvaluator.compute_metrices.<locals>.<listcomp>   r   r8   c                (    dt          | |          z
  S )Nr$   )r   xys     r7   <lambda>z@BinaryClassificationEvaluator.compute_metrices.<locals>.<lambda>   s    -DQ-J-J)J r8   zCosine-Similarity)score_fnr   greater_is_betterc                |    t          j        t          j        |           t          j        |          z  d          S )NrP   )axis)npsumasarrayr   s     r7   r   z@BinaryClassificationEvaluator.compute_metrices.<locals>.<lambda>   s+    
1
10MTV)W)W)W r8   zDot-Productc                $    t          | |           S rh   )r
   r   s     r7   r   z@BinaryClassificationEvaluator.compute_metrices.<locals>.<lambda>       *DQ*J*J)J r8   zManhattan-DistanceFc                $    t          | |           S rh   )r	   r   s     r7   r   z@BinaryClassificationEvaluator.compute_metrices.<locals>.<lambda>  r   r8   zEuclidean-Distancer   r   r   r$   rP   zAccuracy with z:           d   z.2fz	(Threshold: z.4frU   zF1 with z:                 zPrecision with z:          zRecall with z:             zAverage Precision with z:  
)r<   r=   r>   rA   r?   r@   rB   )r   r   truncate_sentence_embeddingslistsetr   r   encoder   r   zip	TypeErrorr+   r   COSINErc   DOT_PRODUCT	MANHATTAN	EUCLIDEANr   r   r   r!   find_best_acc_and_thresholdfind_best_f1_and_thresholdr   r,   ro   )r4   rQ   	sentences
embeddingsembeddings1embeddings2similarity_fnsr   output_scoresrp   similarity_fnrM   r   r   accacc_thresholdr>   r?   r@   rA   rB   r   s                        @r7   rq   z.BinaryClassificationEvaluator.compute_metrices   s:   "/7[]]]U=_=_`d`q=r=r 	K 	KK !T_t%F!G!GHH	 #\\#&*&<%)	 *  
 SRs9j7Q7QRRRJJJJ$/JJJJJJJ$/JJJ'  	A 	A 	A"\\Odo5#&*&<%)	 *  
 ))?3t+?+?)?@(T_)=)=)?)?@	A	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K6 %+JJ+%). .
 *0WW%%)3 3
 (.JJ,%*1 1
 (.JJ,%*1 1!
. DK(("&": 	 	*+=>M.]:.{KHHF -.A B (D!%!A!A&&Rc!d!dC262Q2QRXZ`bs2t2t/B	6<(@Q;Y11WY1Z[[BKKlll39lllXellllmmmKKj4jj28jjjWcjjjjkkkKKP$PP9s?PPPQQQKKMtMM6C<MMMNNNKKK$KK28KKKKLLL  &3 ,& 1 1M,-- s6   E)C A$E A2D52E4D55EEEhigh_score_more_similarc                   t          |           t          |          k    sJ t          t          | |                    }t          |d |          }d}d}d}t	          |dk              }t          t          |          dz
            D ]^}||         \  }	}
|
dk    r|dz  }n|dz  }||z   t          |          z  }||k    r%|}||         d         ||dz            d         z   dz  }_||fS )Nc                    | d         S Nr   ri   r   s    r7   r   zKBinaryClassificationEvaluator.find_best_acc_and_threshold.<locals>.<lambda>-  
    !A$ r8   keyreverser   rP   r$      )r+   r   r   sortedr   range)rM   r   r   rowsmax_accbest_thresholdpositive_so_farremaining_negativesiscorer5   r   s               r7   r   z9BinaryClassificationEvaluator.find_best_acc_and_threshold(  s   6{{c&kk))))C''((d8OPPP!&A+..s4yy1}%% 
	C 
	CA7LE5zz1$#q(#"%88CKKGCW}}"&q'!*tAE{1~"=!B&&r8   c                L   t          |           t          |          k    sJ t          j        |           } t          j        |          }t          t	          | |                    }t          |d |          }dx}x}}d}d}d}	t          |          }
t          t          |          dz
            D ]j}||         \  }}|dz  }|dk    r|	dz  }	|	dk    rG|	|z  }|	|
z  }d|z  |z  ||z   z  }||k    r)|}|}|}||         d         ||dz            d         z   dz  }k||||fS )Nc                    | d         S r   ri   r   s    r7   r   zJBinaryClassificationEvaluator.find_best_f1_and_threshold.<locals>.<lambda>L  r   r8   r   r   r$   r   )r+   r   r   r   r   r   r   r   )rM   r   r   r   best_f1best_precisionbest_recall	thresholdnextractncorrecttotal_num_duplicatesr   r   r5   r?   r@   r>   s                    r7   r   z8BinaryClassificationEvaluator.find_best_f1_and_thresholdC  sa   6{{c&kk))))F##F##C''((d8OPPP1222.;	"6{{s4yy1}%% 	B 	BA7LE5MHzzA!||$x/	!$88]V+y6/AB<< G%.N"(K!%ad1q5k!n!< AIY>>r8   )r   r   FTNN)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   )r!   r   r9   r:   )rG   rH   )NrP   rP   )
rQ   r   rR   r   r'   r   r(   r   r9   rS   )rQ   r   r9   r   )r   r   )__name__
__module____qualname____doc__r*   r3   classmethodrO   r   rq   staticmethodr   r   __classcell__)r6   s   @r7   r   r      s       6 6z "'#'_c$; $; $; $; $; $; $;L4 4 4 4 	= 	= 	= [	= bdE E E E ENN N N N` ' ' ' \'4 !? !? !? \!? !? !? !? !?r8   r   )
__future__r   rx   r.   rs   
contextlibr   typingr   r   numpyr   sklearn.metricsr   sklearn.metrics.pairwiser   r	   r
   2sentence_transformers.evaluation.SentenceEvaluatorr   sentence_transformers.readersr   *sentence_transformers.similarity_functionsr   )sentence_transformers.SentenceTransformerr   	getLoggerr   r,   r   ri   r8   r7   <module>r      s:   " " " " " " 



  				 " " " " " " ) ) ) ) ) ) ) )     3 3 3 3 3 3 t t t t t t t t t t P P P P P P 6 6 6 6 6 6 I I I I I I NMMMMMM		8	$	$N? N? N? N? N?$5 N? N? N? N? N?r8   