wfhS#ddlmZddlZddlZddlmZddlmZmZm Z ddl m Z ddl m Z erddlZddlmZddlmZej(eZGd d e Zy) ) annotationsN) defaultdict) TYPE_CHECKINGAnyLiteral)BinaryClassificationEvaluator)append_to_last_row)Tensor) SparseEncoderceZdZdZ d d fd Zd fd Z d d fd Zdfd Z ddZ d ddZ dfd Z xZ S)#SparseBinaryClassificationEvaluatora This evaluator extends :class:`~sentence_transformers.evaluation.BinaryClassificationEvaluator` but is specifically designed for sparse encoder models. Evaluate a model based on the similarity of the embeddings by calculating the accuracy of identifying similar and dissimilar sentences. The metrics are the cosine similarity, dot score, Euclidean and Manhattan distance The returned score is the accuracy with a specified metric. The results are written in a CSV. If a CSV already exists, then values are appended. The labels need to be 0 for dissimilar pairs and 1 for similar pairs. Args: sentences1 (List[str]): The first column of sentences. sentences2 (List[str]): The second column of sentences. labels (List[int]): labels[i] is the label for the pair (sentences1[i], sentences2[i]). Must be 0 or 1. name (str, optional): Name for the output. Defaults to "". batch_size (int, optional): Batch size used to compute embeddings. Defaults to 32. show_progress_bar (bool, optional): If true, prints a progress bar. Defaults to False. write_csv (bool, optional): Write results to a CSV file. Defaults to True. max_active_dims (Optional[int], optional): The maximum number of active dimensions to use. `None` uses the model's current `max_active_dims`. Defaults to None. similarity_fn_names (Optional[List[Literal["cosine", "dot", "euclidean", "manhattan"]]], optional): The similarity functions to use. If not specified, defaults to the ``similarity_fn_name`` attribute of the model. Defaults to None. Example: :: import logging from datasets import load_dataset from sentence_transformers import SparseEncoder from sentence_transformers.sparse_encoder.evaluation import SparseBinaryClassificationEvaluator logging.basicConfig(format="%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) # Initialize the SPLADE model model = SparseEncoder("naver/splade-cocondenser-ensembledistil") # Load a dataset with two text columns and a class label column (https://huggingface.co/datasets/sentence-transformers/quora-duplicates) eval_dataset = load_dataset("sentence-transformers/quora-duplicates", "pair-class", split="train[-1000:]") # Initialize the evaluator binary_acc_evaluator = SparseBinaryClassificationEvaluator( sentences1=eval_dataset["sentence1"], sentences2=eval_dataset["sentence2"], labels=eval_dataset["label"], name="quora_duplicates_dev", show_progress_bar=True, similarity_fn_names=["cosine", "dot", "euclidean", "manhattan"], ) results = binary_acc_evaluator(model) ''' Accuracy with Cosine-Similarity: 75.00 (Threshold: 0.8668) F1 with Cosine-Similarity: 67.22 (Threshold: 0.5974) Precision with Cosine-Similarity: 54.18 Recall with Cosine-Similarity: 88.51 Average Precision with Cosine-Similarity: 67.81 Matthews Correlation with Cosine-Similarity: 49.56 Accuracy with Dot-Product: 76.50 (Threshold: 23.4236) F1 with Dot-Product: 67.00 (Threshold: 19.0095) Precision with Dot-Product: 55.93 Recall with Dot-Product: 83.54 Average Precision with Dot-Product: 65.89 Matthews Correlation with Dot-Product: 48.88 Accuracy with Euclidean-Distance: 67.70 (Threshold: -10.0041) F1 with Euclidean-Distance: 48.60 (Threshold: -0.1876) Precision with Euclidean-Distance: 32.13 Recall with Euclidean-Distance: 99.69 Average Precision with Euclidean-Distance: 20.52 Matthews Correlation with Euclidean-Distance: -4.59 Accuracy with Manhattan-Distance: 67.70 (Threshold: -103.0263) F1 with Manhattan-Distance: 48.60 (Threshold: -0.8532) Precision with Manhattan-Distance: 32.13 Recall with Manhattan-Distance: 99.69 Average Precision with Manhattan-Distance: 21.05 Matthews Correlation with Manhattan-Distance: -4.59 Model Sparsity: Active Dimensions: 61.2, Sparsity Ratio: 0.9980 ''' # Print the results print(f"Primary metric: {binary_acc_evaluator.primary_metric}") # => Primary metric: quora_duplicates_dev_max_ap print(f"Primary metric value: {results[binary_acc_evaluator.primary_metric]:.4f}") # => Primary metric value: 0.6781 c h||_tt|_t ||||||||| S)N) sentences1 sentences2labelsname batch_sizeshow_progress_bar write_csvsimilarity_fn_names)max_active_dimsrlistsparsity_statssuper__init__) selfrrrrrrrrr __class__s /home/chris/cleankitchens-env/lib/python3.12/site-packages/sentence_transformers/sparse_encoder/evaluation/SparseBinaryClassificationEvaluator.pyrz,SparseBinaryClassificationEvaluator.__init__osI /)$/w!!!/ 3  c^t|||jjddgy)N active_dimssparsity_ratio)r_append_csv_headers csv_headersextend)rrrs rr#z7SparseBinaryClassificationEvaluator._append_csv_headerss+ #$78 0@ ABrctt|_t|||||}|jj D])\}}t |t|z |j|<+|j|j|j|j|j||||tjd|jddd|jdd|\|jrPtt j"j%||j&|jd|jdg|S)N)model output_pathepochstepsz#Model Sparsity: Active Dimensions: r!z.1fz, Sparsity Ratio: r"z.4f)rrrr__call__itemssumlenupdateprefix_name_to_metricsr store_metrics_in_model_card_dataloggerinforr ospathjoincsv_file) rr'r(r)r*metricskeyvaluers rr+z,SparseBinaryClassificationEvaluator.__call__sE*$/'"Ku\a"b--335 ?JC'*5zCJ'>D   $ ? t2243F3F RS --eWeUK 1$2E2Em2TUX1YYklpllAQmRSVlW X   "t~~  [$--8$$]3T5H5HIY5Z[  rc$t||S)N)r')rcompute_metrices)rr'rs rr<z4SparseBinaryClassificationEvaluator.compute_metricessw'e'44rc |j|f|j|jdd|jd|}|j |}|j D]#\}}|j |j|%|S)NT)rrconvert_to_sparse_tensor save_to_cpur)encoderrrsparsityr,rappend)rr' sentenceskwargs embeddingsstatr9r:s r embed_inputsz0SparseBinaryClassificationEvaluator.embed_inputss "U\\  "44%) 00   ~~j)**, 3JC    $ + +E 2 3rcB|jj||||y)N)r)step)model_card_dataset_evaluation_metrics)rr'r8r)rIs rr1zDSparseBinaryClassificationEvaluator.store_metrics_in_model_card_datas" 44T7%VZ4[rcZt|}|j|j|d<|S)Nr)rget_config_dictr)r config_dictrs rrMz3SparseBinaryClassificationEvaluator.get_config_dicts3g-/    +-1-A-AK) *r) FTNN)r list[str]rrQrz list[int]rstrrintrboolrrTrz int | Nonerz?list[Literal['cosine', 'dot', 'euclidean', 'manhattan']] | None)rrQreturnNone)NrW) r'r r(z str | Noner)rSr*rSrUzdict[str, float])r'r rUzdict[str, dict[str, float]])r'r rCzstr | list[str] | np.ndarrayrUr )rr) r'r r8dict[str, Any]r)rSrIrSrUrV)rUrX) __name__ __module__ __qualname____doc__rr#r+r<rGr1rM __classcell__)rs@rr r s W|"'&*_c            $ ] 2C ce"1;KN\_ *50  *Z[\"\-;\DG\SV\ \ rr ) __future__rloggingr4 collectionsrtypingrrr sentence_transformers.evaluationrsentence_transformers.utilr numpynptorchr 2sentence_transformers.sparse_encoder.SparseEncoderr getLoggerrYr2r rrrjsL" #..J9P   8 $l*Glr