
    wfhZ$                        d dl mZ d dlZd dlZd dlmZ d dlmZmZm	Z	 d dl
Z
d dlmZ d dlmZmZ er
d dlZd dlmZ  ej(                  e      Z G d d	e      Zy)
    )annotationsN)defaultdict)TYPE_CHECKINGAnyCallable)RerankingEvaluator)append_to_last_rowcos_sim)SparseEncoderc            	           e Zd ZdZdddedddddf		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZ	 d	 	 	 	 	 	 	 	 	 d fd	Zd fd
Zd fdZd fdZ		 	 d	 	 	 	 	 	 	 	 	 ddZ
	 d	 	 	 	 	 	 	 	 	 ddZd fdZ xZS )SparseRerankingEvaluatora  
    This evaluator extends :class:`~sentence_transformers.evaluation.RerankingEvaluator' but is specifically designed for sparse encoder models.

    This class evaluates a SparseEncoder model for the task of re-ranking.

    Given a query and a list of documents, it computes the score [query, doc_i] for all possible
    documents and sorts them in decreasing order. Then, MRR@10, NDCG@10 and MAP is compute to measure the quality of the ranking.

    Args:
        samples (list): A list of dictionaries, where each dictionary represents a sample and has the following keys:

            - 'query': The search query.
            - 'positive': A list of positive (relevant) documents.
            - 'negative': A list of negative (irrelevant) documents.
        at_k (int, optional): Only consider the top k most similar documents to each query for the evaluation. Defaults to 10.
        name (str, optional): Name of the evaluator. Defaults to "".
        write_csv (bool, optional): Write results to CSV file. Defaults to True.
        similarity_fct (Callable[[torch.Tensor, torch.Tensor], torch.Tensor], optional): Similarity function between sentence embeddings. By default, cosine similarity. Defaults to cos_sim.
        batch_size (int, optional): Batch size to compute sentence embeddings. Defaults to 64.
        show_progress_bar (bool, optional): Show progress bar when computing embeddings. Defaults to False.
        use_batched_encoding (bool, optional): Whether or not to encode queries and documents in batches for greater speed, or 1-by-1 to save memory. Defaults to True.
        max_active_dims (Optional[int], optional): The maximum number of active dimensions to use.
            `None` uses the model's current `max_active_dims`. Defaults to None.
        mrr_at_k (Optional[int], optional): Deprecated parameter. Please use `at_k` instead. Defaults to None.

    Example:
        ::

            import logging

            from datasets import load_dataset

            from sentence_transformers import SparseEncoder
            from sentence_transformers.sparse_encoder.evaluation import SparseRerankingEvaluator

            logging.basicConfig(format="%(message)s", level=logging.INFO)

            # Load a model
            model = SparseEncoder("naver/splade-cocondenser-ensembledistil")

            # Load a dataset with queries, positives, and negatives
            eval_dataset = load_dataset("microsoft/ms_marco", "v1.1", split="validation").select(range(1000))

            samples = [
                {
                    "query": sample["query"],
                    "positive": [
                        text
                        for is_selected, text in zip(sample["passages"]["is_selected"], sample["passages"]["passage_text"])
                        if is_selected
                    ],
                    "negative": [
                        text
                        for is_selected, text in zip(sample["passages"]["is_selected"], sample["passages"]["passage_text"])
                        if not is_selected
                    ],
                }
                for sample in eval_dataset
            ]


            # Now evaluate using only the documents from the 1000 samples
            reranking_evaluator = SparseRerankingEvaluator(
                samples=samples,
                name="ms-marco-dev-small",
                show_progress_bar=True,
                batch_size=32,
            )

            results = reranking_evaluator(model)
            '''
            RerankingEvaluator: Evaluating the model on the ms-marco-dev-small dataset:
            Queries: 967 	 Positives: Min 1.0, Mean 1.1, Max 3.0 	 Negatives: Min 1.0, Mean 7.1, Max 9.0
            MAP: 53.41
            MRR@10: 54.14
            NDCG@10: 65.06
            Model Query Sparsity: Active Dimensions: 42.2, Sparsity Ratio: 0.9986
            Model Corpus Sparsity: Active Dimensions: 126.5, Sparsity Ratio: 0.9959
            '''
            # Print the results
            print(f"Primary metric: {reranking_evaluator.primary_metric}")
            # => Primary metric: ms-marco-dev-small_ndcg@10
            print(f"Primary metric value: {results[reranking_evaluator.primary_metric]:.4f}")
            # => Primary metric value: 0.6506

    
    T@   FNc                    |	| _         t        t              | _        t        |   |||||||||
	       | j                  j                  g d       y )N)	samplesat_kname	write_csvsimilarity_fct
batch_sizeshow_progress_baruse_batched_encodingmrr_at_k)query_active_dimsquery_sparsity_ratiocorpus_active_dimscorpus_sparsity_ratio)max_active_dimsr   listsparsity_statssuper__init__csv_headersextend)selfr   r   r   r   r   r   r   r   r   r   	__class__s              /home/chris/cleankitchens-env/lib/python3.12/site-packages/sentence_transformers/sparse_encoder/evaluation/SparseRerankingEvaluator.pyr#   z!SparseRerankingEvaluator.__init__m   s`      /)$/)!/!5 	 
	
 	h	
    c                   t        t              | _        t        |   ||||      }| j                  j                         D ])  \  }}t        |      t        |      z  | j                  |<   + |j                  | j                  | j                  | j                               | j                  ||||       t        j                  d| j                  d   dd| j                  d   d       t        j                  d| j                  d	   dd| j                  d
   d       |X| j                  rLt        t         j"                  j%                  || j&                        | j                  j)                                |S )N)modeloutput_pathepochstepsz)Model Query Sparsity: Active Dimensions: r   z.1fz, Sparsity Ratio: r   z.4fz*Model Corpus Sparsity: Active Dimensions: r   r   )r   r    r!   r"   __call__itemssumlenupdateprefix_name_to_metricsr    store_metrics_in_model_card_dataloggerinfor   r	   ospathjoincsv_filevalues)	r&   r+   r,   r-   r.   metricskeyvaluer'   s	           r(   r/   z!SparseRerankingEvaluator.__call__   s    *$/'"Ku\a"b--335 	?JC'*5zCJ'>D$	? 	t2243F3F		RS--eWeUK78K8KL_8`ad7eewx|  yL  yL  Mc  yd  eh  xi  j	
 	89L9LMa9bcf8ggyz~  {N  {N  Of  {g  hk  zl  m	
 "t~~[$--8##**,
 r)   c                "    t         |   |      S N)r"   compute_metricesr&   r+   r'   s     r(   rB   z)SparseRerankingEvaluator.compute_metrices   s    w'..r)   c                "    t         |   |      S rA   )r"   compute_metrices_batchedrC   s     r(   rE   z1SparseRerankingEvaluator.compute_metrices_batched   s    w/66r)   c                "    t         |   |      S rA   )r"   compute_metrices_individualrC   s     r(   rG   z4SparseRerankingEvaluator.compute_metrices_individual   s    w2599r)   c           
        ||j                   }n#|dk(  r|j                  }n|dk(  r|j                  } |f| j                  |ddd| j                  d|}|j                  t        j                  |            }|dv rdnd}	|j                         D ](  \  }
}| j                  |	 d|
    j                  |       * |S )	NquerydocumentTF)r   r   convert_to_sparse_tensorconvert_to_tensorsave_to_cpur   )rI   Ncorpus_)encodeencode_queryencode_documentr   r   sparsitytorchstackr0   r!   append)r&   r+   	sentencesencode_fn_namer   kwargs	encode_fn
embeddingsstatprefixr>   r?   s               r(   embed_inputsz%SparseRerankingEvaluator.embed_inputs   s     !Iw&**Iz)--I	
/%)# 00	
 	

 ~~ekk*56*o=8**, 	AJC6(!C5 1299%@	Ar)   c                B    |j                   j                  | |||       y )N)r-   step)model_card_dataset_evaluation_metrics)r&   r+   r=   r-   r`   s        r(   r5   z9SparseRerankingEvaluator.store_metrics_in_model_card_data   s"     	44T7%VZ4[r)   c                Z    t         |          }| j                  | j                  |d<   |S )Nr   )r"   get_config_dictr   )r&   config_dictr'   s     r(   rd   z(SparseRerankingEvaluator.get_config_dict   s3    g-/+-1-A-AK)*r)   )r   z list[dict[str, str | list[str]]]r   intr   strr   boolr   z4Callable[[torch.Tensor, torch.Tensor], torch.Tensor]r   rf   r   rh   r   rh   r   
int | Noner   ri   )Nrj   )
r+   r   r,   
str | Noner-   rf   r.   rf   returnzdict[str, float])r+   r   )NN)
r+   r   rW   zstr | list[str] | np.ndarrayrX   rk   r   zbool | Nonerl   ztorch.Tensor)r   r   )
r+   r   r=   dict[str, Any]r-   rf   r`   rf   rl   None)rl   rm   )__name__
__module____qualname____doc__r
   r#   r/   rB   rE   rG   r^   r5   rd   __classcell__)r'   s   @r(   r   r      sS   Ut OV"'%)&*#
1
 
 	

 
 M
 
  
 #
 $
 
> ce"1;KN\_	0/7: &*)- 0 #	
 ' 
> Z[\"\-;\DG\SV\	\
 r)   r   )
__future__r   loggingr8   collectionsr   typingr   r   r   rT    sentence_transformers.evaluationr   sentence_transformers.utilr	   r
   numpynp2sentence_transformers.sparse_encoder.SparseEncoderr   	getLoggerro   r6   r    r)   r(   <module>r      sJ    "  	 # / /  ? BP			8	$~1 ~r)   