
    wfh9                        d dl mZ d dlZd dlZd dlZd dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ  ej                  e      Z G d de      Zy)    )annotationsN)average_precision_score
ndcg_score)tqdm)SentenceEvaluatorc                  t     e Zd ZdZ	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd	d
dZd ZddZd Z xZ	S )ReciprocalRankFusionEvaluatora  
    This class evaluates a hybrid search approach using Reciprocal Rank Fusion (RRF).

    Given a query and two separate ranked lists of documents from different retrievers (e.g., sparse and dense),
    it combines them using the RRF formula and computes metrics like MRR@k, NDCG@k, and MAP.

    Args:
        dense_samples (list): A list of dictionaries for dense retriever results. Each dictionary should have:
            - 'query_id': The ID of the query
            - 'query': The search query text
            - 'positive': A list of relevant documents
            - 'documents': A list of all documents (including positives)
        sparse_samples (list): A list of dictionaries for sparse retriever results with the same format
        at_k (int): Only consider the top k documents for evaluation. Defaults to 10.
        rrf_k (int): Constant in the RRF formula. Defaults to 60.
        name (str): Name of the evaluator. Defaults to "".
        batch_size (int): Batch size used for the evaluation. Defaults to 32.
        show_progress_bar (bool): Output a progress bar. Defaults to False.
        write_csv (bool): Write results to CSV file. Defaults to True.
        write_predictions (bool): Whether to write the fused predictions to a JSONL file. Defaults to False.

    Example:
        See an example usage `Applications > Retrieve & Rerank <../../../examples/sparse_encoder/applications/retrieve_rerank/README.html>`_

    c
                    t         |           || _        || _        t	        |      t	        |      k7  r$t        dt	        |       dt	        |       d      t        t        ||            D ]D  \  }
\  }}d|vsd|vrt        d|
 d      |d   |d   k7  s,t        d|
 d|d    d	|d           || _        || _	        || _
        || _        || _        |	| _        d
|rd|z   ndz   dz   | _        dddd| j                   d| j                   dd| j                   d| j                   dd| j                   d| j                   g| _        || _        d| j                   | _        | j                  rd
|rd|z   ndz   dz   | _        y y )NzDense samples (z) and sparse samples (z) must have the same lengthquery_idzSample at index z missing 'query_id' fieldzQuery ID mismatch at index z:  != ReciprocalRankFusion_evaluation_ z_results.csvepochsteps	Dense_MAPz
Dense_MRR@zDense_NDCG@
Sparse_MAPzSparse_MRR@zSparse_NDCG@
Fusion_MAPzFusion_MRR@zFusion_NDCG@ndcg@z_predictions.jsonl)super__init__dense_samplessparse_sampleslen
ValueError	enumeratezipat_krrf_kname
batch_sizeshow_progress_barwrite_predictionscsv_filecsv_headers	write_csvprimary_metricpredictions_file)selfr   r   r   r   r    r!   r"   r&   r#   idense_samplesparse_sample	__class__s                /home/chris/cleankitchens-env/lib/python3.12/site-packages/sentence_transformers/sparse_encoder/evaluation/ReciprocalRankFusionEvaluator.pyr   z&ReciprocalRankFusionEvaluator.__init__,   s    	*, }^!44!#m"4!55KCP^L_K``{| 
 1:#m^:\0] 	,A,m-=1P #3A36O!PQQJ'=+DD 1!B|J7O6PPTUbcmUnTop 	 	
	$!2!294S4ZUWX[ii$$))%$))%499+&$))%499+&
 # %dii[1!!14S4ZRPSgg ! "    c                  A |dk7  r|dk(  rd| }nd| d| d}nd}t         j                  d| j                   d| d	       t         j                  d
t        | j                         d       g }g }g }g }g }	g }
g }g }g }d}g }g }t        t        t        | j                  | j                        d| j                   t        | j                                    D ]  \  }\  }}|d   }||d   k(  sJ d| d|d           |d   }|d   }t        |t              r|g}|d   }|d   }|D cg c]  }t        ||v        }}t        |      dk(  rd\  }}}n]|dgt        |      t        |      z
  z  z  }t        j                  t!        t        |      dd            }| j#                  ||      \  }}}|j%                  |       |j%                  |       |j%                  |       |D cg c]  }t        ||v        }}t        |      dk(  rd\  } }!}"n]|dgt        |      t        |      z
  z  z  }t        j                  t!        t        |      dd            }#| j#                  ||#      \  } }!}"|j%                  |        |	j%                  |!       |
j%                  |"       t        |      D $%ci c]  \  }$}%|%|$
 }&}$}%t        |      D $%ci c]  \  }$}%|%|$
 }'}$}%t'        |&j)                               t'        |'j)                               z  }(i A|(D ]`  }%|&j+                  |%t        |            })|'j+                  |%t        |            }*d| j,                  |)z   z  d| j,                  |*z   z  z   A|%<   b t/        Aj)                         Afdd      }+|+D cg c]  }t        ||v        },}|dz  }|j%                  t        |             t        |,      dk(  rd\  }-}.}/n]|,dgt        |      t        |,      z
  z  z  },t        j                  t!        t        |,      dd            }0| j#                  |,|0      \  }-}.}/|j%                  |-       |j%                  |.       |j%                  |/       | j0                  s|j%                  ||||+d        t        j2                  |      }1t        j2                  |      }2t        j2                  |      }3t        j2                  |      }4t        j2                  |	      }5t        j2                  |
      }6t        j2                  |      }7t        j2                  |      }8t        j2                  |      }9d|3d| j4                   |1d| j4                   |2d|6d| j4                   |4d | j4                   |5d!|9d"| j4                   |7d#| j4                   |8i	}:t         j                  d$| d%|rt7        |      ndd&d'|rt        j2                  |      ndd&d(|rt9        |      ndd&       t         j                  d)       t         j                  d*d+d,d-d.d,d/d.d,d0d.d,d1d2d,d3d4d5       t         j                  d6       t         j                  d7|3d8d,|6d8d,|9d8d,|9|3z
  d9d,|9|6z
  d:d5       t         j                  d;| j4                  d<d,|1d8d,|4d8d,|7d8d,|7|1z
  d9d,|7|4z
  d:d5       t         j                  d=| j4                  d>d,|2d8d,|5d8d,|8d8d,|8|2z
  d9d,|8|5z
  d:d5       t         j                  d)       |kt;        j<                  |d?       | j>                  rt:        j@                  jC                  || jD                        };t:        j@                  jG                  |;      }<tI        |;|<rd@ndAdBC      5 }=tK        jL                  |=      }>|<s|>jO                  | jP                         |>jO                  |||3|1|2|6|4|5|9|7|8g       d d d        | j0                  r|rt:        j@                  jC                  || jR                        }?tI        |?dAdBC      5 }=|D ])  }@|=jU                  tW        jX                  |@      dDz          + 	 d d d        t         j                  dE|?        | j[                  |:| j                        }:|:S c c}w c c}w c c}%}$w c c}%}$w c c}w # 1 sw Y   xY w# 1 sw Y   fxY w)FNz after epoch z
 in epoch z after z stepsr   z?ReciprocalRankFusionEvaluator: Evaluating hybrid search on the z dataset:zProcessing z samplesr   
Evaluating)descdisabletotalr   zQuery ID mismatch: r   querypositive	documents)r   r   r      c                    |    S )N )doc
rrf_scoress    r.   <lambda>z8ReciprocalRankFusionEvaluator.__call__.<locals>.<lambda>   s    :c? r/   T)keyreverse)r   r7   r8   r9   	dense_mapz
dense_mrr@zdense_ndcg@
sparse_mapzsparse_mrr@zsparse_ndcg@mapzmrr@r   z	Queries: z	Positives: Min z.1fz, Mean z, Max zK===========================================================================Metricz<7z | Densez^8SparseFusionzGain vs Densez^13zGain vs Sparsez^14z |zK---------------------------------------------------------------------------z
MAP     | z>8.2%z>+13.2%z>+14.2%zMRR@z<3zNDCG@z<2)exist_okawzutf-8)modeencoding
z#Wrote fused ranking predictions to ).loggerinfor    r   r   r   r   r   r   r"   
isinstancestrintsumnparrayrangecompute_metricsappendsetkeysgetr   sortedr#   meanr   minmaxosmakedirsr&   pathjoinr$   isfileopencsvwriterwriterowr%   r(   writejsondumpsprefix_name_to_metrics)Br)   output_pathr   r   out_txtdense_mrr_scoresdense_ndcg_scoresdense_ap_scoressparse_mrr_scoressparse_ndcg_scoressparse_ap_scoresfusion_mrr_scoresfusion_ndcg_scoresfusion_ap_scoresnum_queriesnum_positivesfused_results_listr*   r+   r,   r   r7   r8   
dense_docssparse_docssampledense_is_relevant	dense_mrr
dense_ndcgdense_apdense_pred_scoressparse_is_relevant
sparse_mrrsparse_ndcg	sparse_apsparse_pred_scoresrankr=   dense_rankssparse_ranksall_docs
dense_ranksparse_rank
fused_docsfusion_is_relevant
fusion_mrrfusion_ndcg	fusion_apfusion_pred_scoresmean_dense_mrrmean_dense_ndcgmean_dense_apmean_sparse_mrrmean_sparse_ndcgmean_sparse_apmean_fusion_mrrmean_fusion_ndcgmean_fusion_apmetricscsv_pathoutput_file_existsfrh   	json_pathresultr>   sB                                                                    @r.   __call__z&ReciprocalRankFusionEvaluator.__call__i   s0
   B;{)%1&ugWUG6BGUVZV_V_U``hiphqqrstk#d&8&8"9!:(CD  1:D&&(;(;<! 222$,,-	1
 ]	,A,m $J/H M*55O$XJd=3L2MNO5 !)E#J/H(C($: &k2J'4K HR RVVx%7!8 R R $%*29/	:x!aSCMC@Q<R,R%SS!$&HHU37H3I1b-Q$R!262F2FGXZk2l/	:x##I.$$Z0""8, IT!Tf#f&8"9!T!T %&!+5<2
K"qcS]SAS=T-T&UU"%'XXeC8J4KQPR.S%T"595I5IJ\^p5q2
K$$Z0%%k2##I. 7@
6KLs39LKL7@7MN)$CINLN ;++-.\5F5F5H1IIH J e(__S#j/B
*..sC4DE#$

Z(?#@Q$**WbJbEc"d
3e  
 17R\`aJ IS!Sf#f&8"9!S!S1K  X/ %&!+5<2
K"qcS]SAS=T-T&UU"%'XXeC8J4KQPR.S%T"595I5IJ\^p5q2
K$$Z0%%k2##I. %%"))!)Ex^hiw]	@ !12''"340''"3477#56!12''"3477#56!12 $n$))%.$))%499+&(8>499+DII;!1

 	} %4Ac-0qM N.;BGGM*3G H)63}%AcBD	
 	Hm3wrl#hr]#hr]#o^aMbbefvwze{{}~	
 	Hu-S0Fc.Y^I__bcq  uB  dB  CJ  cK  KN  O]  `n  On  ov  Nw  wy  z	
 	499R.N5#9_U<SSVWfglVmmp  rA  DR  rR  SZ  q[  [^  _n  q@  _@  AH  ^I  IK  L	
 	DIIb>_U$;3?OPU>VVYZjkpYqqt  vF  IX  vX  Y`  ua  ad  eu  xH  eH  IP  dQ  QS  T	
 	H "KKd3~~77<<T]]C%'WW^^H%="(0BV]^ bc ZZ]F-(8(89OO!!)*+*+,*+,, %%*<GGLLd6K6KL	)#@ ;A"4 ;

6 2T 9:;; A)MN --gtyyA !S "U MN  "T` 0; ;s7   .b:-b?6cc
!c/Ac/c!c!c*c                    t        j                  |      ddd   }d}t        |d| j                         D ]  \  }}||   sd|dz   z  } n t	        |g|g| j                        }t        ||      }|||fS )z/Compute MRR, NDCG, and AP metrics using sklearnNr1   r   r:   )k)rU   argsortr   r   r   r   )	r)   y_truey_predrankingmrrr   indexndcgaps	            r.   rX   z-ReciprocalRankFusionEvaluator.compute_metricsA  s    **V$TrT* $WQ%;< 	KD%e}4!8n	 6(VH		: %VV4D"}r/   c                h    |s|S |j                         D ci c]  \  }}| d| | c}}S c c}}w )z0Prefix all metric names with the evaluator name.r   )items)r)   r   prefixr   vs        r.   rm   z4ReciprocalRankFusionEvaluator.prefix_name_to_metricsT  s7    N/6}}?tq!6(!A3"???s   .c                J    | j                   | j                  | j                  dS )Nr   r   r#   r   )r)   s    r.   get_config_dictz-ReciprocalRankFusionEvaluator.get_config_dictZ  s#    IIZZ!%!7!7
 	
r/   )
   <   r       FTF)r    list[dict[str, str | list[str]]]r   r   r   rS   r   rS   r    rR   r!   rS   r"   boolr&   r   r#   r   )Nr1   r1   )rn   z
str | Noner   rS   r   rS   returndict[str, float])r   r   r   rR   r   r   )
__name__
__module____qualname____doc__r   r   rX   rm   r   __classcell__)r-   s   @r.   r	   r	      s    < "'"';7; 9; 	;
 ; ; ;  ; ;  ;zVp&@
r/   r	   )
__future__r   rg   rk   loggingra   numpyrU   sklearn.metricsr   r   r   2sentence_transformers.evaluation.SentenceEvaluatorr   	getLoggerr   rO   r	   r<   r/   r.   <module>r      sA    " 
   	  ?  P			8	$N
$5 N
r/   