Search Quality Verification

from qdrant_client import QdrantClient, models

client = QdrantClient("localhost", port=6333)

def run_baseline_queries(baseline_queries, collection_name):
    """Run pre-recorded baseline queries against Qdrant."""
    results = []
    for bq in baseline_queries:
        qdrant_results = client.query_points(
            collection_name=collection_name,
            query=bq["query_vector"],
            limit=bq["top_k"],
            query_filter=build_qdrant_filter(bq["filters"]) if bq.get("filters") else None,
        )

        results.append({
            "query_id": bq["query_id"],
            "description": bq["description"],
            "source_results": bq["source_results"],
            "qdrant_results": [
                {"id": hit.id, "score": hit.score, "rank": i + 1}
                for i, hit in enumerate(qdrant_results.points)
            ],
        })
    return results

def tier1_report(comparison_results):
    """Generate a human-readable spot-check report."""
    for result in comparison_results:
        source_ids = [r["id"] for r in result["source_results"]]
        qdrant_ids = [r["id"] for r in result["qdrant_results"]]

        overlap = set(source_ids) & set(qdrant_ids)
        overlap_pct = len(overlap) / len(source_ids) * 100

        print(f"\nQuery: {result['description']} ({result['query_id']})")
        print(f"  Result overlap: {len(overlap)}/{len(source_ids)} ({overlap_pct:.0f}%)")

        # Check if top result matches
        if source_ids and qdrant_ids:
            if source_ids[0] == qdrant_ids[0]:
                print(f"  Top result: ✓ matches")
            else:
                print(f"  Top result: ✗ differs "
                      f"(source={source_ids[0]}, qdrant={qdrant_ids[0]})")

        # Check score distribution
        if result["qdrant_results"]:
            scores = [r["score"] for r in result["qdrant_results"]]
            print(f"  Score range: {min(scores):.4f} to {max(scores):.4f}")

def recall_at_k(source_results, qdrant_results, k):
    """Compute recall@k: fraction of source top-k present in Qdrant top-k."""
    source_ids = set(r["id"] for r in source_results[:k])
    qdrant_ids = set(r["id"] for r in qdrant_results[:k])
    if not source_ids:
        return 1.0  # No source results = vacuously correct
    return len(source_ids & qdrant_ids) / len(source_ids)

import numpy as np
import json

def tier2_evaluation(baseline_queries, collection_name, client, k=10):
    """Run Tier 2 recall evaluation across all baseline queries."""
    recalls = []

    for bq in baseline_queries:
        qdrant_results = client.query_points(
            collection_name=collection_name,
            query=bq["query_vector"],
            limit=k,
            query_filter=build_qdrant_filter(bq["filters"]) if bq.get("filters") else None,
        )

        qdrant_ranked = [
            {"id": hit.id, "score": hit.score}
            for hit in qdrant_results.points
        ]

        r_at_k = recall_at_k(bq["source_results"], qdrant_ranked, k)
        recalls.append({
            "query_id": bq["query_id"],
            "recall_at_k": r_at_k,
        })

    # Compute aggregate statistics
    recall_values = [r["recall_at_k"] for r in recalls]
    stats = {
        "num_queries": len(recalls),
        "k": k,
        "mean_recall": float(np.mean(recall_values)),
        "median_recall": float(np.median(recall_values)),
        "min_recall": float(np.min(recall_values)),
        "p5_recall": float(np.percentile(recall_values, 5)),
        "p25_recall": float(np.percentile(recall_values, 25)),
        "std_recall": float(np.std(recall_values)),
    }

    return recalls, stats

def tier2_report(recalls, stats):
    """Print Tier 2 evaluation summary."""
    print(f"Recall@{stats['k']} across {stats['num_queries']} queries:")
    print(f"  Mean:   {stats['mean_recall']:.3f}")
    print(f"  Median: {stats['median_recall']:.3f}")
    print(f"  Min:    {stats['min_recall']:.3f}")
    print(f"  P5:     {stats['p5_recall']:.3f}")
    print(f"  Std:    {stats['std_recall']:.3f}")

    # Flag low-recall queries for investigation
    low_recall = [r for r in recalls if r["recall_at_k"] < 0.7]
    if low_recall:
        print(f"\n⚠ {len(low_recall)} queries with recall < 0.7:")
        for r in low_recall:
            print(f"  {r['query_id']}: {r['recall_at_k']:.3f}")

from scipy import stats as scipy_stats

def score_correlation(source_results, qdrant_results):
    """Compute rank correlation between source and Qdrant scores for overlapping results."""
    # Find overlapping IDs
    source_map = {r["id"]: r["score"] for r in source_results}
    qdrant_map = {r["id"]: r["score"] for r in qdrant_results}
    common_ids = set(source_map.keys()) & set(qdrant_map.keys())

    if len(common_ids) < 3:
        return None  # Not enough overlap to compute correlation

    source_scores = [source_map[id] for id in common_ids]
    qdrant_scores = [qdrant_map[id] for id in common_ids]

    # Spearman rank correlation (order matters more than magnitude)
    correlation, p_value = scipy_stats.spearmanr(source_scores, qdrant_scores)
    return {"correlation": correlation, "p_value": p_value, "n_common": len(common_ids)}

import numpy as np

def ndcg_at_k(retrieved_ids, relevance_map, k):
    """Normalized Discounted Cumulative Gain at k."""
    dcg = 0.0
    for i, doc_id in enumerate(retrieved_ids[:k]):
        rel = relevance_map.get(doc_id, 0)
        dcg += (2**rel - 1) / np.log2(i + 2)  # i+2 because log2(1) = 0

    # Ideal DCG: sort all relevance scores descending
    ideal_rels = sorted(relevance_map.values(), reverse=True)[:k]
    idcg = sum((2**rel - 1) / np.log2(i + 2) for i, rel in enumerate(ideal_rels))

    return dcg / idcg if idcg > 0 else 0.0

def mrr(retrieved_ids, relevant_ids):
    """Mean Reciprocal Rank: how high is the first relevant result?"""
    for i, doc_id in enumerate(retrieved_ids):
        if doc_id in relevant_ids:
            return 1.0 / (i + 1)
    return 0.0

def tier3_evaluation(eval_queries, collection_name, client, k=10):
    """Full Tier 3 evaluation with NDCG and MRR."""
    ndcgs = []
    mrrs = []

    for eq in eval_queries:
        qdrant_results = client.query_points(
            collection_name=collection_name,
            query=eq["query_vector"],
            limit=k,
            query_filter=build_qdrant_filter(eq["filters"]) if eq.get("filters") else None,
        )
        retrieved_ids = [hit.id for hit in qdrant_results.points]

        relevance_map = {d["id"]: d["relevance"] for d in eq["relevant_docs"]}
        relevant_ids = set(relevance_map.keys())

        ndcgs.append(ndcg_at_k(retrieved_ids, relevance_map, k))
        mrrs.append(mrr(retrieved_ids, relevant_ids))

    return {
        "mean_ndcg": float(np.mean(ndcgs)),
        "mean_mrr": float(np.mean(mrrs)),
        "median_ndcg": float(np.median(ndcgs)),
        "num_queries": len(eval_queries),
    }