from sentence_transformers import SentenceTransformer
from sentence_transformers.sentence_transformer.evaluation import (
    RerankingEvaluator,
)


samples = [
    {
        "query": "How do I reset a forgotten password?",
        "positive": [
            "Reset a lost account password from the profile security page.",
        ],
        "negative": [
            "Generate quarterly revenue charts from a CSV export.",
            "Tune the database connection pool for a busy API server.",
            "Schedule a sales demo with the accounts team.",
        ],
    },
    {
        "query": "How can I rotate API keys?",
        "positive": [
            "Rotate API tokens before sharing a new integration.",
        ],
        "negative": [
            "Download a password reset email from account settings.",
            "Create a dashboard with monthly revenue charts.",
            "Archive old support tickets after closing a case.",
        ],
    },
]

model = SentenceTransformer(
    "sentence-transformers/all-MiniLM-L6-v2",
    device="cpu",
)

evaluator = RerankingEvaluator(
    samples=samples,
    name="support-smoke",
    at_k=3,
    write_csv=True,
    show_progress_bar=False,
)

results = evaluator(model, output_path="reranking-results")
threshold = 0.80

print(f"primary metric: {evaluator.primary_metric}")
print(f"primary score: {results[evaluator.primary_metric]:.4f}")
for key in sorted(results):
    print(f"{key}: {results[key]:.4f}")

if results[evaluator.primary_metric] < threshold:
    raise SystemExit(
        f"primary score below threshold {threshold:.2f}: "
        f"{results[evaluator.primary_metric]:.4f}"
    )

print(f"verification: PASS threshold {threshold:.2f} reached")