Real evaluations on real data.
Reproducible from the SDK.

from multivon_eval import EvalCase, EvalSuite, Faithfulness, Relevance
from multivon_eval import JudgeConfig, configure

configure(JudgeConfig(provider="anthropic", model="claude-haiku-4-5"))

CASES = [
    # 4 grounded answers + 1 deliberately ungrounded:
    {"question": "What is the rental car reimbursement limit on a standard policy?",
     "answer":   "Acme reimburses rental cars at up to $75 per day for a maximum "
                 "of 30 days. Premium policyholders receive unlimited rental "
                 "reimbursement."},  # ← not in the KB at all
    # ...
]

cases = [EvalCase(input=c["question"], context=FULL_CONTEXT) for c in CASES]
suite = EvalSuite("RAG Faithfulness — Insurance KB")
suite.add_cases(cases)
suite.add_evaluators(Faithfulness(), Relevance())
report = suite.run(lambda q: next(c["answer"] for c in CASES if c["question"] == q))

─────────── RAG Faithfulness — Insurance KB ───────────
  Model: precomputed-answers

  #   Input                              Score  Status
  1   What is the standard deductible…   0.75   PASS
  2   How do I file a claim and how lo…  1.00   PASS
  3   What discounts are available for…  0.88   PASS
  4   Does my auto policy cover items …  0.88   PASS
  5   What is the rental car reimburs…   0.38   FAIL

           By Evaluator
  Evaluator      Avg Score   Pass Rate
  faithfulness        0.80         80%
  relevance           0.75        100%

  Summary
  Total: 5   Passed: 4   Failed: 1
  Pass Rate: 80.0% [38%–96% 95% CI]
  Avg Score: 0.78 [0.57–0.93]

  === Faithfulness verdict per case ===
  [PASS] faithfulness=1.00  Q: What is the standard deductible for collision coverage?
  [PASS] faithfulness=1.00  Q: How do I file a claim and how long does it take?
  [PASS] faithfulness=1.00  Q: What discounts are available for bundling and safe driving?
  [PASS] faithfulness=1.00  Q: Does my auto policy cover items stolen from inside my car?
  [FAIL] faithfulness=0.00  Q: What is the rental car reimbursement limit on a standard pol...

  Result: FAIL — 1/5 case(s) ungrounded.

from openai import OpenAI
from pdfhell.generators.footnote_override import generate
from pdfhell.scorer import score_case
import base64

# Generate the adversarial PDF deterministically from a seed.
pdf_bytes, case = generate(seed=2)
# case.expected_tokens == ["3 month", "uncapped", "4.2", "6.1", "6.2"]
# case.forbidden_answers[0] == "Liability is capped at 3 months of fees paid."

pdf_b64 = base64.b64encode(pdf_bytes).decode("ascii")
response = OpenAI().chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": [
        {"type": "file", "file": {
            "filename": "contract.pdf",
            "file_data": f"data:application/pdf;base64,{pdf_b64}"}},
        {"type": "text", "text": case.question},
    ]}],
    max_tokens=400,
)

# Code-based ground truth — no LLM-as-judge.
result = score_case(case, response.choices[0].message.content)

==============================================================================
Contract analysis trap — footnote_override (pdfhell)
==============================================================================
PDF:              02_contract.pdf  (2,542 bytes)
Trap family:      footnote_override
Question to model: Read the attached Master Services Agreement. What is the
                   LIABILITY CAP and what carve-outs (if any) apply? Be precise
                   about which Sections are uncapped.
Expected answer:  Liability is capped at 3 months of fees paid, EXCEPT that
                  liability arising from Sections 4.2, 6.1, 6.2 is uncapped.
Expected tokens:  ['3 month', 'uncapped', '4.2', '6.1', '6.2']
Forbidden answer: Liability is capped at 3 months of fees paid.

Calling gpt-4o ...

=== Model output ===
The LIABILITY CAP in the Master Services Agreement limits the aggregate liability
of either party to an amount equal to 3 months of fees paid by the Customer during
the twelve (12) month period immediately preceding the event giving rise to such
liability.

Carve-outs that apply, per the document, mean that liability is uncapped for claims
arising from Sections 4.2, 6.1, and 6.2.

=== pdfhell score ===
  correct:           True
  matched_expected:  True
  fell_for_trap:     False
  matched_forbidden: []
  refused:           False

Result: PASS — model captured the footnote carve-out.

from multivon_eval import EvalCase, EvalSuite, Faithfulness, Relevance
from multivon_eval import JudgeConfig, configure
from multivon_eval.evaluators.llm_judge import CheckEvaluator

configure(JudgeConfig(provider="anthropic", model="claude-haiku-4-5"))

suite = EvalSuite("Customer Support QA")
suite.add_cases(cases)  # 10 tickets with input + context + bot answer
suite.add_evaluators(
    Faithfulness(),
    Relevance(),
    CheckEvaluator(
        criterion="Response should name the customer's specific problem and "
                  "provide a concrete next step.",
        questions=[  # pin the questions for CI reproducibility
            "Does the response name or restate the customer's specific problem?",
            "Does the response provide a concrete next step or action?",
            "Does the response avoid vague deferrals like 'we will look into this'?",
        ],
        name="actionability",
    ),
)
report = suite.run(model_fn)

─────── Customer Support QA ───────
  Model: precomputed-answers

  #   Input                              Score  Status
  1   How long does standard shipping…   0.69   FAIL
  2   Can I return a sale item?          0.69   FAIL
  3   I forgot my password — how do I…   1.00   PASS
  4   Do you accept Apple Pay?           0.53   FAIL
  5   How do I cancel my subscription?   0.92   PASS
  6   The app is showing a white scre…   0.42   FAIL
  7   How long do refunds take after …   0.81   FAIL
  8   I want to delete my account.       0.81   FAIL
  9   When will my order arrive if I …   0.47   FAIL
 10   Refund for order #4839 — it nev…   0.50   FAIL

           By Evaluator
  Evaluator       Avg Score   Pass Rate
  faithfulness         0.92         80%
  relevance            0.70         90%
  actionability        0.43         20%

  Summary
  Total: 10   Passed: 2   Failed: 8
  Pass Rate: 20.0% [6%–51% 95% CI]

  === Per-case breakdown ===
  [FAIL]  T1  faith=1.00✓  rel=0.75✓  act=0.33✗
  [FAIL]  T2  faith=1.00✓  rel=0.75✓  act=0.33✗
  [PASS]  T3  faith=1.00✓  rel=1.00✓  act=1.00✓
  [FAIL]  T4  faith=0.83✗  rel=0.75✓  act=0.00✗
  [PASS]  T5  faith=1.00✓  rel=0.75✓  act=1.00✓
  [FAIL]  T6  faith=1.00✓  rel=0.25✗  act=0.00✗
  [FAIL]  T7  faith=1.00✓  rel=0.75✓  act=0.67✗
  [FAIL]  T8  faith=1.00✓  rel=0.75✓  act=0.67✗
  [FAIL]  T9  faith=0.33✗  rel=0.75✓  act=0.33✗
  [FAIL] T10  faith=1.00✓  rel=0.50✓  act=0.00✗

  Result: FAIL — pass rate below 70% gate.

from multivon_eval import EvalCase, EvalSuite
from multivon_eval.evaluators.compliance import PIIEvaluator

RECORDS = [
    {"id": "MR1", "text": "Patient presented with mild dehydration…"},
    {"id": "MR2", "text": "Patient John Doe (SSN 123-45-6789) was admitted on "
                          "03/14/2025… Spouse contact: 415-555-0182."},
    {"id": "MR3", "text": "Routine post-op check after laparoscopic…"},
    {"id": "MR4", "text": "MRN-4471829 — patient reports migraine. Family physician "
                          "notified via patient@example.com…"},
    {"id": "MR5", "text": "Pediatric well-child visit. Growth on expected curve…"},
]

suite = EvalSuite("PII Detection — Medical Records")
suite.add_cases([EvalCase(input=r["id"]) for r in RECORDS])
# "hipaa" adds MRN, fax, admission dates, NPI/DEA, etc. on top of base patterns.
suite.add_evaluators(PIIEvaluator(jurisdiction="hipaa"))

report = suite.run(lambda rid: next(r["text"] for r in RECORDS if r["id"] == rid))
# No API key required. No network call. Fully deterministic.

─────── PII Detection — Medical Records ───────
  Model: static-records

  #   Input   Output                              Score   Status
  1   MR1     Patient presented with mild deh…    1.00    PASS
  2   MR2     Patient John Doe (SSN 123-45-67…    0.00    FAIL
  3   MR3     Routine post-op check after lap…    1.00    PASS
  4   MR4     MRN-4471829 — patient reports m…    0.00    FAIL
  5   MR5     Pediatric well-child visit. Gro…    1.00    PASS

  === Per-record PII findings ===
  [CLEAN] MR1  No PII detected
  [LEAK ] MR2
           phone_us: "415-555-0182"
           ssn: "123-45-6789"
           address: "2025 with chest"
           fax_number: "415-555-0182"
           admission_date: "admitted on 03/14/2025"
  [CLEAN] MR3
  [LEAK ] MR4
           email: "patient@example.com"
           medical_record_number: "MRN-4471829"
  [CLEAN] MR5

  Final: 2/5 record(s) contain PII.
  (Regex-only — no API calls, $0 cost, fully deterministic.)