From: Stefan Gasser <redacted>
Date: Wed, 17 Jun 2026 20:24:19 +0000 (+0200)
Subject: Add PII detection accuracy benchmark (#99)
X-Git-Tag: v0.5.0~1
X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=747caa2fa1c6ce50a0cb4264f636fb6f2c796cc2;p=sgasser-llm-shield.git

Add PII detection accuracy benchmark (#99)

Adds a corpus-driven PII accuracy benchmark against the Presidio /analyze endpoint, plus pl/ro precision negatives, non-western PERSON coverage, and address-like LOCATION cases.
---

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 301a330..6458f5b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,6 +24,9 @@ jobs:
       - name: Type check
         run: bun run typecheck
 
+      - name: Benchmark type check
+        run: bun run typecheck:benchmarks
+
       - name: Lint & format check
         run: bun run check
 
@@ -38,4 +41,4 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Test Docker build
-        run: docker build -f docker/Dockerfile -t pasteguard:test .
\ No newline at end of file
+        run: docker build -f docker/Dockerfile -t pasteguard:test .
diff --git a/benchmarks/pii-accuracy/README.md b/benchmarks/pii-accuracy/README.md
new file mode 100644
index 0000000..2f8b4ef
--- /dev/null
+++ b/benchmarks/pii-accuracy/README.md
@@ -0,0 +1,68 @@
+# PII accuracy benchmark
+
+This benchmark targets the analyzer `/analyze` endpoint on `http://localhost:3000/analyze` by
+default. The request/response contract is the analyzer contract, so the same corpus can also be
+pointed at another compatible analyzer endpoint with `--url`.
+
+The corpus only includes entities exposed by Pasteguard's default PII configuration:
+`PERSON`, `EMAIL_ADDRESS`, `PHONE_NUMBER`, `CREDIT_CARD`, `IBAN_CODE`, `IP_ADDRESS`, and
+`LOCATION`. Cases are not skipped dynamically; if the active runtime cannot analyze a configured
+language or entity, the benchmark fails.
+
+The corpus covers the European image language set: `en`, `de`, `es`, `fr`, `it`, `nl`, `pl`,
+`pt`, and `ro`.
+
+The runner validates the corpus before applying filters. Unknown YAML fields, unsupported
+entities, unsupported languages, unknown suites, duplicate case IDs, and expected strings that do
+not occur in the case text fail the run.
+
+## Design
+
+Cases are selected from the product behavior Pasteguard should provide, not from what the current
+detector already happens to pass.
+
+- `core` covers the minimum detection promise for configured entities. These are gating tests.
+- `precision` covers the minimum false-positive promise for configured entities. These are gating
+  tests.
+- `eval` contains realistic multilingual workflow cases for quality tracking.
+- `hard` contains difficult, ambiguous, or aspirational cases for configured entities.
+
+Additional suites make the benchmark easier to read by intent:
+
+- `multilingual-sentences` checks every configured entity in sentence form across all supported
+  languages.
+- `multilingual-paragraphs` checks every configured entity inside realistic multi-sentence
+  workflow text across all supported languages.
+- `boundaries` checks whether spans stop cleanly around punctuation, brackets, and quotes.
+- `precision-paragraphs` checks longer negative controls with operational lookalike strings.
+
+`core` and `precision` cases are gating by default. `eval` and `hard` cases are report-only by
+default. Individual cases can override this with `gate`. Analyzer errors, HTTP errors, and invalid
+responses always fail the benchmark run.
+
+Match modes:
+
+- `exact` requires the normalized detected text to equal the expected text.
+- `contains` requires the detected span to fully cover the expected text with at most two extra
+  characters on either side.
+- `overlap` is reserved for deliberately loose edge cases where any span overlap is meaningful.
+
+## Sources
+
+- GDPR Article 4 definition of personal data:
+  https://eur-lex.europa.eu/eli/reg/2016/679/oj
+
+## Run
+
+```bash
+bun run benchmark:accuracy
+```
+
+Useful filters:
+
+```bash
+bun run benchmark:accuracy --suite core,precision
+bun run benchmark:accuracy --category core,precision
+bun run benchmark:accuracy --languages en,de,it,pl,ro
+bun run benchmark:accuracy --url http://localhost:3000/analyze --verbose
+```
diff --git a/benchmarks/pii-accuracy/run.ts b/benchmarks/pii-accuracy/run.ts
index b7ab343..fea0bef 100644
--- a/benchmarks/pii-accuracy/run.ts
+++ b/benchmarks/pii-accuracy/run.ts
@@ -1,412 +1,634 @@
-#!/usr/bin/env bun
-/**
- * PII Detection Accuracy Benchmark
- *
- * Measures precision, recall, and F1 score of the PII detection system.
- *
- * Usage:
- *   bun run benchmarks/pii-accuracy/run.ts
- *   bun run benchmarks/pii-accuracy/run.ts --threshold 0.5
- *   bun run benchmarks/pii-accuracy/run.ts --languages de,en
- *   bun run benchmarks/pii-accuracy/run.ts --verbose
- */
-
-import { parseArgs } from "util";
-import { Glob } from "bun";
+import { readdir, readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
 import { parse as parseYaml } from "yaml";
+import { isKnownSuite, SUITES, suiteNames } from "./taxonomy";
 import {
-  TestCaseSchema,
-  type AccuracyMetrics,
-  type DetectedEntity,
-  type ExpectedEntity,
-  type TestCase,
+  type BenchmarkCase,
+  BenchmarkFileSchema,
+  type Detection,
+  type ExpectedSpan,
   type TestResult,
 } from "./types";
 
-// Configuration
+const DEFAULT_ANALYZE_URL = "http://localhost:3000/analyze";
 const DEFAULT_THRESHOLD = 0.7;
-const PRESIDIO_URL = process.env.PRESIDIO_URL || "http://localhost:5002";
-const TEST_DATA_DIR = import.meta.dir + "/test-data";
-
-// Parse command line arguments
-const { values: args } = parseArgs({
-  args: Bun.argv.slice(2),
+const MAX_FAILURES_TO_PRINT = 40;
+const MAX_CONTAINS_EDGE_CHARS = 2;
+
+type Filters = {
+  suites?: Set<string>;
+  categories?: Set<string>;
+  languages?: Set<string>;
+  split?: Set<string>;
+};
+
+type Metrics = {
+  cases: number;
+  passed: number;
+  errors: number;
+  tp: number;
+  fp: number;
+  fn: number;
+};
+
+const argv = parseArgs({
   options: {
-    threshold: { type: "string", short: "t" },
-    languages: { type: "string", short: "l" },
-    verbose: { type: "boolean", short: "v", default: false },
-    help: { type: "boolean", short: "h", default: false },
+    url: { type: "string" },
+    threshold: { type: "string" },
+    suite: { type: "string" },
+    category: { type: "string" },
+    languages: { type: "string" },
+    split: { type: "string" },
+    verbose: { type: "boolean", default: false },
+    "list-suites": { type: "boolean", default: false },
+    help: { type: "boolean", default: false },
   },
+  allowPositionals: false,
 });
 
-if (args.help) {
-  console.log(`
-PII Detection Accuracy Benchmark
-
-Usage:
-  bun run benchmarks/pii-accuracy/run.ts [options]
+if (argv.values.help) {
+  printHelp();
+  process.exit(0);
+}
 
-Options:
-  -t, --threshold <value>    Score threshold (default: ${DEFAULT_THRESHOLD})
-  -l, --languages <langs>    Comma-separated languages to test (e.g., de,en)
-  -v, --verbose              Show detailed results for each test case
-  -h, --help                 Show this help message
-
-Examples:
-  bun run benchmarks/pii-accuracy/run.ts
-  bun run benchmarks/pii-accuracy/run.ts --threshold 0.5
-  bun run benchmarks/pii-accuracy/run.ts --languages de,en
-  bun run benchmarks/pii-accuracy/run.ts --verbose
-`);
+if (argv.values["list-suites"]) {
+  for (const name of suiteNames()) {
+    console.log(`${name}: ${SUITES[name].description}`);
+  }
   process.exit(0);
 }
 
-const threshold = args.threshold ? Number.parseFloat(args.threshold) : DEFAULT_THRESHOLD;
-const verbose = args.verbose ?? false;
-const languageFilter = args.languages?.split(",").map((l) => l.trim().toLowerCase());
-
-// Entity types we test for
-const ENTITY_TYPES = [
-  "PERSON",
-  "EMAIL_ADDRESS",
-  "PHONE_NUMBER",
-  "CREDIT_CARD",
-  "IBAN_CODE",
-  "IP_ADDRESS",
-  "LOCATION",
-];
-
-/**
- * Load test cases from YAML files in test-data directory
- */
-async function loadTestCases(): Promise<TestCase[]> {
-  const testCases: TestCase[] = [];
-  const glob = new Glob("*.yaml");
-
-  for await (const file of glob.scan(TEST_DATA_DIR)) {
-    const filePath = `${TEST_DATA_DIR}/${file}`;
-    const content = await Bun.file(filePath).text();
-    const data = parseYaml(content) as { test_cases: unknown[] };
-
-    if (!data.test_cases || !Array.isArray(data.test_cases)) {
-      console.warn(`Warning: ${file} has no test_cases array`);
-      continue;
+const analyzeUrl = argv.values.url ?? process.env.PII_BENCHMARK_ANALYZE_URL ?? DEFAULT_ANALYZE_URL;
+const threshold = parseThreshold(argv.values.threshold ?? String(DEFAULT_THRESHOLD));
+const filters = buildFilters(argv.values);
+const verbose = Boolean(argv.values.verbose);
+
+const benchmarkDir = dirname(fileURLToPath(import.meta.url));
+const testDataDir = join(benchmarkDir, "test-data");
+
+const allCases = await loadCases(testDataDir);
+validateCorpus(allCases);
+
+const cases = applyFilters(allCases, filters);
+
+if (cases.length === 0) {
+  console.error("No benchmark cases matched the selected filters.");
+  process.exit(1);
+}
+
+const results: TestResult[] = [];
+
+for (const testCase of cases) {
+  results.push(await runCase(testCase, analyzeUrl, threshold));
+}
+
+printReport(results, analyzeUrl, threshold, verbose);
+
+const gatingFailures = results.filter((result) => result.gating && !result.passed);
+const errors = results.filter((result) => result.error);
+
+if (gatingFailures.length > 0 || errors.length > 0) {
+  process.exitCode = 1;
+}
+
+async function loadCases(dir: string): Promise<BenchmarkCase[]> {
+  const files = (await readdir(dir)).filter((file) => file.endsWith(".yaml")).sort();
+  const casesFromFiles = await Promise.all(
+    files.map(async (file) => {
+      const raw = await readFile(join(dir, file), "utf8");
+      const parsed = BenchmarkFileSchema.safeParse(parseYaml(raw));
+
+      if (!parsed.success) {
+        const details = parsed.error.errors
+          .map((error) => `${error.path.join(".")}: ${error.message}`)
+          .join("; ");
+        throw new Error(`Invalid benchmark file ${file}: ${details}`);
+      }
+
+      return parsed.data.cases;
+    }),
+  );
+
+  return casesFromFiles.flat();
+}
+
+function validateCorpus(casesToValidate: BenchmarkCase[]) {
+  const errors: string[] = [];
+  const seenIds = new Set<string>();
+
+  for (const testCase of casesToValidate) {
+    if (seenIds.has(testCase.id)) {
+      errors.push(`Duplicate case id: ${testCase.id}`);
+    }
+    seenIds.add(testCase.id);
+
+    if (!isKnownSuite(testCase.suite)) {
+      errors.push(`Unknown suite in ${testCase.id}: ${testCase.suite}`);
     }
 
-    for (const testCase of data.test_cases) {
-      const parsed = TestCaseSchema.safeParse(testCase);
-      if (parsed.success) {
-        // Filter by language if specified
-        if (!languageFilter || languageFilter.includes(parsed.data.language)) {
-          testCases.push(parsed.data);
-        }
-      } else {
-        console.warn(`Warning: Invalid test case in ${file}:`, parsed.error.format());
+    for (const expected of testCase.expected) {
+      if (!testCase.text.includes(expected.text)) {
+        errors.push(
+          `Expected text is not present in ${testCase.id}: ${expected.entity}(${expected.text})`,
+        );
       }
     }
   }
 
-  // Sort by ID for deterministic output
-  return testCases.sort((a, b) => a.id.localeCompare(b.id));
+  if (errors.length > 0) {
+    console.error(`Invalid benchmark corpus:\n${errors.map((error) => `- ${error}`).join("\n")}`);
+    process.exit(1);
+  }
+}
+
+function applyFilters(casesToFilter: BenchmarkCase[], activeFilters: Filters): BenchmarkCase[] {
+  return casesToFilter.filter((testCase) => {
+    if (activeFilters.suites && !activeFilters.suites.has(testCase.suite)) {
+      return false;
+    }
+    if (activeFilters.categories && !activeFilters.categories.has(testCase.category)) {
+      return false;
+    }
+    if (activeFilters.languages && !activeFilters.languages.has(testCase.language)) {
+      return false;
+    }
+    if (activeFilters.split && !activeFilters.split.has(testCase.split)) {
+      return false;
+    }
+    return true;
+  });
 }
 
-/**
- * Call Presidio analyzer API
- */
-async function detectPII(text: string, language: string): Promise<DetectedEntity[]> {
-  if (!text) return [];
+async function runCase(
+  testCase: BenchmarkCase,
+  endpoint: string,
+  globalThreshold: number,
+): Promise<TestResult> {
+  const gating = isGatingCase(testCase);
 
-  const response = await fetch(`${PRESIDIO_URL}/analyze`, {
+  try {
+    const detections = await analyze(testCase, endpoint, globalThreshold);
+    const { matched, missing, unexpected } = scoreDetections(testCase, detections);
+
+    return {
+      case: testCase,
+      passed: missing.length === 0 && unexpected.length === 0,
+      gating,
+      detections,
+      matched,
+      missing,
+      unexpected,
+    };
+  } catch (error) {
+    return {
+      case: testCase,
+      passed: false,
+      gating,
+      detections: [],
+      matched: [],
+      missing: testCase.expected,
+      unexpected: [],
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+async function analyze(
+  testCase: BenchmarkCase,
+  endpoint: string,
+  globalThreshold: number,
+): Promise<Detection[]> {
+  const response = await fetch(endpoint, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({
-      text,
-      language,
-      entities: ENTITY_TYPES,
-      score_threshold: threshold,
+      text: testCase.text,
+      language: testCase.language,
+      entities: entitiesForCase(testCase),
+      score_threshold: globalThreshold,
     }),
   });
 
   if (!response.ok) {
-    throw new Error(`Presidio error: ${response.status} ${await response.text()}`);
+    const body = await response.text();
+    throw new Error(`HTTP ${response.status}: ${body || response.statusText}`);
   }
 
-  const entities = (await response.json()) as Array<{
-    entity_type: string;
-    start: number;
-    end: number;
-    score: number;
-  }>;
-
-  return entities.map((e) => ({
-    type: e.entity_type,
-    text: text.slice(e.start, e.end),
-    start: e.start,
-    end: e.end,
-    score: e.score,
-  }));
-}
-
-/**
- * Check if Presidio is available
- */
-async function checkPresidio(): Promise<boolean> {
-  try {
-    const response = await fetch(`${PRESIDIO_URL}/health`);
-    return response.ok;
-  } catch {
-    return false;
+  const payload = await response.json();
+
+  if (!Array.isArray(payload)) {
+    throw new Error(`Expected analyzer array response, got ${typeof payload}`);
   }
+
+  return payload.map((item) => normalizeDetection(testCase, item));
 }
 
-/**
- * Run a single test case and compare results
- */
-async function runTestCase(testCase: TestCase): Promise<TestResult> {
-  const detected = await detectPII(testCase.text, testCase.language);
+function normalizeDetection(testCase: BenchmarkCase, item: unknown): Detection {
+  if (!item || typeof item !== "object") {
+    throw new Error("Analyzer returned a non-object detection");
+  }
+
+  const record = item as Record<string, unknown>;
+  const entity = record.entity_type ?? record.entity;
+  const start = record.start;
+  const end = record.end;
+  const score = record.score;
 
-  // Match expected entities with detected ones
-  const truePositives: ExpectedEntity[] = [];
-  const falseNegatives: ExpectedEntity[] = [];
-  const matchedDetected = new Set<number>();
+  if (typeof entity !== "string") {
+    throw new Error("Analyzer detection is missing entity_type");
+  }
+  if (typeof start !== "number" || typeof end !== "number") {
+    throw new Error(`Analyzer detection ${entity} is missing numeric offsets`);
+  }
+
+  return {
+    entity,
+    start,
+    end,
+    score: typeof score === "number" ? score : 0,
+    text: testCase.text.slice(start, end),
+  };
+}
+
+function scoreDetections(testCase: BenchmarkCase, detections: Detection[]) {
+  const unmatched = [...detections];
+  const matched: Array<{ expected: ExpectedSpan; detection: Detection }> = [];
+  const missing: ExpectedSpan[] = [];
 
   for (const expected of testCase.expected) {
-    // Find a matching detected entity
-    const matchIndex = detected.findIndex(
-      (d, i) =>
-        !matchedDetected.has(i) &&
-        d.type === expected.type &&
-        normalizeText(d.text).includes(normalizeText(expected.text)),
+    const matchIndex = unmatched.findIndex((detection) =>
+      detectionMatchesExpected(testCase, detection, expected),
     );
 
-    if (matchIndex !== -1) {
-      truePositives.push(expected);
-      matchedDetected.add(matchIndex);
-    } else {
-      falseNegatives.push(expected);
+    if (matchIndex === -1) {
+      missing.push(expected);
+      continue;
     }
+
+    const [detection] = unmatched.splice(matchIndex, 1);
+    matched.push({ expected, detection });
   }
 
-  // Remaining detected entities are false positives
-  const falsePositives = detected.filter((_, i) => !matchedDetected.has(i));
+  return { matched, missing, unexpected: unmatched };
+}
 
-  const passed = falseNegatives.length === 0 && falsePositives.length === 0;
+function detectionMatchesExpected(
+  testCase: BenchmarkCase,
+  detection: Detection,
+  expected: ExpectedSpan,
+): boolean {
+  if (!entityMatches(detection.entity, expected)) {
+    return false;
+  }
 
-  return {
-    id: testCase.id,
-    text: testCase.text,
-    language: testCase.language,
-    passed,
-    expected: testCase.expected,
-    detected,
-    falseNegatives,
-    falsePositives,
-    truePositives,
-  };
+  if (expected.match === "exact") {
+    return normalized(detection.text) === normalized(expected.text);
+  }
+
+  if (expected.match === "overlap") {
+    return spansOverlap(testCase, detection, expected);
+  }
+
+  return boundedContainsMatch(testCase, detection, expected);
 }
 
-/**
- * Normalize text for comparison (lowercase, trim)
- */
-function normalizeText(text: string): string {
-  return text.toLowerCase().trim();
+function entityMatches(entity: string, expected: ExpectedSpan): boolean {
+  return entity === expected.entity || expected.aliases.includes(entity);
 }
 
-/**
- * Calculate accuracy metrics from results
- */
-function calculateMetrics(results: TestResult[]): AccuracyMetrics {
-  let tp = 0;
-  let fp = 0;
-  let fn = 0;
+function boundedContainsMatch(
+  testCase: BenchmarkCase,
+  detection: Detection,
+  expected: ExpectedSpan,
+): boolean {
+  const expectedStart = testCase.text.indexOf(expected.text);
 
-  for (const result of results) {
-    tp += result.truePositives.length;
-    fp += result.falsePositives.length;
-    fn += result.falseNegatives.length;
+  if (expectedStart === -1) {
+    return false;
   }
 
-  const precision = tp + fp > 0 ? tp / (tp + fp) : 1;
-  const recall = tp + fn > 0 ? tp / (tp + fn) : 1;
-  const f1 = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+  const expectedEnd = expectedStart + expected.text.length;
+  const detectionCoversExpected = detection.start <= expectedStart && detection.end >= expectedEnd;
 
-  return {
-    total: results.length,
-    passed: results.filter((r) => r.passed).length,
-    failed: results.filter((r) => !r.passed).length,
-    precision,
-    recall,
-    f1,
-    truePositives: tp,
-    falsePositives: fp,
-    falseNegatives: fn,
-  };
+  if (!detectionCoversExpected || !normalizedContains(detection.text, expected.text)) {
+    return false;
+  }
+
+  const leadingExtraChars = expectedStart - detection.start;
+  const trailingExtraChars = detection.end - expectedEnd;
+
+  return (
+    leadingExtraChars <= MAX_CONTAINS_EDGE_CHARS && trailingExtraChars <= MAX_CONTAINS_EDGE_CHARS
+  );
 }
 
-/**
- * Format percentage for display
- */
-function formatPercent(value: number): string {
-  return `${(value * 100).toFixed(1)}%`;
+function normalizedContains(detectedText: string, expectedText: string): boolean {
+  const detected = normalized(detectedText);
+  const expected = normalized(expectedText);
+
+  return detected.includes(expected);
 }
 
-/**
- * Print metrics in a formatted way
- */
-function printMetrics(label: string, metrics: AccuracyMetrics): void {
-  const status = metrics.failed === 0 ? "✓" : "⚠";
+function spansOverlap(
+  testCase: BenchmarkCase,
+  detection: Detection,
+  expected: ExpectedSpan,
+): boolean {
+  const expectedStart = testCase.text.indexOf(expected.text);
+
+  if (expectedStart === -1) {
+    return false;
+  }
+
+  const expectedEnd = expectedStart + expected.text.length;
+  return detection.start < expectedEnd && detection.end > expectedStart;
+}
+
+function normalized(value: string): string {
+  return value
+    .normalize("NFKC")
+    .toLowerCase()
+    .replace(/^[\s"'([<{]+/g, "")
+    .replace(/[\s"').,;:!?>\]}]+$/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function entitiesForCase(testCase: BenchmarkCase): string[] {
+  if (testCase.entities) {
+    return testCase.entities;
+  }
+
+  if (testCase.expected.length > 0) {
+    return [...new Set(testCase.expected.map((expected) => expected.entity))];
+  }
+
+  return [...(SUITES[testCase.suite]?.defaultEntities ?? [])];
+}
+
+function isGatingCase(testCase: BenchmarkCase): boolean {
+  return testCase.gate ?? (testCase.category === "core" || testCase.category === "precision");
+}
+
+function printReport(
+  results: TestResult[],
+  endpoint: string,
+  activeThreshold: number,
+  printVerbose: boolean,
+) {
+  const gatingResults = results.filter((result) => result.gating);
+  const reportOnlyResults = results.filter((result) => !result.gating);
+  const failed = results.filter((result) => !result.passed);
+  const gatingFailures = gatingResults.filter((result) => !result.passed);
+  const errors = results.filter((result) => result.error);
+
+  console.log("PII accuracy benchmark");
+  console.log(`Endpoint: ${endpoint}`);
+  console.log(`Default threshold: ${activeThreshold}`);
   console.log(
-    `  ${label.padEnd(20)} P=${formatPercent(metrics.precision).padStart(6)}  ` +
-      `R=${formatPercent(metrics.recall).padStart(6)}  ` +
-      `F1=${formatPercent(metrics.f1).padStart(6)}  ${status}`,
+    `Cases: ${results.length} (${gatingResults.length} gating, ${reportOnlyResults.length} report-only)`,
   );
-}
+  console.log("");
+
+  printMetrics("Overall", [["all", aggregate(results)]]);
+  printMetrics(
+    "By suite",
+    groupMetrics(results, (result) => result.case.suite),
+  );
+  printMetrics(
+    "By category",
+    groupMetrics(results, (result) => result.case.category),
+  );
+  printMetrics(
+    "By language",
+    groupMetrics(results, (result) => result.case.language),
+  );
+  printMetrics("By entity", entityMetrics(results));
 
-/**
- * Main benchmark execution
- */
-async function main(): Promise<void> {
-  console.log("\n╔════════════════════════════════════════════════════════════╗");
-  console.log("║           PII Detection Accuracy Benchmark                 ║");
-  console.log("╚════════════════════════════════════════════════════════════╝\n");
+  if (failed.length > 0) {
+    console.log("");
+    console.log(
+      `Failures: ${failed.length} total, ${gatingFailures.length} gating, ${errors.length} errors`,
+    );
 
-  // Check Presidio availability
-  console.log(`Presidio URL: ${PRESIDIO_URL}`);
-  console.log(`Threshold: ${threshold}`);
-  if (languageFilter) {
-    console.log(`Languages: ${languageFilter.join(", ")}`);
+    const failuresToPrint = printVerbose ? failed : failed.slice(0, MAX_FAILURES_TO_PRINT);
+    for (const result of failuresToPrint) {
+      printFailure(result);
+    }
+
+    if (!printVerbose && failed.length > failuresToPrint.length) {
+      console.log(`... ${failed.length - failuresToPrint.length} more failures hidden`);
+      console.log("Run with --verbose to print all failure details.");
+    }
   }
+}
 
-  if (!(await checkPresidio())) {
-    console.error("\n✗ Presidio is not available. Start it with:");
-    console.error("  docker compose up presidio-analyzer -d\n");
-    process.exit(1);
+function printMetrics(title: string, rows: Array<[string, Metrics]>) {
+  console.log(title);
+  console.log(
+    [
+      "name".padEnd(24),
+      "cases".padStart(5),
+      "pass".padStart(7),
+      "P".padStart(7),
+      "R".padStart(7),
+      "F1".padStart(7),
+      "F2".padStart(7),
+      "err".padStart(5),
+    ].join(" "),
+  );
+
+  for (const [name, metrics] of rows) {
+    console.log(
+      [
+        name.slice(0, 24).padEnd(24),
+        String(metrics.cases).padStart(5),
+        percent(metrics.passed, metrics.cases).padStart(7),
+        percent(metrics.tp, metrics.tp + metrics.fp).padStart(7),
+        percent(metrics.tp, metrics.tp + metrics.fn).padStart(7),
+        fScore(metrics.tp, metrics.fp, metrics.fn, 1).padStart(7),
+        fScore(metrics.tp, metrics.fp, metrics.fn, 2).padStart(7),
+        String(metrics.errors).padStart(5),
+      ].join(" "),
+    );
   }
-  console.log("Presidio: ✓ Connected\n");
+  console.log("");
+}
 
-  // Load test cases from directory
-  const testCases = await loadTestCases();
+function groupMetrics(
+  results: TestResult[],
+  keyFn: (result: TestResult) => string,
+): Array<[string, Metrics]> {
+  const groups = new Map<string, TestResult[]>();
 
-  if (testCases.length === 0) {
-    console.error("No test cases found in", TEST_DATA_DIR);
-    process.exit(1);
+  for (const result of results) {
+    const key = keyFn(result);
+    groups.set(key, [...(groups.get(key) ?? []), result]);
   }
 
-  console.log(`Running ${testCases.length} test cases...\n`);
-
-  // Run all test cases
-  const results: TestResult[] = [];
-  for (const testCase of testCases) {
-    try {
-      const result = await runTestCase(testCase);
-      results.push(result);
-
-      if (verbose) {
-        const icon = result.passed ? "✓" : "✗";
-        console.log(`${icon} ${result.id}`);
-        if (!result.passed) {
-          if (result.falseNegatives.length > 0) {
-            console.log(`    Missed: ${result.falseNegatives.map((e) => `${e.type}:"${e.text}"`).join(", ")}`);
-          }
-          if (result.falsePositives.length > 0) {
-            console.log(
-              `    Wrong:  ${result.falsePositives.map((e) => `${e.type}:"${e.text}" (${e.score.toFixed(2)})`).join(", ")}`,
-            );
-          }
-        }
+  return [...groups.entries()]
+    .map(([key, group]) => [key, aggregate(group)] as [string, Metrics])
+    .sort(([left], [right]) => left.localeCompare(right));
+}
+
+function entityMetrics(results: TestResult[]): Array<[string, Metrics]> {
+  const entities = new Map<string, Metrics>();
+
+  for (const result of results) {
+    const seenEntities = new Set(entitiesForCase(result.case));
+    for (const expected of result.case.expected) {
+      seenEntities.add(expected.entity);
+    }
+    for (const detection of result.unexpected) {
+      seenEntities.add(detection.entity);
+    }
+
+    for (const entity of seenEntities) {
+      if (!entities.has(entity)) {
+        entities.set(entity, emptyMetrics());
       }
-    } catch (error) {
-      console.error(`Error in test ${testCase.id}:`, error);
     }
-  }
 
-  // Calculate overall metrics
-  const overall = calculateMetrics(results);
-
-  // Calculate metrics by entity type
-  const byEntityType: Record<string, AccuracyMetrics> = {};
-  for (const entityType of ENTITY_TYPES) {
-    const filtered = results.map((r) => ({
-      ...r,
-      expected: r.expected.filter((e) => e.type === entityType),
-      truePositives: r.truePositives.filter((e) => e.type === entityType),
-      falseNegatives: r.falseNegatives.filter((e) => e.type === entityType),
-      falsePositives: r.falsePositives.filter((e) => e.type === entityType),
-    }));
-    // Only include if there are test cases for this type
-    const hasTestCases = filtered.some((r) => r.expected.length > 0 || r.falsePositives.length > 0);
-    if (hasTestCases) {
-      byEntityType[entityType] = calculateMetrics(filtered);
+    for (const entity of seenEntities) {
+      const bucket = entities.get(entity);
+
+      if (!bucket) {
+        continue;
+      }
+
+      bucket.cases += 1;
+      bucket.errors += result.error ? 1 : 0;
+      const tp = result.matched.filter((match) => match.expected.entity === entity).length;
+      const fn = result.missing.filter((missing) => missing.entity === entity).length;
+      const fp = result.unexpected.filter((unexpected) => unexpected.entity === entity).length;
+
+      bucket.tp += tp;
+      bucket.fn += fn;
+      bucket.fp += fp;
+      bucket.passed += result.error || fn > 0 || fp > 0 ? 0 : 1;
     }
   }
 
-  // Calculate metrics by language
-  const languages = [...new Set(results.map((r) => r.language))];
-  const byLanguage: Record<string, AccuracyMetrics> = {};
-  for (const lang of languages) {
-    byLanguage[lang] = calculateMetrics(results.filter((r) => r.language === lang));
+  return [...entities.entries()].sort(([left], [right]) => left.localeCompare(right));
+}
+
+function aggregate(results: TestResult[]): Metrics {
+  const metrics = emptyMetrics();
+
+  for (const result of results) {
+    metrics.cases += 1;
+    metrics.passed += result.passed ? 1 : 0;
+    metrics.errors += result.error ? 1 : 0;
+    metrics.tp += result.matched.length;
+    metrics.fp += result.unexpected.length;
+    metrics.fn += result.error ? result.case.expected.length : result.missing.length;
   }
 
-  // Print report
-  console.log("\n────────────────────────────────────────────────────────────");
-  console.log("                         RESULTS");
-  console.log("────────────────────────────────────────────────────────────\n");
+  return metrics;
+}
 
-  console.log(`Overall: ${overall.passed}/${overall.total} passed\n`);
+function emptyMetrics(): Metrics {
+  return {
+    cases: 0,
+    passed: 0,
+    errors: 0,
+    tp: 0,
+    fp: 0,
+    fn: 0,
+  };
+}
 
-  console.log("Metrics (P=Precision, R=Recall, F1=F1-Score):\n");
+function printFailure(result: TestResult) {
+  const mode = result.gating ? "gating" : "report-only";
+  console.log(`- ${result.case.id} [${result.case.suite}/${result.case.category}/${mode}]`);
 
-  console.log(
-    `  ${"OVERALL".padEnd(20)} P=${formatPercent(overall.precision).padStart(6)}  ` +
-      `R=${formatPercent(overall.recall).padStart(6)}  ` +
-      `F1=${formatPercent(overall.f1).padStart(6)}\n`,
-  );
+  if (result.error) {
+    console.log(`  error: ${result.error}`);
+    return;
+  }
 
-  console.log("By Entity Type:");
-  for (const [type, metrics] of Object.entries(byEntityType)) {
-    printMetrics(type, metrics);
+  if (result.missing.length > 0) {
+    console.log(
+      `  missing: ${result.missing
+        .map((expected) => `${expected.entity}(${expected.text})`)
+        .join(", ")}`,
+    );
   }
 
-  console.log("\nBy Language:");
-  for (const [lang, metrics] of Object.entries(byLanguage)) {
-    printMetrics(lang.toUpperCase(), metrics);
+  if (result.unexpected.length > 0) {
+    console.log(
+      `  unexpected: ${result.unexpected
+        .map((detection) => `${detection.entity}(${detection.text}, ${detection.score.toFixed(2)})`)
+        .join(", ")}`,
+    );
   }
+}
 
-  // Print false negatives (missed PII)
-  const allFalseNegatives = results.flatMap((r) =>
-    r.falseNegatives.map((fn) => ({ testId: r.id, text: r.text, entity: fn })),
-  );
-  if (allFalseNegatives.length > 0) {
-    console.log("\n────────────────────────────────────────────────────────────");
-    console.log(`False Negatives (Missed PII): ${allFalseNegatives.length}`);
-    console.log("────────────────────────────────────────────────────────────");
-    for (const { testId, entity } of allFalseNegatives) {
-      console.log(`  ✗ "${entity.text}" (${entity.type}) in ${testId}`);
-    }
+function percent(numerator: number, denominator: number): string {
+  if (denominator === 0) {
+    return "n/a";
   }
 
-  // Print false positives (wrong detections)
-  const allFalsePositives = results.flatMap((r) =>
-    r.falsePositives.map((fp) => ({ testId: r.id, text: r.text, entity: fp })),
-  );
-  if (allFalsePositives.length > 0) {
-    console.log("\n────────────────────────────────────────────────────────────");
-    console.log(`False Positives (Wrong Detections): ${allFalsePositives.length}`);
-    console.log("────────────────────────────────────────────────────────────");
-    for (const { testId, entity } of allFalsePositives) {
-      console.log(`  ✗ "${entity.text}" (${entity.type}, score=${entity.score.toFixed(2)}) in ${testId}`);
-    }
+  return `${((numerator / denominator) * 100).toFixed(1)}%`;
+}
+
+function fScore(tp: number, fp: number, fn: number, beta: number): string {
+  const betaSquared = beta * beta;
+  const denominator = (1 + betaSquared) * tp + betaSquared * fn + fp;
+
+  if (denominator === 0) {
+    return "n/a";
   }
 
-  console.log("\n────────────────────────────────────────────────────────────\n");
+  return `${(((1 + betaSquared) * tp) / denominator).toFixed(3)}`;
+}
 
-  // Exit with error code if tests failed
-  if (overall.failed > 0) {
-    process.exit(1);
+function buildFilters(values: typeof argv.values): Filters {
+  return {
+    suites: csvSet(values.suite),
+    categories: csvSet(values.category),
+    languages: csvSet(values.languages),
+    split: csvSet(values.split),
+  };
+}
+
+function csvSet(value: string | boolean | undefined): Set<string> | undefined {
+  if (typeof value !== "string" || value.trim() === "") {
+    return undefined;
   }
+
+  return new Set(
+    value
+      .split(",")
+      .map((part) => part.trim())
+      .filter(Boolean),
+  );
 }
 
-main().catch((error) => {
-  console.error("Benchmark failed:", error);
-  process.exit(1);
-});
+function parseThreshold(value: string): number {
+  const parsed = Number(value);
+
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new Error(`Invalid threshold: ${value}`);
+  }
+
+  return parsed;
+}
+
+function printHelp() {
+  console.log(`Usage: bun run benchmarks/pii-accuracy/run.ts [options]
+
+Options:
+  --url <url>              Analyze endpoint. Default: ${DEFAULT_ANALYZE_URL}
+  --threshold <0..1>       Default score threshold. Default: ${DEFAULT_THRESHOLD}
+  --suite <csv>            Filter suites, e.g. core,precision
+  --category <csv>         Filter categories: core,precision,eval,hard
+  --languages <csv>        Filter languages, e.g. en,de,it
+  --split <csv>            Filter split: dev,test
+  --list-suites            Print suites and exit
+  --verbose                Print all failure details
+  --help                   Print this help
+`);
+}
diff --git a/benchmarks/pii-accuracy/taxonomy.ts b/benchmarks/pii-accuracy/taxonomy.ts
new file mode 100644
index 0000000..31c6de6
--- /dev/null
+++ b/benchmarks/pii-accuracy/taxonomy.ts
@@ -0,0 +1,59 @@
+export type SuiteDefinition = {
+  description: string;
+  defaultEntities: readonly string[];
+};
+
+export const CONFIGURED_PII_ENTITIES = [
+  "CREDIT_CARD",
+  "EMAIL_ADDRESS",
+  "IBAN_CODE",
+  "IP_ADDRESS",
+  "LOCATION",
+  "PERSON",
+  "PHONE_NUMBER",
+] as const;
+
+export const SUPPORTED_LANGUAGES = ["en", "de", "es", "fr", "it", "nl", "pl", "pt", "ro"] as const;
+
+export const SUITES: Record<string, SuiteDefinition> = {
+  core: {
+    description: "Configured PII entities used by Pasteguard's default detection setup.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  precision: {
+    description: "Negative controls where common non-PII strings must not be flagged.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  "precision-paragraphs": {
+    description: "Longer non-PII paragraphs with lookalike strings and operational text.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  "multilingual-sentences": {
+    description: "Sentence-level coverage for every configured entity across supported languages.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  "multilingual-paragraphs": {
+    description: "Paragraph-level workflow coverage for every configured entity across languages.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  boundaries: {
+    description: "Span boundary checks around punctuation, brackets, quotes, and repeated context.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  hard: {
+    description: "Known difficult, ambiguous, or edge-case strings. Report-only by default.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+  eval: {
+    description: "Multilingual smoke corpus for trend tracking across configured languages.",
+    defaultEntities: CONFIGURED_PII_ENTITIES,
+  },
+};
+
+export function isKnownSuite(suite: string): boolean {
+  return suite in SUITES;
+}
+
+export function suiteNames(): string[] {
+  return Object.keys(SUITES).sort();
+}
diff --git a/benchmarks/pii-accuracy/test-data/boundaries.yaml b/benchmarks/pii-accuracy/test-data/boundaries.yaml
new file mode 100644
index 0000000..8061f43
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/boundaries.yaml
@@ -0,0 +1,88 @@
+cases:
+  - id: boundary_email_parentheses
+    suite: boundaries
+    category: hard
+    split: test
+    language: en
+    text: "Primary contact (zoe.parker+legal@example.co.uk) approved the change."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: zoe.parker+legal@example.co.uk
+        match: exact
+
+  - id: boundary_phone_quotes
+    suite: boundaries
+    category: hard
+    split: test
+    language: en
+    text: "The note says call '+1 212-555-0199' after the legal review."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +1 212-555-0199
+        match: contains
+
+  - id: boundary_credit_card_commas
+    suite: boundaries
+    category: hard
+    split: test
+    language: en
+    text: "The escrow record has card 4111 1111 1111 1111, but the invoice does not."
+    expected:
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+
+  - id: boundary_iban_brackets
+    suite: boundaries
+    category: hard
+    split: test
+    language: de
+    text: "Das Konto [DE89370400440532013000] wurde nur im Audit-Log genannt."
+    expected:
+      - entity: IBAN_CODE
+        text: DE89370400440532013000
+        match: exact
+
+  - id: boundary_ip_brackets
+    suite: boundaries
+    category: hard
+    split: test
+    language: en
+    text: "The firewall event recorded src=[198.51.100.44] and no destination owner."
+    expected:
+      - entity: IP_ADDRESS
+        text: 198.51.100.44
+        match: exact
+
+  - id: boundary_person_sentence_start
+    suite: boundaries
+    category: hard
+    split: test
+    language: en
+    text: "Priya Shah approved access. The rest of the sentence is generic."
+    expected:
+      - entity: PERSON
+        text: Priya Shah
+        match: contains
+
+  - id: boundary_location_with_punctuation
+    suite: boundaries
+    category: hard
+    split: test
+    language: fr
+    text: "Le bureau concerne Lyon; aucune autre ville n'est dans le message."
+    expected:
+      - entity: LOCATION
+        text: Lyon
+        match: contains
+
+  - id: boundary_repeated_context_single_email
+    suite: boundaries
+    category: hard
+    split: test
+    language: en
+    text: "Email the customer email owner at customer.owner@example.com; do not email the team alias."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: customer.owner@example.com
+        match: exact
diff --git a/benchmarks/pii-accuracy/test-data/core.yaml b/benchmarks/pii-accuracy/test-data/core.yaml
new file mode 100644
index 0000000..5fcc54a
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/core.yaml
@@ -0,0 +1,341 @@
+cases:
+  - id: core_email_en
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "Email Alice at alice.smith@example.com before noon."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: alice.smith@example.com
+        match: exact
+
+  - id: core_email_de
+    suite: core
+    category: core
+    split: test
+    language: de
+    text: "Schreibe an max.mueller@example.de, wenn der Export fertig ist."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: max.mueller@example.de
+        match: exact
+
+  - id: core_email_plus_subdomain
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "Send invoices to alice.billing+eu@sub.example.co.uk today."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: alice.billing+eu@sub.example.co.uk
+        match: exact
+
+  - id: core_email_uppercase
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "The backup mailbox is SECURITY.ALERTS@EXAMPLE.ORG."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: SECURITY.ALERTS@EXAMPLE.ORG
+        match: exact
+
+  - id: core_phone_en
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "The customer phone number is +1 415-555-2671."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +1 415-555-2671
+        match: contains
+
+  - id: core_phone_de
+    suite: core
+    category: core
+    split: test
+    language: de
+    text: "Die Telefonnummer des Kontakts ist +49 30 12345678."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +49 30 12345678
+        match: contains
+
+  - id: core_phone_fr
+    suite: core
+    category: core
+    split: test
+    language: fr
+    text: "Le numéro de téléphone est +33 1 42 68 53 00."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +33 1 42 68 53 00
+        match: contains
+
+  - id: core_phone_es
+    suite: core
+    category: core
+    split: test
+    language: es
+    text: "El teléfono de soporte es +34 91 123 45 67."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +34 91 123 45 67
+        match: contains
+
+  - id: core_phone_it
+    suite: core
+    category: core
+    split: test
+    language: it
+    text: "Il numero di telefono è +39 06 1234 5678."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +39 06 1234 5678
+        match: contains
+
+  - id: core_phone_nl
+    suite: core
+    category: core
+    split: test
+    language: nl
+    text: "Het telefoonnummer is +31 20 123 4567."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +31 20 123 4567
+        match: contains
+
+  - id: core_phone_pt
+    suite: core
+    category: core
+    split: test
+    language: pt
+    text: "O número de telefone é +351 21 123 4567."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +351 21 123 4567
+        match: contains
+
+  - id: core_phone_pl
+    suite: core
+    category: core
+    split: test
+    language: pl
+    text: "Numer telefonu kontaktowego to +48 22 123 45 67."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +48 22 123 45 67
+        match: contains
+
+  - id: core_phone_ro
+    suite: core
+    category: core
+    split: test
+    language: ro
+    text: "Numărul de telefon este +40 21 123 4567."
+    expected:
+      - entity: PHONE_NUMBER
+        text: +40 21 123 4567
+        match: contains
+
+  - id: core_credit_card_spaces
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "Use test card 4111 1111 1111 1111 for the checkout sandbox."
+    expected:
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+
+  - id: core_credit_card_hyphen
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "Use card 4012-8888-8888-1881 for the payment test."
+    expected:
+      - entity: CREDIT_CARD
+        text: 4012-8888-8888-1881
+        match: exact
+
+  - id: core_iban_de
+    suite: core
+    category: core
+    split: test
+    language: de
+    text: "Das Auszahlungskonto lautet DE89370400440532013000."
+    expected:
+      - entity: IBAN_CODE
+        text: DE89370400440532013000
+        match: exact
+
+  - id: core_iban_it
+    suite: core
+    category: core
+    split: test
+    language: it
+    text: "Il conto del fornitore e IT60X0542811101000000123456."
+    expected:
+      - entity: IBAN_CODE
+        text: IT60X0542811101000000123456
+        match: exact
+
+  - id: core_iban_fr
+    suite: core
+    category: core
+    split: test
+    language: fr
+    text: "Le compte fournisseur est FR1420041010050500013M02606."
+    expected:
+      - entity: IBAN_CODE
+        text: FR1420041010050500013M02606
+        match: exact
+
+  - id: core_iban_es
+    suite: core
+    category: core
+    split: test
+    language: es
+    text: "La cuenta de pago es ES9121000418450200051332."
+    expected:
+      - entity: IBAN_CODE
+        text: ES9121000418450200051332
+        match: exact
+
+  - id: core_iban_nl
+    suite: core
+    category: core
+    split: test
+    language: nl
+    text: "De betaalrekening is NL91ABNA0417164300."
+    expected:
+      - entity: IBAN_CODE
+        text: NL91ABNA0417164300
+        match: exact
+
+  - id: core_iban_pt
+    suite: core
+    category: core
+    split: test
+    language: pt
+    text: "A conta de pagamento e PT50000201231234567890154."
+    expected:
+      - entity: IBAN_CODE
+        text: PT50000201231234567890154
+        match: exact
+
+  - id: core_iban_pl
+    suite: core
+    category: core
+    split: test
+    language: pl
+    text: "Rachunek płatności to PL61109010140000071219812874."
+    expected:
+      - entity: IBAN_CODE
+        text: PL61109010140000071219812874
+        match: exact
+
+  - id: core_iban_ro
+    suite: core
+    category: core
+    split: test
+    language: ro
+    text: "Contul de plată este RO49AAAA1B31007593840000."
+    expected:
+      - entity: IBAN_CODE
+        text: RO49AAAA1B31007593840000
+        match: exact
+
+  - id: core_ipv4_public
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "The last login came from IP address 203.0.113.42."
+    expected:
+      - entity: IP_ADDRESS
+        text: 203.0.113.42
+        match: exact
+
+  - id: core_ipv4_private
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "The admin panel is reachable at private IP 10.24.8.5."
+    expected:
+      - entity: IP_ADDRESS
+        text: 10.24.8.5
+        match: exact
+
+  - id: core_person_en
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "John Smith approved the access request yesterday."
+    expected:
+      - entity: PERSON
+        text: John Smith
+        match: contains
+
+  - id: core_person_non_western_arabic
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "Please add Amina Al-Rashid to the project access list."
+    expected:
+      - entity: PERSON
+        text: Amina Al-Rashid
+        match: contains
+
+  - id: core_person_non_western_chinese
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "The reviewer Wei Zhang approved the vendor exception."
+    expected:
+      - entity: PERSON
+        text: Wei Zhang
+        match: contains
+
+  - id: core_person_non_western_indian
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "Escalate the open ticket to Priya Krishnan in support."
+    expected:
+      - entity: PERSON
+        text: Priya Krishnan
+        match: contains
+
+  - id: core_person_non_western_turkish_de
+    suite: core
+    category: core
+    split: test
+    language: de
+    text: "Bitte trage Mehmet Yıldız in die Zugriffsliste ein."
+    expected:
+      - entity: PERSON
+        text: Mehmet Yıldız
+        match: contains
+
+  - id: core_location_en
+    suite: core
+    category: core
+    split: test
+    language: en
+    text: "The backup office is located in Berlin."
+    expected:
+      - entity: LOCATION
+        text: Berlin
+        match: contains
diff --git a/benchmarks/pii-accuracy/test-data/de.yaml b/benchmarks/pii-accuracy/test-data/de.yaml
deleted file mode 100644
index 530fd04..0000000
--- a/benchmarks/pii-accuracy/test-data/de.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-test_cases:
-  # Phone (needs German context words)
-  - id: de_phone
-    text: "Telefon: +49 171 1234567"
-    language: de
-    expected:
-      - type: PHONE_NUMBER
-        text: "+49 171 1234567"
-
-  # Person (NER model)
-  - id: de_person
-    text: "Meeting mit Max Müller morgen um 10 Uhr"
-    language: de
-    expected:
-      - type: PERSON
-        text: "Max Müller"
-
-  # Location (NER model)
-  - id: de_location
-    text: "Das Meeting findet in München statt"
-    language: de
-    expected:
-      - type: LOCATION
-        text: "München"
-
-  # Mixed (real-world prompt)
-  - id: de_mixed
-    text: "Kontaktdaten von Hans Meier: hans.meier@example.com, Telefon 0171-1234567"
-    language: de
-    expected:
-      - type: PERSON
-        text: "Hans Meier"
-      - type: EMAIL_ADDRESS
-        text: "hans.meier@example.com"
-      - type: PHONE_NUMBER
-        text: "0171-1234567"
-
-  # False Positive
-  - id: de_fp
-    text: "Das Wetter ist heute sehr schön"
-    language: de
-    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/en.yaml b/benchmarks/pii-accuracy/test-data/en.yaml
deleted file mode 100644
index 13bd6aa..0000000
--- a/benchmarks/pii-accuracy/test-data/en.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-test_cases:
-  # Phone (needs English context words)
-  - id: en_phone
-    text: "Phone: (555) 123-4567"
-    language: en
-    expected:
-      - type: PHONE_NUMBER
-        text: "(555) 123-4567"
-
-  # Person (NER model)
-  - id: en_person
-    text: "Schedule a meeting with John Smith tomorrow"
-    language: en
-    expected:
-      - type: PERSON
-        text: "John Smith"
-
-  # Location (NER model)
-  - id: en_location
-    text: "The conference is in New York next week"
-    language: en
-    expected:
-      - type: LOCATION
-        text: "New York"
-
-  # Mixed (real-world prompt)
-  - id: en_mixed
-    text: "Customer John Doe (john@example.com), phone (555) 987-6543"
-    language: en
-    expected:
-      - type: PERSON
-        text: "John Doe"
-      - type: EMAIL_ADDRESS
-        text: "john@example.com"
-      - type: PHONE_NUMBER
-        text: "(555) 987-6543"
-
-  # False Positive
-  - id: en_fp
-    text: "The weather is nice today"
-    language: en
-    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/es.yaml b/benchmarks/pii-accuracy/test-data/es.yaml
deleted file mode 100644
index ac7b305..0000000
--- a/benchmarks/pii-accuracy/test-data/es.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-test_cases:
-  # Phone (needs Spanish context words)
-  - id: es_phone
-    text: "Teléfono: +34 612 345 678"
-    language: es
-    expected:
-      - type: PHONE_NUMBER
-        text: "+34 612 345 678"
-
-  # Person (NER model)
-  - id: es_person
-    text: "Reunión con Ana Martínez mañana a las 10"
-    language: es
-    expected:
-      - type: PERSON
-        text: "Ana Martínez"
-
-  # Location (NER model)
-  - id: es_location
-    text: "La reunión será en Madrid la próxima semana"
-    language: es
-    expected:
-      - type: LOCATION
-        text: "Madrid"
-
-  # Mixed (real-world prompt)
-  - id: es_mixed
-    text: "Contacto de Carlos García: carlos.garcia@example.es, teléfono +34 698 765 432"
-    language: es
-    expected:
-      - type: PERSON
-        text: "Carlos García"
-      - type: EMAIL_ADDRESS
-        text: "carlos.garcia@example.es"
-      - type: PHONE_NUMBER
-        text: "+34 698 765 432"
-
-  # False Positive
-  - id: es_fp
-    text: "El clima está muy agradable hoy"
-    language: es
-    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/eval.yaml b/benchmarks/pii-accuracy/test-data/eval.yaml
new file mode 100644
index 0000000..66e86ca
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/eval.yaml
@@ -0,0 +1,257 @@
+cases:
+  - id: eval_de_mixed_contact
+    suite: eval
+    category: eval
+    split: test
+    language: de
+    text: "Maria Keller aus Berlin ist unter maria.keller@example.de erreichbar."
+    expected:
+      - entity: PERSON
+        text: Maria Keller
+        match: contains
+      - entity: LOCATION
+        text: Berlin
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: maria.keller@example.de
+        match: exact
+
+  - id: eval_it_mixed_contact
+    suite: eval
+    category: eval
+    split: test
+    language: it
+    text: "Luca Bianchi lavora a Roma e usa luca.bianchi@example.it."
+    expected:
+      - entity: PERSON
+        text: Luca Bianchi
+        match: contains
+      - entity: LOCATION
+        text: Roma
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: luca.bianchi@example.it
+        match: exact
+
+  - id: eval_fr_mixed_contact
+    suite: eval
+    category: eval
+    split: test
+    language: fr
+    text: "Claire Martin vit a Paris et son email est claire.martin@example.fr."
+    expected:
+      - entity: PERSON
+        text: Claire Martin
+        match: contains
+      - entity: LOCATION
+        text: Paris
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: claire.martin@example.fr
+        match: exact
+
+  - id: eval_es_mixed_contact
+    suite: eval
+    category: eval
+    split: test
+    language: es
+    text: "Carlos Garcia vive en Madrid y responde en carlos.garcia@example.es."
+    expected:
+      - entity: PERSON
+        text: Carlos Garcia
+        match: contains
+      - entity: LOCATION
+        text: Madrid
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: carlos.garcia@example.es
+        match: exact
+
+  - id: eval_nl_phone_email
+    suite: eval
+    category: eval
+    split: test
+    language: nl
+    text: "Sanne de Vries gebruikt sanne@example.nl en telefoon +31 20 123 4567."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: sanne@example.nl
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +31 20 123 4567
+        match: contains
+
+  - id: eval_nl_person_location
+    suite: eval
+    category: eval
+    split: test
+    language: nl
+    text: "Jan de Vries woont in Rotterdam en gebruikt jan.devries@example.nl."
+    expected:
+      - entity: PERSON
+        text: Jan de Vries
+        match: contains
+      - entity: LOCATION
+        text: Rotterdam
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: jan.devries@example.nl
+        match: exact
+
+  - id: eval_pt_phone_email
+    suite: eval
+    category: eval
+    split: test
+    language: pt
+    text: "Joao Silva usa joao.silva@example.pt e telefone +351 21 123 4567."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: joao.silva@example.pt
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +351 21 123 4567
+        match: contains
+
+  - id: eval_pt_person_location
+    suite: eval
+    category: eval
+    split: test
+    language: pt
+    text: "Tiago Costa mora no Porto e usa tiago.costa@example.pt."
+    expected:
+      - entity: PERSON
+        text: Tiago Costa
+        match: contains
+      - entity: LOCATION
+        text: Porto
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: tiago.costa@example.pt
+        match: exact
+
+  - id: eval_pl_mixed_contact
+    suite: eval
+    category: eval
+    split: test
+    language: pl
+    text: "Anna Kowalska z Warszawy używa adresu anna.kowalska@example.pl."
+    expected:
+      - entity: PERSON
+        text: Anna Kowalska
+        match: contains
+      - entity: LOCATION
+        text: Warszawy
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: anna.kowalska@example.pl
+        match: exact
+
+  - id: eval_ro_mixed_contact
+    suite: eval
+    category: eval
+    split: test
+    language: ro
+    text: "Andrei Popescu din București folosește adresa andrei.popescu@example.ro."
+    expected:
+      - entity: PERSON
+        text: Andrei Popescu
+        match: contains
+      - entity: LOCATION
+        text: București
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: andrei.popescu@example.ro
+        match: exact
+
+  - id: eval_en_security_event
+    suite: eval
+    category: eval
+    split: test
+    language: en
+    text: "Alice Brown signed in from 8.8.8.8 and confirmed alice.brown@example.com."
+    expected:
+      - entity: PERSON
+        text: Alice Brown
+        match: contains
+      - entity: IP_ADDRESS
+        text: 8.8.8.8
+        match: exact
+      - entity: EMAIL_ADDRESS
+        text: alice.brown@example.com
+        match: exact
+
+  - id: eval_de_payment_contact
+    suite: eval
+    category: eval
+    split: test
+    language: de
+    text: "Max Berger aus Hamburg nutzt das Konto DE89370400440532013000 und max.berger@example.de."
+    expected:
+      - entity: PERSON
+        text: Max Berger
+        match: contains
+      - entity: LOCATION
+        text: Hamburg
+        match: contains
+      - entity: IBAN_CODE
+        text: DE89370400440532013000
+        match: exact
+      - entity: EMAIL_ADDRESS
+        text: max.berger@example.de
+        match: exact
+
+  - id: eval_de_spaced_iban
+    suite: eval
+    category: eval
+    split: test
+    language: de
+    text: "Bitte ueberweisen Sie auf IBAN DE89 3704 0044 0532 0130 00."
+    expected:
+      - entity: IBAN_CODE
+        text: DE89 3704 0044 0532 0130 00
+        match: exact
+
+  - id: eval_it_payment_contact
+    suite: eval
+    category: eval
+    split: test
+    language: it
+    text: "Giulia Rossi vive a Milano e usa il conto IT60X0542811101000000123456."
+    expected:
+      - entity: PERSON
+        text: Giulia Rossi
+        match: contains
+      - entity: LOCATION
+        text: Milano
+        match: contains
+      - entity: IBAN_CODE
+        text: IT60X0542811101000000123456
+        match: exact
+
+  - id: eval_it_spaced_iban
+    suite: eval
+    category: eval
+    split: test
+    language: it
+    text: "Bonifico sull'IBAN IT60 X054 2811 1010 0000 0123 456 entro lunedi."
+    expected:
+      - entity: IBAN_CODE
+        text: IT60 X054 2811 1010 0000 0123 456
+        match: exact
+
+  - id: eval_fr_payment_contact
+    suite: eval
+    category: eval
+    split: test
+    language: fr
+    text: "Nicolas Bernard à Lyon utilise le compte FR1420041010050500013M02606."
+    expected:
+      - entity: PERSON
+        text: Nicolas Bernard
+        match: contains
+      - entity: LOCATION
+        text: Lyon
+        match: contains
+      - entity: IBAN_CODE
+        text: FR1420041010050500013M02606
+        match: exact
diff --git a/benchmarks/pii-accuracy/test-data/fr.yaml b/benchmarks/pii-accuracy/test-data/fr.yaml
deleted file mode 100644
index e2423c8..0000000
--- a/benchmarks/pii-accuracy/test-data/fr.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-test_cases:
-  # Phone (needs French context words)
-  - id: fr_phone
-    text: "Téléphone: 06 12 34 56 78"
-    language: fr
-    expected:
-      - type: PHONE_NUMBER
-        text: "06 12 34 56 78"
-
-  # Person (NER model)
-  - id: fr_person
-    text: "J'ai une réunion avec Marie Dubois demain à 10h"
-    language: fr
-    expected:
-      - type: PERSON
-        text: "Marie Dubois"
-
-  # Location (NER model)
-  - id: fr_location
-    text: "La conférence aura lieu à Lyon la semaine prochaine"
-    language: fr
-    expected:
-      - type: LOCATION
-        text: "Lyon"
-
-  # Mixed (real-world prompt)
-  - id: fr_mixed
-    text: "Contact de Jean Dupont: jean.dupont@example.fr, téléphone 06 98 76 54 32"
-    language: fr
-    expected:
-      - type: PERSON
-        text: "Jean Dupont"
-      - type: EMAIL_ADDRESS
-        text: "jean.dupont@example.fr"
-      - type: PHONE_NUMBER
-        text: "06 98 76 54 32"
-
-  # False Positive
-  - id: fr_fp
-    text: "Le temps est magnifique aujourd'hui"
-    language: fr
-    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/global.yaml b/benchmarks/pii-accuracy/test-data/global.yaml
deleted file mode 100644
index fa8e8a4..0000000
--- a/benchmarks/pii-accuracy/test-data/global.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-test_cases:
-  # Pattern-based recognizers - language independent
-  # Tested once since regex/checksum works the same for all languages
-
-  # Email
-  - id: global_email
-    text: "Contact me at john.doe@company.com"
-    language: en
-    expected:
-      - type: EMAIL_ADDRESS
-        text: "john.doe@company.com"
-
-  # IBAN
-  - id: global_iban
-    text: "Transfer to IBAN DE89370400440532013000"
-    language: en
-    expected:
-      - type: IBAN_CODE
-        text: "DE89370400440532013000"
-
-  # Credit Card
-  - id: global_credit_card
-    text: "Card number: 4111 1111 1111 1111"
-    language: en
-    expected:
-      - type: CREDIT_CARD
-        text: "4111 1111 1111 1111"
-
-  # IP Address
-  - id: global_ip
-    text: "Server IP is 8.8.8.8"
-    language: en
-    expected:
-      - type: IP_ADDRESS
-        text: "8.8.8.8"
-
-  # Empty text
-  - id: global_empty
-    text: ""
-    language: en
-    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/hard.yaml b/benchmarks/pii-accuracy/test-data/hard.yaml
new file mode 100644
index 0000000..0545ce3
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/hard.yaml
@@ -0,0 +1,136 @@
+cases:
+  - id: hard_ipv6_full
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "The IPv6 source address was 2001:0db8:85a3:0000:0000:8a2e:0370:7334."
+    expected:
+      - entity: IP_ADDRESS
+        text: 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+        match: exact
+
+  - id: hard_ipv6_compressed
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "The compressed IPv6 address is 2001:db8::8a2e:370:7334."
+    expected:
+      - entity: IP_ADDRESS
+        text: 2001:db8::8a2e:370:7334
+        match: exact
+
+  - id: hard_email_inside_sentence
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "Please email alice.smith+billing@example.co.uk, not the shared inbox."
+    expected:
+      - entity: EMAIL_ADDRESS
+        text: alice.smith+billing@example.co.uk
+        match: exact
+
+  - id: hard_name_lowercase
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "The note says john smith can approve only read-only access."
+    expected:
+      - entity: PERSON
+        text: john smith
+        match: contains
+
+  - id: hard_name_lowercase_it
+    suite: hard
+    category: hard
+    split: test
+    language: it
+    text: "ricorda di chiamare mario rossi per la fattura"
+    expected:
+      - entity: PERSON
+        text: mario rossi
+        match: contains
+
+  - id: hard_person_hyphenated
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "Anne-Marie Johnson approved the vendor exception."
+    expected:
+      - entity: PERSON
+        text: Anne-Marie Johnson
+        match: contains
+
+  - id: hard_phone_national_uk
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "The London callback number is 020 7946 0958."
+    expected:
+      - entity: PHONE_NUMBER
+        text: 020 7946 0958
+        match: contains
+
+  - id: hard_location_ambiguous_city
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    text: "The request mentions Reading as the office location."
+    expected:
+      - entity: LOCATION
+        text: Reading
+        match: contains
+
+  - id: hard_address_street_de
+    suite: hard
+    category: hard
+    split: test
+    language: de
+    entities: [LOCATION]
+    text: "Die Lieferadresse lautet Bahnhofstraße 12."
+    expected:
+      - entity: LOCATION
+        text: Bahnhofstraße 12
+        match: contains
+
+  - id: hard_address_full_de
+    suite: hard
+    category: hard
+    split: test
+    language: de
+    entities: [LOCATION]
+    text: "Die Lieferadresse lautet Bahnhofstraße 12, 10115 Berlin."
+    expected:
+      - entity: LOCATION
+        text: Bahnhofstraße 12, 10115 Berlin
+        match: contains
+
+  - id: hard_address_full_en
+    suite: hard
+    category: hard
+    split: test
+    language: en
+    entities: [LOCATION]
+    text: "Ship the order to 1600 Pennsylvania Avenue, Washington."
+    expected:
+      - entity: LOCATION
+        text: 1600 Pennsylvania Avenue, Washington
+        match: contains
+
+  - id: hard_address_full_fr
+    suite: hard
+    category: hard
+    split: test
+    language: fr
+    entities: [LOCATION]
+    text: "L'adresse de livraison est 25 Rue du Faubourg Saint-Honoré, Paris."
+    expected:
+      - entity: LOCATION
+        text: 25 Rue du Faubourg Saint-Honoré, Paris
+        match: contains
diff --git a/benchmarks/pii-accuracy/test-data/it.yaml b/benchmarks/pii-accuracy/test-data/it.yaml
deleted file mode 100644
index cc6a37f..0000000
--- a/benchmarks/pii-accuracy/test-data/it.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-test_cases:
-  # Phone (needs Italian context words)
-  - id: it_phone
-    text: "Telefono: 333 1234567"
-    language: it
-    expected:
-      - type: PHONE_NUMBER
-        text: "333 1234567"
-
-  # Person (NER model)
-  - id: it_person
-    text: "Riunione con Giuseppe Verdi domani alle 10"
-    language: it
-    expected:
-      - type: PERSON
-        text: "Giuseppe Verdi"
-
-  # Location (NER model)
-  - id: it_location
-    text: "L'evento si terrà a Milano il prossimo mese"
-    language: it
-    expected:
-      - type: LOCATION
-        text: "Milano"
-
-  # Mixed (real-world prompt)
-  - id: it_mixed
-    text: "Contatto di Marco Rossi: marco.rossi@example.it, telefono 333 9876543"
-    language: it
-    expected:
-      - type: PERSON
-        text: "Marco Rossi"
-      - type: EMAIL_ADDRESS
-        text: "marco.rossi@example.it"
-      - type: PHONE_NUMBER
-        text: "333 9876543"
-
-  # False Positive
-  - id: it_fp
-    text: "Il caffè italiano è il migliore del mondo"
-    language: it
-    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/multilingual-paragraphs.yaml b/benchmarks/pii-accuracy/test-data/multilingual-paragraphs.yaml
new file mode 100644
index 0000000..a3b9c9d
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/multilingual-paragraphs.yaml
@@ -0,0 +1,261 @@
+cases:
+  - id: paragraph_en_customer_handoff
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: en
+    text: "The support handoff says Oliver Bennett is based in Manchester and should be contacted at oliver.bennett@example.com. The ticket also lists +44 161 496 0123, card 4242 4242 4242 4242, settlement IBAN GB82WEST12345698765432, and the last portal login from 203.0.113.77."
+    expected:
+      - entity: PERSON
+        text: Oliver Bennett
+        match: contains
+      - entity: LOCATION
+        text: Manchester
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: oliver.bennett@example.com
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +44 161 496 0123
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4242 4242 4242 4242
+        match: exact
+      - entity: IBAN_CODE
+        text: GB82WEST12345698765432
+        match: exact
+      - entity: IP_ADDRESS
+        text: 203.0.113.77
+        match: exact
+
+  - id: paragraph_de_vendor_review
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: de
+    text: "Im Lieferantenprotokoll steht, dass Nora Weber in München arbeitet und nora.weber@example.de nutzt. Fuer den Rueckruf ist +49 89 1234 5678 hinterlegt, die Karte lautet 4111 1111 1111 1111, das Konto DE89370400440532013000 und der Zugriff kam von 198.51.100.21."
+    expected:
+      - entity: PERSON
+        text: Nora Weber
+        match: contains
+      - entity: LOCATION
+        text: München
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: nora.weber@example.de
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +49 89 1234 5678
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+      - entity: IBAN_CODE
+        text: DE89370400440532013000
+        match: exact
+      - entity: IP_ADDRESS
+        text: 198.51.100.21
+        match: exact
+
+  - id: paragraph_es_refund_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: es
+    text: "La solicitud de reembolso indica que Mateo Ruiz vive en Sevilla y responde en mateo.ruiz@example.es. En la nota aparecen el telefono +34 95 123 45 67, la tarjeta 4012 8888 8888 1881, el IBAN ES9121000418450200051332 y la IP 192.0.2.34 usada en el portal."
+    expected:
+      - entity: PERSON
+        text: Mateo Ruiz
+        match: contains
+      - entity: LOCATION
+        text: Sevilla
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: mateo.ruiz@example.es
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +34 95 123 45 67
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4012 8888 8888 1881
+        match: exact
+      - entity: IBAN_CODE
+        text: ES9121000418450200051332
+        match: exact
+      - entity: IP_ADDRESS
+        text: 192.0.2.34
+        match: exact
+
+  - id: paragraph_fr_payment_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: fr
+    text: "Le dossier client mentionne Sophie Lambert, rattachee au bureau de Marseille, avec l'adresse sophie.lambert@example.fr. Le suivi contient aussi le telephone +33 4 91 12 34 56, la carte 5555 5555 5555 4444, l'IBAN FR1420041010050500013M02606 et l'adresse IP 10.44.12.8."
+    expected:
+      - entity: PERSON
+        text: Sophie Lambert
+        match: contains
+      - entity: LOCATION
+        text: Marseille
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: sophie.lambert@example.fr
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +33 4 91 12 34 56
+        match: contains
+      - entity: CREDIT_CARD
+        text: 5555 5555 5555 4444
+        match: exact
+      - entity: IBAN_CODE
+        text: FR1420041010050500013M02606
+        match: exact
+      - entity: IP_ADDRESS
+        text: 10.44.12.8
+        match: exact
+
+  - id: paragraph_it_booking_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: it
+    text: "Nel riepilogo prenotazione, Chiara Ferri risulta a Bologna e usa chiara.ferri@example.it. Il record include il telefono +39 051 123 4567, la carta 4242 4242 4242 4242, l'IBAN IT60X0542811101000000123456 e l'IP 172.20.4.9."
+    expected:
+      - entity: PERSON
+        text: Chiara Ferri
+        match: contains
+      - entity: LOCATION
+        text: Bologna
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: chiara.ferri@example.it
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +39 051 123 4567
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4242 4242 4242 4242
+        match: exact
+      - entity: IBAN_CODE
+        text: IT60X0542811101000000123456
+        match: exact
+      - entity: IP_ADDRESS
+        text: 172.20.4.9
+        match: exact
+
+  - id: paragraph_nl_account_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: nl
+    text: "Het accountoverzicht noemt Eva Bakker in Haarlem met e-mailadres eva.bakker@example.nl. Voor verificatie staan telefoon +31 23 123 4567, kaart 4111 1111 1111 1111, IBAN NL91ABNA0417164300 en IP-adres 8.8.8.8 in hetzelfde bericht."
+    expected:
+      - entity: PERSON
+        text: Eva Bakker
+        match: contains
+      - entity: LOCATION
+        text: Haarlem
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: eva.bakker@example.nl
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +31 23 123 4567
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+      - entity: IBAN_CODE
+        text: NL91ABNA0417164300
+        match: exact
+      - entity: IP_ADDRESS
+        text: 8.8.8.8
+        match: exact
+
+  - id: paragraph_pl_support_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: pl
+    text: "W zgłoszeniu serwisowym Marta Zielinska podaje Kraków jako lokalizację i adres marta.zielinska@example.pl. W dalszej części są telefon +48 12 123 45 67, karta 5555 5555 5555 4444, IBAN PL61109010140000071219812874 oraz IP 1.0.0.1."
+    expected:
+      - entity: PERSON
+        text: Marta Zielinska
+        match: contains
+      - entity: LOCATION
+        text: Kraków
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: marta.zielinska@example.pl
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +48 12 123 45 67
+        match: contains
+      - entity: CREDIT_CARD
+        text: 5555 5555 5555 4444
+        match: exact
+      - entity: IBAN_CODE
+        text: PL61109010140000071219812874
+        match: exact
+      - entity: IP_ADDRESS
+        text: 1.0.0.1
+        match: exact
+
+  - id: paragraph_pt_invoice_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: pt
+    text: "A fatura pendente pertence a Bruno Costa no Porto, com contacto bruno.costa@example.pt. O mesmo paragrafo inclui telefone +351 22 123 4567, cartao 4242 4242 4242 4242, IBAN PT50000201231234567890154 e IP 45.33.32.156."
+    expected:
+      - entity: PERSON
+        text: Bruno Costa
+        match: contains
+      - entity: LOCATION
+        text: Porto
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: bruno.costa@example.pt
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +351 22 123 4567
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4242 4242 4242 4242
+        match: exact
+      - entity: IBAN_CODE
+        text: PT50000201231234567890154
+        match: exact
+      - entity: IP_ADDRESS
+        text: 45.33.32.156
+        match: exact
+
+  - id: paragraph_ro_contract_case
+    suite: multilingual-paragraphs
+    category: eval
+    split: test
+    language: ro
+    text: "Contractul intern il mentioneaza pe Mihai Stan din Cluj-Napoca si adresa mihai.stan@example.ro. Pentru validare apar telefonul +40 264 123 456, cardul 4012 8888 8888 1881, IBAN RO49AAAA1B31007593840000 si IP 185.199.109.153."
+    expected:
+      - entity: PERSON
+        text: Mihai Stan
+        match: contains
+      - entity: LOCATION
+        text: Cluj-Napoca
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: mihai.stan@example.ro
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +40 264 123 456
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4012 8888 8888 1881
+        match: exact
+      - entity: IBAN_CODE
+        text: RO49AAAA1B31007593840000
+        match: exact
+      - entity: IP_ADDRESS
+        text: 185.199.109.153
+        match: exact
diff --git a/benchmarks/pii-accuracy/test-data/multilingual-sentences.yaml b/benchmarks/pii-accuracy/test-data/multilingual-sentences.yaml
new file mode 100644
index 0000000..11d85da
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/multilingual-sentences.yaml
@@ -0,0 +1,261 @@
+cases:
+  - id: sentence_en_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: en
+    text: "Emma Carter in London uses emma.carter@example.com, phone +44 20 7946 0958, card 4111 1111 1111 1111, IBAN GB82WEST12345698765432, and IP 203.0.113.42."
+    expected:
+      - entity: PERSON
+        text: Emma Carter
+        match: contains
+      - entity: LOCATION
+        text: London
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: emma.carter@example.com
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +44 20 7946 0958
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+      - entity: IBAN_CODE
+        text: GB82WEST12345698765432
+        match: exact
+      - entity: IP_ADDRESS
+        text: 203.0.113.42
+        match: exact
+
+  - id: sentence_de_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: de
+    text: "Lena Fischer in Berlin nutzt lena.fischer@example.de, Telefon +49 30 12345678, Karte 4012 8888 8888 1881, IBAN DE89370400440532013000 und IP 198.51.100.17."
+    expected:
+      - entity: PERSON
+        text: Lena Fischer
+        match: contains
+      - entity: LOCATION
+        text: Berlin
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: lena.fischer@example.de
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +49 30 12345678
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4012 8888 8888 1881
+        match: exact
+      - entity: IBAN_CODE
+        text: DE89370400440532013000
+        match: exact
+      - entity: IP_ADDRESS
+        text: 198.51.100.17
+        match: exact
+
+  - id: sentence_es_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: es
+    text: "Lucia Romero en Madrid usa lucia.romero@example.es, telefono +34 91 123 45 67, tarjeta 5555 5555 5555 4444, IBAN ES9121000418450200051332 e IP 192.0.2.88."
+    expected:
+      - entity: PERSON
+        text: Lucia Romero
+        match: contains
+      - entity: LOCATION
+        text: Madrid
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: lucia.romero@example.es
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +34 91 123 45 67
+        match: contains
+      - entity: CREDIT_CARD
+        text: 5555 5555 5555 4444
+        match: exact
+      - entity: IBAN_CODE
+        text: ES9121000418450200051332
+        match: exact
+      - entity: IP_ADDRESS
+        text: 192.0.2.88
+        match: exact
+
+  - id: sentence_fr_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: fr
+    text: "Camille Durand a Paris utilise camille.durand@example.fr, telephone +33 1 42 68 53 00, carte 4242 4242 4242 4242, IBAN FR1420041010050500013M02606 et IP 10.24.8.5."
+    expected:
+      - entity: PERSON
+        text: Camille Durand
+        match: contains
+      - entity: LOCATION
+        text: Paris
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: camille.durand@example.fr
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +33 1 42 68 53 00
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4242 4242 4242 4242
+        match: exact
+      - entity: IBAN_CODE
+        text: FR1420041010050500013M02606
+        match: exact
+      - entity: IP_ADDRESS
+        text: 10.24.8.5
+        match: exact
+
+  - id: sentence_it_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: it
+    text: "Marco Conti a Torino usa marco.conti@example.it, telefono +39 06 1234 5678, carta 4111 1111 1111 1111, IBAN IT60X0542811101000000123456 e IP 172.16.4.20."
+    expected:
+      - entity: PERSON
+        text: Marco Conti
+        match: contains
+      - entity: LOCATION
+        text: Torino
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: marco.conti@example.it
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +39 06 1234 5678
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+      - entity: IBAN_CODE
+        text: IT60X0542811101000000123456
+        match: exact
+      - entity: IP_ADDRESS
+        text: 172.16.4.20
+        match: exact
+
+  - id: sentence_nl_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: nl
+    text: "Daan Jansen in Utrecht gebruikt daan.jansen@example.nl, telefoon +31 20 123 4567, kaart 5555 5555 5555 4444, IBAN NL91ABNA0417164300 en IP 8.8.4.4."
+    expected:
+      - entity: PERSON
+        text: Daan Jansen
+        match: contains
+      - entity: LOCATION
+        text: Utrecht
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: daan.jansen@example.nl
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +31 20 123 4567
+        match: contains
+      - entity: CREDIT_CARD
+        text: 5555 5555 5555 4444
+        match: exact
+      - entity: IBAN_CODE
+        text: NL91ABNA0417164300
+        match: exact
+      - entity: IP_ADDRESS
+        text: 8.8.4.4
+        match: exact
+
+  - id: sentence_pl_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: pl
+    text: "Piotr Nowak w Warszawie używa piotr.nowak@example.pl, telefonu +48 22 123 45 67, karty 4111 1111 1111 1111, IBAN PL61109010140000071219812874 i IP 1.1.1.1."
+    expected:
+      - entity: PERSON
+        text: Piotr Nowak
+        match: contains
+      - entity: LOCATION
+        text: Warszawie
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: piotr.nowak@example.pl
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +48 22 123 45 67
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4111 1111 1111 1111
+        match: exact
+      - entity: IBAN_CODE
+        text: PL61109010140000071219812874
+        match: exact
+      - entity: IP_ADDRESS
+        text: 1.1.1.1
+        match: exact
+
+  - id: sentence_pt_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: pt
+    text: "Ana Pereira em Lisboa usa ana.pereira@example.pt, telefone +351 21 123 4567, cartao 4012 8888 8888 1881, IBAN PT50000201231234567890154 e IP 45.67.89.10."
+    expected:
+      - entity: PERSON
+        text: Ana Pereira
+        match: contains
+      - entity: LOCATION
+        text: Lisboa
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: ana.pereira@example.pt
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +351 21 123 4567
+        match: contains
+      - entity: CREDIT_CARD
+        text: 4012 8888 8888 1881
+        match: exact
+      - entity: IBAN_CODE
+        text: PT50000201231234567890154
+        match: exact
+      - entity: IP_ADDRESS
+        text: 45.67.89.10
+        match: exact
+
+  - id: sentence_ro_all_entities
+    suite: multilingual-sentences
+    category: eval
+    split: test
+    language: ro
+    text: "Elena Ionescu din Bucuresti foloseste elena.ionescu@example.ro, telefon +40 21 123 4567, card 5555 5555 5555 4444, IBAN RO49AAAA1B31007593840000 si IP 185.199.108.153."
+    expected:
+      - entity: PERSON
+        text: Elena Ionescu
+        match: contains
+      - entity: LOCATION
+        text: Bucuresti
+        match: contains
+      - entity: EMAIL_ADDRESS
+        text: elena.ionescu@example.ro
+        match: exact
+      - entity: PHONE_NUMBER
+        text: +40 21 123 4567
+        match: contains
+      - entity: CREDIT_CARD
+        text: 5555 5555 5555 4444
+        match: exact
+      - entity: IBAN_CODE
+        text: RO49AAAA1B31007593840000
+        match: exact
+      - entity: IP_ADDRESS
+        text: 185.199.108.153
+        match: exact
diff --git a/benchmarks/pii-accuracy/test-data/precision-paragraphs.yaml b/benchmarks/pii-accuracy/test-data/precision-paragraphs.yaml
new file mode 100644
index 0000000..69b0336
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/precision-paragraphs.yaml
@@ -0,0 +1,60 @@
+cases:
+  - id: precision_paragraph_release_numbers
+    suite: precision-paragraphs
+    category: precision
+    split: test
+    language: en
+    gate: false
+    entities: [CREDIT_CARD, PHONE_NUMBER, IP_ADDRESS]
+    text: "The release checklist mentions build 4111-1111, ticket 5552671, version 12.4.8, shard 10.24, and rollout window 20:30. None of these strings is a payment card, phone number, or IP address."
+    expected: []
+
+  - id: precision_paragraph_email_lookalikes
+    suite: precision-paragraphs
+    category: precision
+    split: test
+    language: en
+    gate: false
+    entities: [EMAIL_ADDRESS]
+    text: "The onboarding guide says to contact support [at] example [dot] com, use the placeholder user@@example..com only in screenshots, and never paste a real mailbox into templates."
+    expected: []
+
+  - id: precision_paragraph_invalid_accounts
+    suite: precision-paragraphs
+    category: precision
+    split: test
+    language: de
+    gate: false
+    entities: [IBAN_CODE, CREDIT_CARD]
+    text: "Der Testlauf enthaelt die Beispielwerte DE001234, DE89370400440532013001 und 4111 1111 1111 1112. Diese Werte sind absichtlich ungueltig und sollen nicht als echte Konten oder Karten gelten."
+    expected: []
+
+  - id: precision_paragraph_operational_locations
+    suite: precision-paragraphs
+    category: precision
+    split: test
+    language: en
+    gate: false
+    entities: [LOCATION, PERSON]
+    text: "The deployment moved from eu-west-1 to us-east-1, the owner field stayed empty, and the runbook only refers to customer, administrator, reviewer, and approver roles."
+    expected: []
+
+  - id: precision_paragraph_multilingual_roles
+    suite: precision-paragraphs
+    category: precision
+    split: test
+    language: es
+    gate: false
+    entities: [PERSON, LOCATION]
+    text: "El cliente solicita una factura, el administrador revisa la cola y el equipo confirma que no hay nombres de personas ni ciudades reales en este parrafo."
+    expected: []
+
+  - id: precision_paragraph_log_noise
+    suite: precision-paragraphs
+    category: precision
+    split: test
+    language: en
+    gate: false
+    entities: [EMAIL_ADDRESS, PHONE_NUMBER, IP_ADDRESS, LOCATION, PERSON]
+    text: "Log sample: user=anonymous, host=local-dev, route=/api/v1/orders, status=204, latency=123ms, trace=abc123def456, region=staging-west, retry=0. The sample intentionally avoids direct personal data."
+    expected: []
diff --git a/benchmarks/pii-accuracy/test-data/precision.yaml b/benchmarks/pii-accuracy/test-data/precision.yaml
new file mode 100644
index 0000000..7a49f70
--- /dev/null
+++ b/benchmarks/pii-accuracy/test-data/precision.yaml
@@ -0,0 +1,236 @@
+cases:
+  - id: precision_obfuscated_email
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [EMAIL_ADDRESS]
+    text: "Reach support at support [at] example [dot] com."
+    expected: []
+
+  - id: precision_malformed_email
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [EMAIL_ADDRESS]
+    text: "The placeholder alice@@example..com is not a valid mailbox."
+    expected: []
+
+  - id: precision_invalid_credit_card
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [CREDIT_CARD]
+    text: "The internal test number 4111 1111 1111 1112 is invalid."
+    expected: []
+
+  - id: precision_short_iban
+    suite: precision
+    category: precision
+    split: test
+    language: de
+    entities: [IBAN_CODE]
+    text: "Die Zeichenfolge DE001234 ist keine IBAN."
+    expected: []
+
+  - id: precision_invalid_iban_checksum
+    suite: precision
+    category: precision
+    split: test
+    language: de
+    entities: [IBAN_CODE]
+    text: "Die Beispielnummer DE89370400440532013001 hat keine gueltige Pruefsumme."
+    expected: []
+
+  - id: precision_invalid_iban_it_checksum
+    suite: precision
+    category: precision
+    split: test
+    language: it
+    entities: [IBAN_CODE]
+    text: "IBAN con cifra di controllo errata: IT60X0542811101000000123457."
+    expected: []
+
+  - id: precision_version_not_phone
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [PHONE_NUMBER]
+    text: "Release 12.4.8 shipped before ticket 5552671 was closed."
+    expected: []
+
+  - id: precision_order_id_not_phone
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [PHONE_NUMBER]
+    text: "Order 5552671 was closed after release 12.4.8."
+    expected: []
+
+  - id: precision_invalid_ip
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [IP_ADDRESS]
+    text: "The release number 999.12.3.456 is not an IP address."
+    expected: []
+
+  - id: precision_common_words_not_person
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [PERSON]
+    text: "The administrator role approved only read access."
+    expected: []
+
+  - id: precision_fr_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: fr
+    entities: [PERSON, LOCATION]
+    text: "Le client souhaite un rendez-vous la semaine prochaine."
+    expected: []
+
+  - id: precision_es_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: es
+    entities: [PERSON, LOCATION]
+    text: "El cliente solicita la factura del mes corriente."
+    expected: []
+
+  - id: precision_it_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: it
+    entities: [PERSON, LOCATION]
+    text: "Il cliente ha richiesto la fattura entro venerdi."
+    expected: []
+
+  - id: precision_nl_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: nl
+    entities: [PERSON, LOCATION]
+    text: "De klant vraagt om een snelle levering deze week."
+    expected: []
+
+  - id: precision_pt_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: pt
+    entities: [PERSON, LOCATION]
+    text: "O cliente pediu a fatura ate sexta-feira."
+    expected: []
+
+  - id: precision_cloud_region_not_location
+    suite: precision
+    category: precision
+    split: test
+    language: en
+    entities: [LOCATION]
+    text: "The workload moved from eu-west-1 to us-east-1 during the test."
+    expected: []
+
+  - id: precision_pl_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: pl
+    entities: [PERSON, LOCATION]
+    text: "Klient prosi o fakturę przed końcem tygodnia."
+    expected: []
+
+  - id: precision_pl_invalid_credit_card
+    suite: precision
+    category: precision
+    split: test
+    language: pl
+    entities: [CREDIT_CARD]
+    text: "Numer testowy 4111 1111 1111 1112 jest nieprawidłowy."
+    expected: []
+
+  - id: precision_pl_invalid_iban_checksum
+    suite: precision
+    category: precision
+    split: test
+    language: pl
+    entities: [IBAN_CODE]
+    text: "Ciąg PL00109010140000071219812874 ma błędną sumę kontrolną."
+    expected: []
+
+  - id: precision_pl_version_not_phone
+    suite: precision
+    category: precision
+    split: test
+    language: pl
+    entities: [PHONE_NUMBER]
+    text: "Wersja 12.4.8 została wydana przed zgłoszeniem 5552671."
+    expected: []
+
+  - id: precision_pl_cloud_region_not_location
+    suite: precision
+    category: precision
+    split: test
+    language: pl
+    entities: [LOCATION]
+    text: "Zadanie przeniesiono z eu-west-1 do us-east-1 podczas testu."
+    expected: []
+
+  - id: precision_ro_roles_not_person_location
+    suite: precision
+    category: precision
+    split: test
+    language: ro
+    entities: [PERSON, LOCATION]
+    gate: false
+    note: "Instance of a broader Presidio over-detection: non-name tokens (role nouns, emails, IBANs) are labelled PERSON/LOCATION at ~0.85 across several languages, e.g. 'Clientul' here. Tracked as a known false positive; report-only until recognizer precision improves."
+    text: "Clientul cere factura până la sfârșitul săptămânii."
+    expected: []
+
+  - id: precision_ro_invalid_credit_card
+    suite: precision
+    category: precision
+    split: test
+    language: ro
+    entities: [CREDIT_CARD]
+    text: "Numărul de test 4111 1111 1111 1112 este invalid."
+    expected: []
+
+  - id: precision_ro_invalid_iban_checksum
+    suite: precision
+    category: precision
+    split: test
+    language: ro
+    entities: [IBAN_CODE]
+    text: "Șirul RO00AAAA1B31007593840000 are o sumă de control greșită."
+    expected: []
+
+  - id: precision_ro_version_not_phone
+    suite: precision
+    category: precision
+    split: test
+    language: ro
+    entities: [PHONE_NUMBER]
+    text: "Versiunea 12.4.8 a fost lansată înainte de tichetul 5552671."
+    expected: []
+
+  - id: precision_ro_cloud_region_not_location
+    suite: precision
+    category: precision
+    split: test
+    language: ro
+    entities: [LOCATION]
+    text: "Sarcina a fost mutată din eu-west-1 în us-east-1 în timpul testului."
+    expected: []
diff --git a/benchmarks/pii-accuracy/types.ts b/benchmarks/pii-accuracy/types.ts
index a78bc5a..e351e1c 100644
--- a/benchmarks/pii-accuracy/types.ts
+++ b/benchmarks/pii-accuracy/types.ts
@@ -1,53 +1,68 @@
 import { z } from "zod";
+import { CONFIGURED_PII_ENTITIES, SUPPORTED_LANGUAGES } from "./taxonomy";
 
-// Schema for expected PII entity in test data
-export const ExpectedEntitySchema = z.object({
-  type: z.string(),
-  text: z.string(),
-});
-
-// Schema for a single test case
-export const TestCaseSchema = z.object({
-  id: z.string(),
-  text: z.string(),
-  language: z.string(),
-  expected: z.array(ExpectedEntitySchema),
-  description: z.string().optional(),
-});
-
-export type ExpectedEntity = z.infer<typeof ExpectedEntitySchema>;
-export type TestCase = z.infer<typeof TestCaseSchema>;
-
-// Result of running a single test case
-export interface TestResult {
-  id: string;
-  text: string;
-  language: string;
-  passed: boolean;
-  expected: ExpectedEntity[];
-  detected: DetectedEntity[];
-  falseNegatives: ExpectedEntity[]; // Expected but not detected
-  falsePositives: DetectedEntity[]; // Detected but not expected
-  truePositives: ExpectedEntity[]; // Correctly detected
-}
-
-export interface DetectedEntity {
-  type: string;
+export const ConfiguredPiiEntitySchema = z.enum(CONFIGURED_PII_ENTITIES);
+export type ConfiguredPiiEntity = z.infer<typeof ConfiguredPiiEntitySchema>;
+
+export const SupportedLanguageSchema = z.enum(SUPPORTED_LANGUAGES);
+export type SupportedLanguage = z.infer<typeof SupportedLanguageSchema>;
+
+export const MatchModeSchema = z.enum(["exact", "contains", "overlap"]);
+export type MatchMode = z.infer<typeof MatchModeSchema>;
+
+export const CategorySchema = z.enum(["core", "precision", "eval", "hard"]);
+export type BenchmarkCategory = z.infer<typeof CategorySchema>;
+
+export const SplitSchema = z.enum(["dev", "test"]);
+export type BenchmarkSplit = z.infer<typeof SplitSchema>;
+
+export const ExpectedSpanSchema = z
+  .object({
+    entity: ConfiguredPiiEntitySchema,
+    text: z.string().min(1),
+    match: MatchModeSchema.default("contains"),
+    aliases: z.array(z.string().min(1)).default([]),
+  })
+  .strict();
+export type ExpectedSpan = z.infer<typeof ExpectedSpanSchema>;
+
+export const BenchmarkCaseSchema = z
+  .object({
+    id: z.string().min(1),
+    suite: z.string().min(1),
+    category: CategorySchema.default("core"),
+    split: SplitSchema.default("test"),
+    language: SupportedLanguageSchema,
+    text: z.string().min(1),
+    entities: z.array(ConfiguredPiiEntitySchema).optional(),
+    gate: z.boolean().optional(),
+    note: z.string().optional(),
+    expected: z.array(ExpectedSpanSchema).default([]),
+  })
+  .strict();
+export type BenchmarkCase = z.infer<typeof BenchmarkCaseSchema>;
+
+export const BenchmarkFileSchema = z
+  .object({
+    cases: z.array(BenchmarkCaseSchema),
+  })
+  .strict();
+
+export type Detection = {
+  entity: string;
   text: string;
   start: number;
   end: number;
   score: number;
-}
-
-// Aggregated metrics
-export interface AccuracyMetrics {
-  total: number;
-  passed: number;
-  failed: number;
-  precision: number; // TP / (TP + FP)
-  recall: number; // TP / (TP + FN)
-  f1: number; // 2 * (P * R) / (P + R)
-  truePositives: number;
-  falsePositives: number;
-  falseNegatives: number;
-}
+};
+
+export type TestResult = {
+  case: BenchmarkCase;
+  passed: boolean;
+  gating: boolean;
+  detections: Detection[];
+  matched: Array<{ expected: ExpectedSpan; detection: Detection }>;
+  missing: ExpectedSpan[];
+  unexpected: Detection[];
+  error?: string;
+};
diff --git a/biome.json b/biome.json
index 530e667..b5bcb45 100644
--- a/biome.json
+++ b/biome.json
@@ -7,7 +7,7 @@
   },
   "files": {
     "ignoreUnknown": false,
-    "includes": ["src/**/*.ts"]
+    "includes": ["src/**/*.ts", "benchmarks/pii-accuracy/*.ts"]
   },
   "formatter": {
     "enabled": true,
diff --git a/package.json b/package.json
index 81a1310..25e0e9c 100644
--- a/package.json
+++ b/package.json
@@ -10,9 +10,10 @@
     "build": "bun build src/index.ts --outdir dist --target bun --external lightningcss",
     "test": "bun test",
     "typecheck": "tsc --noEmit",
-    "lint": "biome lint src",
-    "format": "biome format src --write",
-    "check": "biome check src",
+    "typecheck:benchmarks": "tsc --noEmit --project tsconfig.benchmarks.json",
+    "lint": "biome lint src benchmarks/pii-accuracy/*.ts",
+    "format": "biome format src benchmarks/pii-accuracy/*.ts --write",
+    "check": "biome check src benchmarks/pii-accuracy/*.ts",
     "benchmark:accuracy": "bun run benchmarks/pii-accuracy/run.ts"
   },
   "dependencies": {
diff --git a/tsconfig.benchmarks.json b/tsconfig.benchmarks.json
new file mode 100644
index 0000000..e0604d5
--- /dev/null
+++ b/tsconfig.benchmarks.json
@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "."
+  },
+  "include": ["benchmarks/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}