Adds a corpus-driven PII accuracy benchmark against the Presidio /analyze endpoint, plus pl/ro precision negatives, non-western PERSON coverage, and address-like LOCATION cases.
- name: Type check
run: bun run typecheck
+ - name: Benchmark type check
+ run: bun run typecheck:benchmarks
+
- name: Lint & format check
run: bun run check
- uses: actions/checkout@v4
- name: Test Docker build
- run: docker build -f docker/Dockerfile -t pasteguard:test .
\ No newline at end of file
+ run: docker build -f docker/Dockerfile -t pasteguard:test .
--- /dev/null
+# PII accuracy benchmark
+
+This benchmark targets the analyzer `/analyze` endpoint on `http://localhost:3000/analyze` by
+default. The request/response contract is the analyzer contract, so the same corpus can also be
+pointed at another compatible analyzer endpoint with `--url`.
+
+The corpus only includes entities exposed by Pasteguard's default PII configuration:
+`PERSON`, `EMAIL_ADDRESS`, `PHONE_NUMBER`, `CREDIT_CARD`, `IBAN_CODE`, `IP_ADDRESS`, and
+`LOCATION`. Cases are not skipped dynamically; if the active runtime cannot analyze a configured
+language or entity, the benchmark fails.
+
+The corpus covers the European image language set: `en`, `de`, `es`, `fr`, `it`, `nl`, `pl`,
+`pt`, and `ro`.
+
+The runner validates the corpus before applying filters. Unknown YAML fields, unsupported
+entities, unsupported languages, unknown suites, duplicate case IDs, and expected strings that do
+not occur in the case text fail the run.
+
+## Design
+
+Cases are selected from the product behavior Pasteguard should provide, not from what the current
+detector already happens to pass.
+
+- `core` covers the minimum detection promise for configured entities. These are gating tests.
+- `precision` covers the minimum false-positive promise for configured entities. These are gating
+ tests.
+- `eval` contains realistic multilingual workflow cases for quality tracking.
+- `hard` contains difficult, ambiguous, or aspirational cases for configured entities.
+
+Additional suites make the benchmark easier to read by intent:
+
+- `multilingual-sentences` checks every configured entity in sentence form across all supported
+ languages.
+- `multilingual-paragraphs` checks every configured entity inside realistic multi-sentence
+ workflow text across all supported languages.
+- `boundaries` checks whether spans stop cleanly around punctuation, brackets, and quotes.
+- `precision-paragraphs` checks longer negative controls with operational lookalike strings.
+
+`core` and `precision` cases are gating by default. `eval` and `hard` cases are report-only by
+default. Individual cases can override this with `gate`. Analyzer errors, HTTP errors, and invalid
+responses always fail the benchmark run.
+
+Match modes:
+
+- `exact` requires the normalized detected text to equal the expected text.
+- `contains` requires the detected span to fully cover the expected text with at most two extra
+ characters on either side.
+- `overlap` is reserved for deliberately loose edge cases where any span overlap is meaningful.
+
+## Sources
+
+- GDPR Article 4 definition of personal data:
+ https://eur-lex.europa.eu/eli/reg/2016/679/oj
+
+## Run
+
+```bash
+bun run benchmark:accuracy
+```
+
+Useful filters:
+
+```bash
+bun run benchmark:accuracy --suite core,precision
+bun run benchmark:accuracy --category core,precision
+bun run benchmark:accuracy --languages en,de,it,pl,ro
+bun run benchmark:accuracy --url http://localhost:3000/analyze --verbose
+```
-#!/usr/bin/env bun
-/**
- * PII Detection Accuracy Benchmark
- *
- * Measures precision, recall, and F1 score of the PII detection system.
- *
- * Usage:
- * bun run benchmarks/pii-accuracy/run.ts
- * bun run benchmarks/pii-accuracy/run.ts --threshold 0.5
- * bun run benchmarks/pii-accuracy/run.ts --languages de,en
- * bun run benchmarks/pii-accuracy/run.ts --verbose
- */
-
-import { parseArgs } from "util";
-import { Glob } from "bun";
+import { readdir, readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
import { parse as parseYaml } from "yaml";
+import { isKnownSuite, SUITES, suiteNames } from "./taxonomy";
import {
- TestCaseSchema,
- type AccuracyMetrics,
- type DetectedEntity,
- type ExpectedEntity,
- type TestCase,
+ type BenchmarkCase,
+ BenchmarkFileSchema,
+ type Detection,
+ type ExpectedSpan,
type TestResult,
} from "./types";
-// Configuration
+const DEFAULT_ANALYZE_URL = "http://localhost:3000/analyze";
const DEFAULT_THRESHOLD = 0.7;
-const PRESIDIO_URL = process.env.PRESIDIO_URL || "http://localhost:5002";
-const TEST_DATA_DIR = import.meta.dir + "/test-data";
-
-// Parse command line arguments
-const { values: args } = parseArgs({
- args: Bun.argv.slice(2),
+const MAX_FAILURES_TO_PRINT = 40;
+const MAX_CONTAINS_EDGE_CHARS = 2;
+
+type Filters = {
+ suites?: Set<string>;
+ categories?: Set<string>;
+ languages?: Set<string>;
+ split?: Set<string>;
+};
+
+type Metrics = {
+ cases: number;
+ passed: number;
+ errors: number;
+ tp: number;
+ fp: number;
+ fn: number;
+};
+
+const argv = parseArgs({
options: {
- threshold: { type: "string", short: "t" },
- languages: { type: "string", short: "l" },
- verbose: { type: "boolean", short: "v", default: false },
- help: { type: "boolean", short: "h", default: false },
+ url: { type: "string" },
+ threshold: { type: "string" },
+ suite: { type: "string" },
+ category: { type: "string" },
+ languages: { type: "string" },
+ split: { type: "string" },
+ verbose: { type: "boolean", default: false },
+ "list-suites": { type: "boolean", default: false },
+ help: { type: "boolean", default: false },
},
+ allowPositionals: false,
});
-if (args.help) {
- console.log(`
-PII Detection Accuracy Benchmark
-
-Usage:
- bun run benchmarks/pii-accuracy/run.ts [options]
+if (argv.values.help) {
+ printHelp();
+ process.exit(0);
+}
-Options:
- -t, --threshold <value> Score threshold (default: ${DEFAULT_THRESHOLD})
- -l, --languages <langs> Comma-separated languages to test (e.g., de,en)
- -v, --verbose Show detailed results for each test case
- -h, --help Show this help message
-
-Examples:
- bun run benchmarks/pii-accuracy/run.ts
- bun run benchmarks/pii-accuracy/run.ts --threshold 0.5
- bun run benchmarks/pii-accuracy/run.ts --languages de,en
- bun run benchmarks/pii-accuracy/run.ts --verbose
-`);
+if (argv.values["list-suites"]) {
+ for (const name of suiteNames()) {
+ console.log(`${name}: ${SUITES[name].description}`);
+ }
process.exit(0);
}
-const threshold = args.threshold ? Number.parseFloat(args.threshold) : DEFAULT_THRESHOLD;
-const verbose = args.verbose ?? false;
-const languageFilter = args.languages?.split(",").map((l) => l.trim().toLowerCase());
-
-// Entity types we test for
-const ENTITY_TYPES = [
- "PERSON",
- "EMAIL_ADDRESS",
- "PHONE_NUMBER",
- "CREDIT_CARD",
- "IBAN_CODE",
- "IP_ADDRESS",
- "LOCATION",
-];
-
-/**
- * Load test cases from YAML files in test-data directory
- */
-async function loadTestCases(): Promise<TestCase[]> {
- const testCases: TestCase[] = [];
- const glob = new Glob("*.yaml");
-
- for await (const file of glob.scan(TEST_DATA_DIR)) {
- const filePath = `${TEST_DATA_DIR}/${file}`;
- const content = await Bun.file(filePath).text();
- const data = parseYaml(content) as { test_cases: unknown[] };
-
- if (!data.test_cases || !Array.isArray(data.test_cases)) {
- console.warn(`Warning: ${file} has no test_cases array`);
- continue;
+const analyzeUrl = argv.values.url ?? process.env.PII_BENCHMARK_ANALYZE_URL ?? DEFAULT_ANALYZE_URL;
+const threshold = parseThreshold(argv.values.threshold ?? String(DEFAULT_THRESHOLD));
+const filters = buildFilters(argv.values);
+const verbose = Boolean(argv.values.verbose);
+
+const benchmarkDir = dirname(fileURLToPath(import.meta.url));
+const testDataDir = join(benchmarkDir, "test-data");
+
+const allCases = await loadCases(testDataDir);
+validateCorpus(allCases);
+
+const cases = applyFilters(allCases, filters);
+
+if (cases.length === 0) {
+ console.error("No benchmark cases matched the selected filters.");
+ process.exit(1);
+}
+
+const results: TestResult[] = [];
+
+for (const testCase of cases) {
+ results.push(await runCase(testCase, analyzeUrl, threshold));
+}
+
+printReport(results, analyzeUrl, threshold, verbose);
+
+const gatingFailures = results.filter((result) => result.gating && !result.passed);
+const errors = results.filter((result) => result.error);
+
+if (gatingFailures.length > 0 || errors.length > 0) {
+ process.exitCode = 1;
+}
+
+async function loadCases(dir: string): Promise<BenchmarkCase[]> {
+ const files = (await readdir(dir)).filter((file) => file.endsWith(".yaml")).sort();
+ const casesFromFiles = await Promise.all(
+ files.map(async (file) => {
+ const raw = await readFile(join(dir, file), "utf8");
+ const parsed = BenchmarkFileSchema.safeParse(parseYaml(raw));
+
+ if (!parsed.success) {
+ const details = parsed.error.errors
+ .map((error) => `${error.path.join(".")}: ${error.message}`)
+ .join("; ");
+ throw new Error(`Invalid benchmark file ${file}: ${details}`);
+ }
+
+ return parsed.data.cases;
+ }),
+ );
+
+ return casesFromFiles.flat();
+}
+
+function validateCorpus(casesToValidate: BenchmarkCase[]) {
+ const errors: string[] = [];
+ const seenIds = new Set<string>();
+
+ for (const testCase of casesToValidate) {
+ if (seenIds.has(testCase.id)) {
+ errors.push(`Duplicate case id: ${testCase.id}`);
+ }
+ seenIds.add(testCase.id);
+
+ if (!isKnownSuite(testCase.suite)) {
+ errors.push(`Unknown suite in ${testCase.id}: ${testCase.suite}`);
}
- for (const testCase of data.test_cases) {
- const parsed = TestCaseSchema.safeParse(testCase);
- if (parsed.success) {
- // Filter by language if specified
- if (!languageFilter || languageFilter.includes(parsed.data.language)) {
- testCases.push(parsed.data);
- }
- } else {
- console.warn(`Warning: Invalid test case in ${file}:`, parsed.error.format());
+ for (const expected of testCase.expected) {
+ if (!testCase.text.includes(expected.text)) {
+ errors.push(
+ `Expected text is not present in ${testCase.id}: ${expected.entity}(${expected.text})`,
+ );
}
}
}
- // Sort by ID for deterministic output
- return testCases.sort((a, b) => a.id.localeCompare(b.id));
+ if (errors.length > 0) {
+ console.error(`Invalid benchmark corpus:\n${errors.map((error) => `- ${error}`).join("\n")}`);
+ process.exit(1);
+ }
+}
+
+function applyFilters(casesToFilter: BenchmarkCase[], activeFilters: Filters): BenchmarkCase[] {
+ return casesToFilter.filter((testCase) => {
+ if (activeFilters.suites && !activeFilters.suites.has(testCase.suite)) {
+ return false;
+ }
+ if (activeFilters.categories && !activeFilters.categories.has(testCase.category)) {
+ return false;
+ }
+ if (activeFilters.languages && !activeFilters.languages.has(testCase.language)) {
+ return false;
+ }
+ if (activeFilters.split && !activeFilters.split.has(testCase.split)) {
+ return false;
+ }
+ return true;
+ });
}
-/**
- * Call Presidio analyzer API
- */
-async function detectPII(text: string, language: string): Promise<DetectedEntity[]> {
- if (!text) return [];
+async function runCase(
+ testCase: BenchmarkCase,
+ endpoint: string,
+ globalThreshold: number,
+): Promise<TestResult> {
+ const gating = isGatingCase(testCase);
- const response = await fetch(`${PRESIDIO_URL}/analyze`, {
+ try {
+ const detections = await analyze(testCase, endpoint, globalThreshold);
+ const { matched, missing, unexpected } = scoreDetections(testCase, detections);
+
+ return {
+ case: testCase,
+ passed: missing.length === 0 && unexpected.length === 0,
+ gating,
+ detections,
+ matched,
+ missing,
+ unexpected,
+ };
+ } catch (error) {
+ return {
+ case: testCase,
+ passed: false,
+ gating,
+ detections: [],
+ matched: [],
+ missing: testCase.expected,
+ unexpected: [],
+ error: error instanceof Error ? error.message : String(error),
+ };
+ }
+}
+
+async function analyze(
+ testCase: BenchmarkCase,
+ endpoint: string,
+ globalThreshold: number,
+): Promise<Detection[]> {
+ const response = await fetch(endpoint, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
- text,
- language,
- entities: ENTITY_TYPES,
- score_threshold: threshold,
+ text: testCase.text,
+ language: testCase.language,
+ entities: entitiesForCase(testCase),
+ score_threshold: globalThreshold,
}),
});
if (!response.ok) {
- throw new Error(`Presidio error: ${response.status} ${await response.text()}`);
+ const body = await response.text();
+ throw new Error(`HTTP ${response.status}: ${body || response.statusText}`);
}
- const entities = (await response.json()) as Array<{
- entity_type: string;
- start: number;
- end: number;
- score: number;
- }>;
-
- return entities.map((e) => ({
- type: e.entity_type,
- text: text.slice(e.start, e.end),
- start: e.start,
- end: e.end,
- score: e.score,
- }));
-}
-
-/**
- * Check if Presidio is available
- */
-async function checkPresidio(): Promise<boolean> {
- try {
- const response = await fetch(`${PRESIDIO_URL}/health`);
- return response.ok;
- } catch {
- return false;
+ const payload = await response.json();
+
+ if (!Array.isArray(payload)) {
+ throw new Error(`Expected analyzer array response, got ${typeof payload}`);
}
+
+ return payload.map((item) => normalizeDetection(testCase, item));
}
-/**
- * Run a single test case and compare results
- */
-async function runTestCase(testCase: TestCase): Promise<TestResult> {
- const detected = await detectPII(testCase.text, testCase.language);
+function normalizeDetection(testCase: BenchmarkCase, item: unknown): Detection {
+ if (!item || typeof item !== "object") {
+ throw new Error("Analyzer returned a non-object detection");
+ }
+
+ const record = item as Record<string, unknown>;
+ const entity = record.entity_type ?? record.entity;
+ const start = record.start;
+ const end = record.end;
+ const score = record.score;
- // Match expected entities with detected ones
- const truePositives: ExpectedEntity[] = [];
- const falseNegatives: ExpectedEntity[] = [];
- const matchedDetected = new Set<number>();
+ if (typeof entity !== "string") {
+ throw new Error("Analyzer detection is missing entity_type");
+ }
+ if (typeof start !== "number" || typeof end !== "number") {
+ throw new Error(`Analyzer detection ${entity} is missing numeric offsets`);
+ }
+
+ return {
+ entity,
+ start,
+ end,
+ score: typeof score === "number" ? score : 0,
+ text: testCase.text.slice(start, end),
+ };
+}
+
+function scoreDetections(testCase: BenchmarkCase, detections: Detection[]) {
+ const unmatched = [...detections];
+ const matched: Array<{ expected: ExpectedSpan; detection: Detection }> = [];
+ const missing: ExpectedSpan[] = [];
for (const expected of testCase.expected) {
- // Find a matching detected entity
- const matchIndex = detected.findIndex(
- (d, i) =>
- !matchedDetected.has(i) &&
- d.type === expected.type &&
- normalizeText(d.text).includes(normalizeText(expected.text)),
+ const matchIndex = unmatched.findIndex((detection) =>
+ detectionMatchesExpected(testCase, detection, expected),
);
- if (matchIndex !== -1) {
- truePositives.push(expected);
- matchedDetected.add(matchIndex);
- } else {
- falseNegatives.push(expected);
+ if (matchIndex === -1) {
+ missing.push(expected);
+ continue;
}
+
+ const [detection] = unmatched.splice(matchIndex, 1);
+ matched.push({ expected, detection });
}
- // Remaining detected entities are false positives
- const falsePositives = detected.filter((_, i) => !matchedDetected.has(i));
+ return { matched, missing, unexpected: unmatched };
+}
- const passed = falseNegatives.length === 0 && falsePositives.length === 0;
+function detectionMatchesExpected(
+ testCase: BenchmarkCase,
+ detection: Detection,
+ expected: ExpectedSpan,
+): boolean {
+ if (!entityMatches(detection.entity, expected)) {
+ return false;
+ }
- return {
- id: testCase.id,
- text: testCase.text,
- language: testCase.language,
- passed,
- expected: testCase.expected,
- detected,
- falseNegatives,
- falsePositives,
- truePositives,
- };
+ if (expected.match === "exact") {
+ return normalized(detection.text) === normalized(expected.text);
+ }
+
+ if (expected.match === "overlap") {
+ return spansOverlap(testCase, detection, expected);
+ }
+
+ return boundedContainsMatch(testCase, detection, expected);
}
-/**
- * Normalize text for comparison (lowercase, trim)
- */
-function normalizeText(text: string): string {
- return text.toLowerCase().trim();
+function entityMatches(entity: string, expected: ExpectedSpan): boolean {
+ return entity === expected.entity || expected.aliases.includes(entity);
}
-/**
- * Calculate accuracy metrics from results
- */
-function calculateMetrics(results: TestResult[]): AccuracyMetrics {
- let tp = 0;
- let fp = 0;
- let fn = 0;
+function boundedContainsMatch(
+ testCase: BenchmarkCase,
+ detection: Detection,
+ expected: ExpectedSpan,
+): boolean {
+ const expectedStart = testCase.text.indexOf(expected.text);
- for (const result of results) {
- tp += result.truePositives.length;
- fp += result.falsePositives.length;
- fn += result.falseNegatives.length;
+ if (expectedStart === -1) {
+ return false;
}
- const precision = tp + fp > 0 ? tp / (tp + fp) : 1;
- const recall = tp + fn > 0 ? tp / (tp + fn) : 1;
- const f1 = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
+ const expectedEnd = expectedStart + expected.text.length;
+ const detectionCoversExpected = detection.start <= expectedStart && detection.end >= expectedEnd;
- return {
- total: results.length,
- passed: results.filter((r) => r.passed).length,
- failed: results.filter((r) => !r.passed).length,
- precision,
- recall,
- f1,
- truePositives: tp,
- falsePositives: fp,
- falseNegatives: fn,
- };
+ if (!detectionCoversExpected || !normalizedContains(detection.text, expected.text)) {
+ return false;
+ }
+
+ const leadingExtraChars = expectedStart - detection.start;
+ const trailingExtraChars = detection.end - expectedEnd;
+
+ return (
+ leadingExtraChars <= MAX_CONTAINS_EDGE_CHARS && trailingExtraChars <= MAX_CONTAINS_EDGE_CHARS
+ );
}
-/**
- * Format percentage for display
- */
-function formatPercent(value: number): string {
- return `${(value * 100).toFixed(1)}%`;
+function normalizedContains(detectedText: string, expectedText: string): boolean {
+ const detected = normalized(detectedText);
+ const expected = normalized(expectedText);
+
+ return detected.includes(expected);
}
-/**
- * Print metrics in a formatted way
- */
-function printMetrics(label: string, metrics: AccuracyMetrics): void {
- const status = metrics.failed === 0 ? "✓" : "⚠";
+function spansOverlap(
+ testCase: BenchmarkCase,
+ detection: Detection,
+ expected: ExpectedSpan,
+): boolean {
+ const expectedStart = testCase.text.indexOf(expected.text);
+
+ if (expectedStart === -1) {
+ return false;
+ }
+
+ const expectedEnd = expectedStart + expected.text.length;
+ return detection.start < expectedEnd && detection.end > expectedStart;
+}
+
+function normalized(value: string): string {
+ return value
+ .normalize("NFKC")
+ .toLowerCase()
+ .replace(/^[\s"'([<{]+/g, "")
+ .replace(/[\s"').,;:!?>\]}]+$/g, "")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function entitiesForCase(testCase: BenchmarkCase): string[] {
+ if (testCase.entities) {
+ return testCase.entities;
+ }
+
+ if (testCase.expected.length > 0) {
+ return [...new Set(testCase.expected.map((expected) => expected.entity))];
+ }
+
+ return [...(SUITES[testCase.suite]?.defaultEntities ?? [])];
+}
+
+function isGatingCase(testCase: BenchmarkCase): boolean {
+ return testCase.gate ?? (testCase.category === "core" || testCase.category === "precision");
+}
+
+function printReport(
+ results: TestResult[],
+ endpoint: string,
+ activeThreshold: number,
+ printVerbose: boolean,
+) {
+ const gatingResults = results.filter((result) => result.gating);
+ const reportOnlyResults = results.filter((result) => !result.gating);
+ const failed = results.filter((result) => !result.passed);
+ const gatingFailures = gatingResults.filter((result) => !result.passed);
+ const errors = results.filter((result) => result.error);
+
+ console.log("PII accuracy benchmark");
+ console.log(`Endpoint: ${endpoint}`);
+ console.log(`Default threshold: ${activeThreshold}`);
console.log(
- ` ${label.padEnd(20)} P=${formatPercent(metrics.precision).padStart(6)} ` +
- `R=${formatPercent(metrics.recall).padStart(6)} ` +
- `F1=${formatPercent(metrics.f1).padStart(6)} ${status}`,
+ `Cases: ${results.length} (${gatingResults.length} gating, ${reportOnlyResults.length} report-only)`,
);
-}
+ console.log("");
+
+ printMetrics("Overall", [["all", aggregate(results)]]);
+ printMetrics(
+ "By suite",
+ groupMetrics(results, (result) => result.case.suite),
+ );
+ printMetrics(
+ "By category",
+ groupMetrics(results, (result) => result.case.category),
+ );
+ printMetrics(
+ "By language",
+ groupMetrics(results, (result) => result.case.language),
+ );
+ printMetrics("By entity", entityMetrics(results));
-/**
- * Main benchmark execution
- */
-async function main(): Promise<void> {
- console.log("\n╔════════════════════════════════════════════════════════════╗");
- console.log("║ PII Detection Accuracy Benchmark ║");
- console.log("╚════════════════════════════════════════════════════════════╝\n");
+ if (failed.length > 0) {
+ console.log("");
+ console.log(
+ `Failures: ${failed.length} total, ${gatingFailures.length} gating, ${errors.length} errors`,
+ );
- // Check Presidio availability
- console.log(`Presidio URL: ${PRESIDIO_URL}`);
- console.log(`Threshold: ${threshold}`);
- if (languageFilter) {
- console.log(`Languages: ${languageFilter.join(", ")}`);
+ const failuresToPrint = printVerbose ? failed : failed.slice(0, MAX_FAILURES_TO_PRINT);
+ for (const result of failuresToPrint) {
+ printFailure(result);
+ }
+
+ if (!printVerbose && failed.length > failuresToPrint.length) {
+ console.log(`... ${failed.length - failuresToPrint.length} more failures hidden`);
+ console.log("Run with --verbose to print all failure details.");
+ }
}
+}
- if (!(await checkPresidio())) {
- console.error("\n✗ Presidio is not available. Start it with:");
- console.error(" docker compose up presidio-analyzer -d\n");
- process.exit(1);
+function printMetrics(title: string, rows: Array<[string, Metrics]>) {
+ console.log(title);
+ console.log(
+ [
+ "name".padEnd(24),
+ "cases".padStart(5),
+ "pass".padStart(7),
+ "P".padStart(7),
+ "R".padStart(7),
+ "F1".padStart(7),
+ "F2".padStart(7),
+ "err".padStart(5),
+ ].join(" "),
+ );
+
+ for (const [name, metrics] of rows) {
+ console.log(
+ [
+ name.slice(0, 24).padEnd(24),
+ String(metrics.cases).padStart(5),
+ percent(metrics.passed, metrics.cases).padStart(7),
+ percent(metrics.tp, metrics.tp + metrics.fp).padStart(7),
+ percent(metrics.tp, metrics.tp + metrics.fn).padStart(7),
+ fScore(metrics.tp, metrics.fp, metrics.fn, 1).padStart(7),
+ fScore(metrics.tp, metrics.fp, metrics.fn, 2).padStart(7),
+ String(metrics.errors).padStart(5),
+ ].join(" "),
+ );
}
- console.log("Presidio: ✓ Connected\n");
+ console.log("");
+}
- // Load test cases from directory
- const testCases = await loadTestCases();
+function groupMetrics(
+ results: TestResult[],
+ keyFn: (result: TestResult) => string,
+): Array<[string, Metrics]> {
+ const groups = new Map<string, TestResult[]>();
- if (testCases.length === 0) {
- console.error("No test cases found in", TEST_DATA_DIR);
- process.exit(1);
+ for (const result of results) {
+ const key = keyFn(result);
+ groups.set(key, [...(groups.get(key) ?? []), result]);
}
- console.log(`Running ${testCases.length} test cases...\n`);
-
- // Run all test cases
- const results: TestResult[] = [];
- for (const testCase of testCases) {
- try {
- const result = await runTestCase(testCase);
- results.push(result);
-
- if (verbose) {
- const icon = result.passed ? "✓" : "✗";
- console.log(`${icon} ${result.id}`);
- if (!result.passed) {
- if (result.falseNegatives.length > 0) {
- console.log(` Missed: ${result.falseNegatives.map((e) => `${e.type}:"${e.text}"`).join(", ")}`);
- }
- if (result.falsePositives.length > 0) {
- console.log(
- ` Wrong: ${result.falsePositives.map((e) => `${e.type}:"${e.text}" (${e.score.toFixed(2)})`).join(", ")}`,
- );
- }
- }
+ return [...groups.entries()]
+ .map(([key, group]) => [key, aggregate(group)] as [string, Metrics])
+ .sort(([left], [right]) => left.localeCompare(right));
+}
+
+function entityMetrics(results: TestResult[]): Array<[string, Metrics]> {
+ const entities = new Map<string, Metrics>();
+
+ for (const result of results) {
+ const seenEntities = new Set(entitiesForCase(result.case));
+ for (const expected of result.case.expected) {
+ seenEntities.add(expected.entity);
+ }
+ for (const detection of result.unexpected) {
+ seenEntities.add(detection.entity);
+ }
+
+ for (const entity of seenEntities) {
+ if (!entities.has(entity)) {
+ entities.set(entity, emptyMetrics());
}
- } catch (error) {
- console.error(`Error in test ${testCase.id}:`, error);
}
- }
- // Calculate overall metrics
- const overall = calculateMetrics(results);
-
- // Calculate metrics by entity type
- const byEntityType: Record<string, AccuracyMetrics> = {};
- for (const entityType of ENTITY_TYPES) {
- const filtered = results.map((r) => ({
- ...r,
- expected: r.expected.filter((e) => e.type === entityType),
- truePositives: r.truePositives.filter((e) => e.type === entityType),
- falseNegatives: r.falseNegatives.filter((e) => e.type === entityType),
- falsePositives: r.falsePositives.filter((e) => e.type === entityType),
- }));
- // Only include if there are test cases for this type
- const hasTestCases = filtered.some((r) => r.expected.length > 0 || r.falsePositives.length > 0);
- if (hasTestCases) {
- byEntityType[entityType] = calculateMetrics(filtered);
+ for (const entity of seenEntities) {
+ const bucket = entities.get(entity);
+
+ if (!bucket) {
+ continue;
+ }
+
+ bucket.cases += 1;
+ bucket.errors += result.error ? 1 : 0;
+ const tp = result.matched.filter((match) => match.expected.entity === entity).length;
+ const fn = result.missing.filter((missing) => missing.entity === entity).length;
+ const fp = result.unexpected.filter((unexpected) => unexpected.entity === entity).length;
+
+ bucket.tp += tp;
+ bucket.fn += fn;
+ bucket.fp += fp;
+ bucket.passed += result.error || fn > 0 || fp > 0 ? 0 : 1;
}
}
- // Calculate metrics by language
- const languages = [...new Set(results.map((r) => r.language))];
- const byLanguage: Record<string, AccuracyMetrics> = {};
- for (const lang of languages) {
- byLanguage[lang] = calculateMetrics(results.filter((r) => r.language === lang));
+ return [...entities.entries()].sort(([left], [right]) => left.localeCompare(right));
+}
+
+function aggregate(results: TestResult[]): Metrics {
+ const metrics = emptyMetrics();
+
+ for (const result of results) {
+ metrics.cases += 1;
+ metrics.passed += result.passed ? 1 : 0;
+ metrics.errors += result.error ? 1 : 0;
+ metrics.tp += result.matched.length;
+ metrics.fp += result.unexpected.length;
+ metrics.fn += result.error ? result.case.expected.length : result.missing.length;
}
- // Print report
- console.log("\n────────────────────────────────────────────────────────────");
- console.log(" RESULTS");
- console.log("────────────────────────────────────────────────────────────\n");
+ return metrics;
+}
- console.log(`Overall: ${overall.passed}/${overall.total} passed\n`);
+function emptyMetrics(): Metrics {
+ return {
+ cases: 0,
+ passed: 0,
+ errors: 0,
+ tp: 0,
+ fp: 0,
+ fn: 0,
+ };
+}
- console.log("Metrics (P=Precision, R=Recall, F1=F1-Score):\n");
+function printFailure(result: TestResult) {
+ const mode = result.gating ? "gating" : "report-only";
+ console.log(`- ${result.case.id} [${result.case.suite}/${result.case.category}/${mode}]`);
- console.log(
- ` ${"OVERALL".padEnd(20)} P=${formatPercent(overall.precision).padStart(6)} ` +
- `R=${formatPercent(overall.recall).padStart(6)} ` +
- `F1=${formatPercent(overall.f1).padStart(6)}\n`,
- );
+ if (result.error) {
+ console.log(` error: ${result.error}`);
+ return;
+ }
- console.log("By Entity Type:");
- for (const [type, metrics] of Object.entries(byEntityType)) {
- printMetrics(type, metrics);
+ if (result.missing.length > 0) {
+ console.log(
+ ` missing: ${result.missing
+ .map((expected) => `${expected.entity}(${expected.text})`)
+ .join(", ")}`,
+ );
}
- console.log("\nBy Language:");
- for (const [lang, metrics] of Object.entries(byLanguage)) {
- printMetrics(lang.toUpperCase(), metrics);
+ if (result.unexpected.length > 0) {
+ console.log(
+ ` unexpected: ${result.unexpected
+ .map((detection) => `${detection.entity}(${detection.text}, ${detection.score.toFixed(2)})`)
+ .join(", ")}`,
+ );
}
+}
- // Print false negatives (missed PII)
- const allFalseNegatives = results.flatMap((r) =>
- r.falseNegatives.map((fn) => ({ testId: r.id, text: r.text, entity: fn })),
- );
- if (allFalseNegatives.length > 0) {
- console.log("\n────────────────────────────────────────────────────────────");
- console.log(`False Negatives (Missed PII): ${allFalseNegatives.length}`);
- console.log("────────────────────────────────────────────────────────────");
- for (const { testId, entity } of allFalseNegatives) {
- console.log(` ✗ "${entity.text}" (${entity.type}) in ${testId}`);
- }
+function percent(numerator: number, denominator: number): string {
+ if (denominator === 0) {
+ return "n/a";
}
- // Print false positives (wrong detections)
- const allFalsePositives = results.flatMap((r) =>
- r.falsePositives.map((fp) => ({ testId: r.id, text: r.text, entity: fp })),
- );
- if (allFalsePositives.length > 0) {
- console.log("\n────────────────────────────────────────────────────────────");
- console.log(`False Positives (Wrong Detections): ${allFalsePositives.length}`);
- console.log("────────────────────────────────────────────────────────────");
- for (const { testId, entity } of allFalsePositives) {
- console.log(` ✗ "${entity.text}" (${entity.type}, score=${entity.score.toFixed(2)}) in ${testId}`);
- }
+ return `${((numerator / denominator) * 100).toFixed(1)}%`;
+}
+
+function fScore(tp: number, fp: number, fn: number, beta: number): string {
+ const betaSquared = beta * beta;
+ const denominator = (1 + betaSquared) * tp + betaSquared * fn + fp;
+
+ if (denominator === 0) {
+ return "n/a";
}
- console.log("\n────────────────────────────────────────────────────────────\n");
+ return `${(((1 + betaSquared) * tp) / denominator).toFixed(3)}`;
+}
- // Exit with error code if tests failed
- if (overall.failed > 0) {
- process.exit(1);
+function buildFilters(values: typeof argv.values): Filters {
+ return {
+ suites: csvSet(values.suite),
+ categories: csvSet(values.category),
+ languages: csvSet(values.languages),
+ split: csvSet(values.split),
+ };
+}
+
+function csvSet(value: string | boolean | undefined): Set<string> | undefined {
+ if (typeof value !== "string" || value.trim() === "") {
+ return undefined;
}
+
+ return new Set(
+ value
+ .split(",")
+ .map((part) => part.trim())
+ .filter(Boolean),
+ );
}
-main().catch((error) => {
- console.error("Benchmark failed:", error);
- process.exit(1);
-});
+function parseThreshold(value: string): number {
+ const parsed = Number(value);
+
+ if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+ throw new Error(`Invalid threshold: ${value}`);
+ }
+
+ return parsed;
+}
+
+function printHelp() {
+ console.log(`Usage: bun run benchmarks/pii-accuracy/run.ts [options]
+
+Options:
+ --url <url> Analyze endpoint. Default: ${DEFAULT_ANALYZE_URL}
+ --threshold <0..1> Default score threshold. Default: ${DEFAULT_THRESHOLD}
+ --suite <csv> Filter suites, e.g. core,precision
+ --category <csv> Filter categories: core,precision,eval,hard
+ --languages <csv> Filter languages, e.g. en,de,it
+ --split <csv> Filter split: dev,test
+ --list-suites Print suites and exit
+ --verbose Print all failure details
+ --help Print this help
+`);
+}
--- /dev/null
+export type SuiteDefinition = {
+ description: string;
+ defaultEntities: readonly string[];
+};
+
+export const CONFIGURED_PII_ENTITIES = [
+ "CREDIT_CARD",
+ "EMAIL_ADDRESS",
+ "IBAN_CODE",
+ "IP_ADDRESS",
+ "LOCATION",
+ "PERSON",
+ "PHONE_NUMBER",
+] as const;
+
+export const SUPPORTED_LANGUAGES = ["en", "de", "es", "fr", "it", "nl", "pl", "pt", "ro"] as const;
+
+export const SUITES: Record<string, SuiteDefinition> = {
+ core: {
+ description: "Configured PII entities used by Pasteguard's default detection setup.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ precision: {
+ description: "Negative controls where common non-PII strings must not be flagged.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ "precision-paragraphs": {
+ description: "Longer non-PII paragraphs with lookalike strings and operational text.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ "multilingual-sentences": {
+ description: "Sentence-level coverage for every configured entity across supported languages.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ "multilingual-paragraphs": {
+ description: "Paragraph-level workflow coverage for every configured entity across languages.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ boundaries: {
+ description: "Span boundary checks around punctuation, brackets, quotes, and repeated context.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ hard: {
+ description: "Known difficult, ambiguous, or edge-case strings. Report-only by default.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+ eval: {
+ description: "Multilingual smoke corpus for trend tracking across configured languages.",
+ defaultEntities: CONFIGURED_PII_ENTITIES,
+ },
+};
+
+export function isKnownSuite(suite: string): boolean {
+ return suite in SUITES;
+}
+
+export function suiteNames(): string[] {
+ return Object.keys(SUITES).sort();
+}
--- /dev/null
+cases:
+ - id: boundary_email_parentheses
+ suite: boundaries
+ category: hard
+ split: test
+ language: en
+ text: "Primary contact (zoe.parker+legal@example.co.uk) approved the change."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: zoe.parker+legal@example.co.uk
+ match: exact
+
+ - id: boundary_phone_quotes
+ suite: boundaries
+ category: hard
+ split: test
+ language: en
+ text: "The note says call '+1 212-555-0199' after the legal review."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +1 212-555-0199
+ match: contains
+
+ - id: boundary_credit_card_commas
+ suite: boundaries
+ category: hard
+ split: test
+ language: en
+ text: "The escrow record has card 4111 1111 1111 1111, but the invoice does not."
+ expected:
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+
+ - id: boundary_iban_brackets
+ suite: boundaries
+ category: hard
+ split: test
+ language: de
+ text: "Das Konto [DE89370400440532013000] wurde nur im Audit-Log genannt."
+ expected:
+ - entity: IBAN_CODE
+ text: DE89370400440532013000
+ match: exact
+
+ - id: boundary_ip_brackets
+ suite: boundaries
+ category: hard
+ split: test
+ language: en
+ text: "The firewall event recorded src=[198.51.100.44] and no destination owner."
+ expected:
+ - entity: IP_ADDRESS
+ text: 198.51.100.44
+ match: exact
+
+ - id: boundary_person_sentence_start
+ suite: boundaries
+ category: hard
+ split: test
+ language: en
+ text: "Priya Shah approved access. The rest of the sentence is generic."
+ expected:
+ - entity: PERSON
+ text: Priya Shah
+ match: contains
+
+ - id: boundary_location_with_punctuation
+ suite: boundaries
+ category: hard
+ split: test
+ language: fr
+ text: "Le bureau concerne Lyon; aucune autre ville n'est dans le message."
+ expected:
+ - entity: LOCATION
+ text: Lyon
+ match: contains
+
+ - id: boundary_repeated_context_single_email
+ suite: boundaries
+ category: hard
+ split: test
+ language: en
+ text: "Email the customer email owner at customer.owner@example.com; do not email the team alias."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: customer.owner@example.com
+ match: exact
--- /dev/null
+cases:
+ - id: core_email_en
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "Email Alice at alice.smith@example.com before noon."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: alice.smith@example.com
+ match: exact
+
+ - id: core_email_de
+ suite: core
+ category: core
+ split: test
+ language: de
+ text: "Schreibe an max.mueller@example.de, wenn der Export fertig ist."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: max.mueller@example.de
+ match: exact
+
+ - id: core_email_plus_subdomain
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "Send invoices to alice.billing+eu@sub.example.co.uk today."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: alice.billing+eu@sub.example.co.uk
+ match: exact
+
+ - id: core_email_uppercase
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "The backup mailbox is SECURITY.ALERTS@EXAMPLE.ORG."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: SECURITY.ALERTS@EXAMPLE.ORG
+ match: exact
+
+ - id: core_phone_en
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "The customer phone number is +1 415-555-2671."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +1 415-555-2671
+ match: contains
+
+ - id: core_phone_de
+ suite: core
+ category: core
+ split: test
+ language: de
+ text: "Die Telefonnummer des Kontakts ist +49 30 12345678."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +49 30 12345678
+ match: contains
+
+ - id: core_phone_fr
+ suite: core
+ category: core
+ split: test
+ language: fr
+ text: "Le numéro de téléphone est +33 1 42 68 53 00."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +33 1 42 68 53 00
+ match: contains
+
+ - id: core_phone_es
+ suite: core
+ category: core
+ split: test
+ language: es
+ text: "El teléfono de soporte es +34 91 123 45 67."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +34 91 123 45 67
+ match: contains
+
+ - id: core_phone_it
+ suite: core
+ category: core
+ split: test
+ language: it
+ text: "Il numero di telefono è +39 06 1234 5678."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +39 06 1234 5678
+ match: contains
+
+ - id: core_phone_nl
+ suite: core
+ category: core
+ split: test
+ language: nl
+ text: "Het telefoonnummer is +31 20 123 4567."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +31 20 123 4567
+ match: contains
+
+ - id: core_phone_pt
+ suite: core
+ category: core
+ split: test
+ language: pt
+ text: "O número de telefone é +351 21 123 4567."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +351 21 123 4567
+ match: contains
+
+ - id: core_phone_pl
+ suite: core
+ category: core
+ split: test
+ language: pl
+ text: "Numer telefonu kontaktowego to +48 22 123 45 67."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +48 22 123 45 67
+ match: contains
+
+ - id: core_phone_ro
+ suite: core
+ category: core
+ split: test
+ language: ro
+ text: "Numărul de telefon este +40 21 123 4567."
+ expected:
+ - entity: PHONE_NUMBER
+ text: +40 21 123 4567
+ match: contains
+
+ - id: core_credit_card_spaces
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "Use test card 4111 1111 1111 1111 for the checkout sandbox."
+ expected:
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+
+ - id: core_credit_card_hyphen
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "Use card 4012-8888-8888-1881 for the payment test."
+ expected:
+ - entity: CREDIT_CARD
+ text: 4012-8888-8888-1881
+ match: exact
+
+ - id: core_iban_de
+ suite: core
+ category: core
+ split: test
+ language: de
+ text: "Das Auszahlungskonto lautet DE89370400440532013000."
+ expected:
+ - entity: IBAN_CODE
+ text: DE89370400440532013000
+ match: exact
+
+ - id: core_iban_it
+ suite: core
+ category: core
+ split: test
+ language: it
+ text: "Il conto del fornitore e IT60X0542811101000000123456."
+ expected:
+ - entity: IBAN_CODE
+ text: IT60X0542811101000000123456
+ match: exact
+
+ - id: core_iban_fr
+ suite: core
+ category: core
+ split: test
+ language: fr
+ text: "Le compte fournisseur est FR1420041010050500013M02606."
+ expected:
+ - entity: IBAN_CODE
+ text: FR1420041010050500013M02606
+ match: exact
+
+ - id: core_iban_es
+ suite: core
+ category: core
+ split: test
+ language: es
+ text: "La cuenta de pago es ES9121000418450200051332."
+ expected:
+ - entity: IBAN_CODE
+ text: ES9121000418450200051332
+ match: exact
+
+ - id: core_iban_nl
+ suite: core
+ category: core
+ split: test
+ language: nl
+ text: "De betaalrekening is NL91ABNA0417164300."
+ expected:
+ - entity: IBAN_CODE
+ text: NL91ABNA0417164300
+ match: exact
+
+ - id: core_iban_pt
+ suite: core
+ category: core
+ split: test
+ language: pt
+ text: "A conta de pagamento e PT50000201231234567890154."
+ expected:
+ - entity: IBAN_CODE
+ text: PT50000201231234567890154
+ match: exact
+
+ - id: core_iban_pl
+ suite: core
+ category: core
+ split: test
+ language: pl
+ text: "Rachunek płatności to PL61109010140000071219812874."
+ expected:
+ - entity: IBAN_CODE
+ text: PL61109010140000071219812874
+ match: exact
+
+ - id: core_iban_ro
+ suite: core
+ category: core
+ split: test
+ language: ro
+ text: "Contul de plată este RO49AAAA1B31007593840000."
+ expected:
+ - entity: IBAN_CODE
+ text: RO49AAAA1B31007593840000
+ match: exact
+
+ - id: core_ipv4_public
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "The last login came from IP address 203.0.113.42."
+ expected:
+ - entity: IP_ADDRESS
+ text: 203.0.113.42
+ match: exact
+
+ - id: core_ipv4_private
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "The admin panel is reachable at private IP 10.24.8.5."
+ expected:
+ - entity: IP_ADDRESS
+ text: 10.24.8.5
+ match: exact
+
+ - id: core_person_en
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "John Smith approved the access request yesterday."
+ expected:
+ - entity: PERSON
+ text: John Smith
+ match: contains
+
+ - id: core_person_non_western_arabic
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "Please add Amina Al-Rashid to the project access list."
+ expected:
+ - entity: PERSON
+ text: Amina Al-Rashid
+ match: contains
+
+ - id: core_person_non_western_chinese
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "The reviewer Wei Zhang approved the vendor exception."
+ expected:
+ - entity: PERSON
+ text: Wei Zhang
+ match: contains
+
+ - id: core_person_non_western_indian
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "Escalate the open ticket to Priya Krishnan in support."
+ expected:
+ - entity: PERSON
+ text: Priya Krishnan
+ match: contains
+
+ - id: core_person_non_western_turkish_de
+ suite: core
+ category: core
+ split: test
+ language: de
+ text: "Bitte trage Mehmet Yıldız in die Zugriffsliste ein."
+ expected:
+ - entity: PERSON
+ text: Mehmet Yıldız
+ match: contains
+
+ - id: core_location_en
+ suite: core
+ category: core
+ split: test
+ language: en
+ text: "The backup office is located in Berlin."
+ expected:
+ - entity: LOCATION
+ text: Berlin
+ match: contains
+++ /dev/null
-test_cases:
- # Phone (needs German context words)
- - id: de_phone
- text: "Telefon: +49 171 1234567"
- language: de
- expected:
- - type: PHONE_NUMBER
- text: "+49 171 1234567"
-
- # Person (NER model)
- - id: de_person
- text: "Meeting mit Max Müller morgen um 10 Uhr"
- language: de
- expected:
- - type: PERSON
- text: "Max Müller"
-
- # Location (NER model)
- - id: de_location
- text: "Das Meeting findet in München statt"
- language: de
- expected:
- - type: LOCATION
- text: "München"
-
- # Mixed (real-world prompt)
- - id: de_mixed
- text: "Kontaktdaten von Hans Meier: hans.meier@example.com, Telefon 0171-1234567"
- language: de
- expected:
- - type: PERSON
- text: "Hans Meier"
- - type: EMAIL_ADDRESS
- text: "hans.meier@example.com"
- - type: PHONE_NUMBER
- text: "0171-1234567"
-
- # False Positive
- - id: de_fp
- text: "Das Wetter ist heute sehr schön"
- language: de
- expected: []
+++ /dev/null
-test_cases:
- # Phone (needs English context words)
- - id: en_phone
- text: "Phone: (555) 123-4567"
- language: en
- expected:
- - type: PHONE_NUMBER
- text: "(555) 123-4567"
-
- # Person (NER model)
- - id: en_person
- text: "Schedule a meeting with John Smith tomorrow"
- language: en
- expected:
- - type: PERSON
- text: "John Smith"
-
- # Location (NER model)
- - id: en_location
- text: "The conference is in New York next week"
- language: en
- expected:
- - type: LOCATION
- text: "New York"
-
- # Mixed (real-world prompt)
- - id: en_mixed
- text: "Customer John Doe (john@example.com), phone (555) 987-6543"
- language: en
- expected:
- - type: PERSON
- text: "John Doe"
- - type: EMAIL_ADDRESS
- text: "john@example.com"
- - type: PHONE_NUMBER
- text: "(555) 987-6543"
-
- # False Positive
- - id: en_fp
- text: "The weather is nice today"
- language: en
- expected: []
+++ /dev/null
-test_cases:
- # Phone (needs Spanish context words)
- - id: es_phone
- text: "Teléfono: +34 612 345 678"
- language: es
- expected:
- - type: PHONE_NUMBER
- text: "+34 612 345 678"
-
- # Person (NER model)
- - id: es_person
- text: "Reunión con Ana Martínez mañana a las 10"
- language: es
- expected:
- - type: PERSON
- text: "Ana Martínez"
-
- # Location (NER model)
- - id: es_location
- text: "La reunión será en Madrid la próxima semana"
- language: es
- expected:
- - type: LOCATION
- text: "Madrid"
-
- # Mixed (real-world prompt)
- - id: es_mixed
- text: "Contacto de Carlos García: carlos.garcia@example.es, teléfono +34 698 765 432"
- language: es
- expected:
- - type: PERSON
- text: "Carlos García"
- - type: EMAIL_ADDRESS
- text: "carlos.garcia@example.es"
- - type: PHONE_NUMBER
- text: "+34 698 765 432"
-
- # False Positive
- - id: es_fp
- text: "El clima está muy agradable hoy"
- language: es
- expected: []
--- /dev/null
+cases:
+ - id: eval_de_mixed_contact
+ suite: eval
+ category: eval
+ split: test
+ language: de
+ text: "Maria Keller aus Berlin ist unter maria.keller@example.de erreichbar."
+ expected:
+ - entity: PERSON
+ text: Maria Keller
+ match: contains
+ - entity: LOCATION
+ text: Berlin
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: maria.keller@example.de
+ match: exact
+
+ - id: eval_it_mixed_contact
+ suite: eval
+ category: eval
+ split: test
+ language: it
+ text: "Luca Bianchi lavora a Roma e usa luca.bianchi@example.it."
+ expected:
+ - entity: PERSON
+ text: Luca Bianchi
+ match: contains
+ - entity: LOCATION
+ text: Roma
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: luca.bianchi@example.it
+ match: exact
+
+ - id: eval_fr_mixed_contact
+ suite: eval
+ category: eval
+ split: test
+ language: fr
+ text: "Claire Martin vit a Paris et son email est claire.martin@example.fr."
+ expected:
+ - entity: PERSON
+ text: Claire Martin
+ match: contains
+ - entity: LOCATION
+ text: Paris
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: claire.martin@example.fr
+ match: exact
+
+ - id: eval_es_mixed_contact
+ suite: eval
+ category: eval
+ split: test
+ language: es
+ text: "Carlos Garcia vive en Madrid y responde en carlos.garcia@example.es."
+ expected:
+ - entity: PERSON
+ text: Carlos Garcia
+ match: contains
+ - entity: LOCATION
+ text: Madrid
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: carlos.garcia@example.es
+ match: exact
+
+ - id: eval_nl_phone_email
+ suite: eval
+ category: eval
+ split: test
+ language: nl
+ text: "Sanne de Vries gebruikt sanne@example.nl en telefoon +31 20 123 4567."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: sanne@example.nl
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +31 20 123 4567
+ match: contains
+
+ - id: eval_nl_person_location
+ suite: eval
+ category: eval
+ split: test
+ language: nl
+ text: "Jan de Vries woont in Rotterdam en gebruikt jan.devries@example.nl."
+ expected:
+ - entity: PERSON
+ text: Jan de Vries
+ match: contains
+ - entity: LOCATION
+ text: Rotterdam
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: jan.devries@example.nl
+ match: exact
+
+ - id: eval_pt_phone_email
+ suite: eval
+ category: eval
+ split: test
+ language: pt
+ text: "Joao Silva usa joao.silva@example.pt e telefone +351 21 123 4567."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: joao.silva@example.pt
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +351 21 123 4567
+ match: contains
+
+ - id: eval_pt_person_location
+ suite: eval
+ category: eval
+ split: test
+ language: pt
+ text: "Tiago Costa mora no Porto e usa tiago.costa@example.pt."
+ expected:
+ - entity: PERSON
+ text: Tiago Costa
+ match: contains
+ - entity: LOCATION
+ text: Porto
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: tiago.costa@example.pt
+ match: exact
+
+ - id: eval_pl_mixed_contact
+ suite: eval
+ category: eval
+ split: test
+ language: pl
+ text: "Anna Kowalska z Warszawy używa adresu anna.kowalska@example.pl."
+ expected:
+ - entity: PERSON
+ text: Anna Kowalska
+ match: contains
+ - entity: LOCATION
+ text: Warszawy
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: anna.kowalska@example.pl
+ match: exact
+
+ - id: eval_ro_mixed_contact
+ suite: eval
+ category: eval
+ split: test
+ language: ro
+ text: "Andrei Popescu din București folosește adresa andrei.popescu@example.ro."
+ expected:
+ - entity: PERSON
+ text: Andrei Popescu
+ match: contains
+ - entity: LOCATION
+ text: București
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: andrei.popescu@example.ro
+ match: exact
+
+ - id: eval_en_security_event
+ suite: eval
+ category: eval
+ split: test
+ language: en
+ text: "Alice Brown signed in from 8.8.8.8 and confirmed alice.brown@example.com."
+ expected:
+ - entity: PERSON
+ text: Alice Brown
+ match: contains
+ - entity: IP_ADDRESS
+ text: 8.8.8.8
+ match: exact
+ - entity: EMAIL_ADDRESS
+ text: alice.brown@example.com
+ match: exact
+
+ - id: eval_de_payment_contact
+ suite: eval
+ category: eval
+ split: test
+ language: de
+ text: "Max Berger aus Hamburg nutzt das Konto DE89370400440532013000 und max.berger@example.de."
+ expected:
+ - entity: PERSON
+ text: Max Berger
+ match: contains
+ - entity: LOCATION
+ text: Hamburg
+ match: contains
+ - entity: IBAN_CODE
+ text: DE89370400440532013000
+ match: exact
+ - entity: EMAIL_ADDRESS
+ text: max.berger@example.de
+ match: exact
+
+ - id: eval_de_spaced_iban
+ suite: eval
+ category: eval
+ split: test
+ language: de
+ text: "Bitte ueberweisen Sie auf IBAN DE89 3704 0044 0532 0130 00."
+ expected:
+ - entity: IBAN_CODE
+ text: DE89 3704 0044 0532 0130 00
+ match: exact
+
+ - id: eval_it_payment_contact
+ suite: eval
+ category: eval
+ split: test
+ language: it
+ text: "Giulia Rossi vive a Milano e usa il conto IT60X0542811101000000123456."
+ expected:
+ - entity: PERSON
+ text: Giulia Rossi
+ match: contains
+ - entity: LOCATION
+ text: Milano
+ match: contains
+ - entity: IBAN_CODE
+ text: IT60X0542811101000000123456
+ match: exact
+
+ - id: eval_it_spaced_iban
+ suite: eval
+ category: eval
+ split: test
+ language: it
+ text: "Bonifico sull'IBAN IT60 X054 2811 1010 0000 0123 456 entro lunedi."
+ expected:
+ - entity: IBAN_CODE
+ text: IT60 X054 2811 1010 0000 0123 456
+ match: exact
+
+ - id: eval_fr_payment_contact
+ suite: eval
+ category: eval
+ split: test
+ language: fr
+ text: "Nicolas Bernard à Lyon utilise le compte FR1420041010050500013M02606."
+ expected:
+ - entity: PERSON
+ text: Nicolas Bernard
+ match: contains
+ - entity: LOCATION
+ text: Lyon
+ match: contains
+ - entity: IBAN_CODE
+ text: FR1420041010050500013M02606
+ match: exact
+++ /dev/null
-test_cases:
- # Phone (needs French context words)
- - id: fr_phone
- text: "Téléphone: 06 12 34 56 78"
- language: fr
- expected:
- - type: PHONE_NUMBER
- text: "06 12 34 56 78"
-
- # Person (NER model)
- - id: fr_person
- text: "J'ai une réunion avec Marie Dubois demain à 10h"
- language: fr
- expected:
- - type: PERSON
- text: "Marie Dubois"
-
- # Location (NER model)
- - id: fr_location
- text: "La conférence aura lieu à Lyon la semaine prochaine"
- language: fr
- expected:
- - type: LOCATION
- text: "Lyon"
-
- # Mixed (real-world prompt)
- - id: fr_mixed
- text: "Contact de Jean Dupont: jean.dupont@example.fr, téléphone 06 98 76 54 32"
- language: fr
- expected:
- - type: PERSON
- text: "Jean Dupont"
- - type: EMAIL_ADDRESS
- text: "jean.dupont@example.fr"
- - type: PHONE_NUMBER
- text: "06 98 76 54 32"
-
- # False Positive
- - id: fr_fp
- text: "Le temps est magnifique aujourd'hui"
- language: fr
- expected: []
+++ /dev/null
-test_cases:
- # Pattern-based recognizers - language independent
- # Tested once since regex/checksum works the same for all languages
-
- # Email
- - id: global_email
- text: "Contact me at john.doe@company.com"
- language: en
- expected:
- - type: EMAIL_ADDRESS
- text: "john.doe@company.com"
-
- # IBAN
- - id: global_iban
- text: "Transfer to IBAN DE89370400440532013000"
- language: en
- expected:
- - type: IBAN_CODE
- text: "DE89370400440532013000"
-
- # Credit Card
- - id: global_credit_card
- text: "Card number: 4111 1111 1111 1111"
- language: en
- expected:
- - type: CREDIT_CARD
- text: "4111 1111 1111 1111"
-
- # IP Address
- - id: global_ip
- text: "Server IP is 8.8.8.8"
- language: en
- expected:
- - type: IP_ADDRESS
- text: "8.8.8.8"
-
- # Empty text
- - id: global_empty
- text: ""
- language: en
- expected: []
--- /dev/null
+cases:
+ - id: hard_ipv6_full
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "The IPv6 source address was 2001:0db8:85a3:0000:0000:8a2e:0370:7334."
+ expected:
+ - entity: IP_ADDRESS
+ text: 2001:0db8:85a3:0000:0000:8a2e:0370:7334
+ match: exact
+
+ - id: hard_ipv6_compressed
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "The compressed IPv6 address is 2001:db8::8a2e:370:7334."
+ expected:
+ - entity: IP_ADDRESS
+ text: 2001:db8::8a2e:370:7334
+ match: exact
+
+ - id: hard_email_inside_sentence
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "Please email alice.smith+billing@example.co.uk, not the shared inbox."
+ expected:
+ - entity: EMAIL_ADDRESS
+ text: alice.smith+billing@example.co.uk
+ match: exact
+
+ - id: hard_name_lowercase
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "The note says john smith can approve only read-only access."
+ expected:
+ - entity: PERSON
+ text: john smith
+ match: contains
+
+ - id: hard_name_lowercase_it
+ suite: hard
+ category: hard
+ split: test
+ language: it
+ text: "ricorda di chiamare mario rossi per la fattura"
+ expected:
+ - entity: PERSON
+ text: mario rossi
+ match: contains
+
+ - id: hard_person_hyphenated
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "Anne-Marie Johnson approved the vendor exception."
+ expected:
+ - entity: PERSON
+ text: Anne-Marie Johnson
+ match: contains
+
+ - id: hard_phone_national_uk
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "The London callback number is 020 7946 0958."
+ expected:
+ - entity: PHONE_NUMBER
+ text: 020 7946 0958
+ match: contains
+
+ - id: hard_location_ambiguous_city
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ text: "The request mentions Reading as the office location."
+ expected:
+ - entity: LOCATION
+ text: Reading
+ match: contains
+
+ - id: hard_address_street_de
+ suite: hard
+ category: hard
+ split: test
+ language: de
+ entities: [LOCATION]
+ text: "Die Lieferadresse lautet Bahnhofstraße 12."
+ expected:
+ - entity: LOCATION
+ text: Bahnhofstraße 12
+ match: contains
+
+ - id: hard_address_full_de
+ suite: hard
+ category: hard
+ split: test
+ language: de
+ entities: [LOCATION]
+ text: "Die Lieferadresse lautet Bahnhofstraße 12, 10115 Berlin."
+ expected:
+ - entity: LOCATION
+ text: Bahnhofstraße 12, 10115 Berlin
+ match: contains
+
+ - id: hard_address_full_en
+ suite: hard
+ category: hard
+ split: test
+ language: en
+ entities: [LOCATION]
+ text: "Ship the order to 1600 Pennsylvania Avenue, Washington."
+ expected:
+ - entity: LOCATION
+ text: 1600 Pennsylvania Avenue, Washington
+ match: contains
+
+ - id: hard_address_full_fr
+ suite: hard
+ category: hard
+ split: test
+ language: fr
+ entities: [LOCATION]
+ text: "L'adresse de livraison est 25 Rue du Faubourg Saint-Honoré, Paris."
+ expected:
+ - entity: LOCATION
+ text: 25 Rue du Faubourg Saint-Honoré, Paris
+ match: contains
+++ /dev/null
-test_cases:
- # Phone (needs Italian context words)
- - id: it_phone
- text: "Telefono: 333 1234567"
- language: it
- expected:
- - type: PHONE_NUMBER
- text: "333 1234567"
-
- # Person (NER model)
- - id: it_person
- text: "Riunione con Giuseppe Verdi domani alle 10"
- language: it
- expected:
- - type: PERSON
- text: "Giuseppe Verdi"
-
- # Location (NER model)
- - id: it_location
- text: "L'evento si terrà a Milano il prossimo mese"
- language: it
- expected:
- - type: LOCATION
- text: "Milano"
-
- # Mixed (real-world prompt)
- - id: it_mixed
- text: "Contatto di Marco Rossi: marco.rossi@example.it, telefono 333 9876543"
- language: it
- expected:
- - type: PERSON
- text: "Marco Rossi"
- - type: EMAIL_ADDRESS
- text: "marco.rossi@example.it"
- - type: PHONE_NUMBER
- text: "333 9876543"
-
- # False Positive
- - id: it_fp
- text: "Il caffè italiano è il migliore del mondo"
- language: it
- expected: []
--- /dev/null
+cases:
+ - id: paragraph_en_customer_handoff
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: en
+ text: "The support handoff says Oliver Bennett is based in Manchester and should be contacted at oliver.bennett@example.com. The ticket also lists +44 161 496 0123, card 4242 4242 4242 4242, settlement IBAN GB82WEST12345698765432, and the last portal login from 203.0.113.77."
+ expected:
+ - entity: PERSON
+ text: Oliver Bennett
+ match: contains
+ - entity: LOCATION
+ text: Manchester
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: oliver.bennett@example.com
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +44 161 496 0123
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4242 4242 4242 4242
+ match: exact
+ - entity: IBAN_CODE
+ text: GB82WEST12345698765432
+ match: exact
+ - entity: IP_ADDRESS
+ text: 203.0.113.77
+ match: exact
+
+ - id: paragraph_de_vendor_review
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: de
+ text: "Im Lieferantenprotokoll steht, dass Nora Weber in München arbeitet und nora.weber@example.de nutzt. Fuer den Rueckruf ist +49 89 1234 5678 hinterlegt, die Karte lautet 4111 1111 1111 1111, das Konto DE89370400440532013000 und der Zugriff kam von 198.51.100.21."
+ expected:
+ - entity: PERSON
+ text: Nora Weber
+ match: contains
+ - entity: LOCATION
+ text: München
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: nora.weber@example.de
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +49 89 1234 5678
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+ - entity: IBAN_CODE
+ text: DE89370400440532013000
+ match: exact
+ - entity: IP_ADDRESS
+ text: 198.51.100.21
+ match: exact
+
+ - id: paragraph_es_refund_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: es
+ text: "La solicitud de reembolso indica que Mateo Ruiz vive en Sevilla y responde en mateo.ruiz@example.es. En la nota aparecen el telefono +34 95 123 45 67, la tarjeta 4012 8888 8888 1881, el IBAN ES9121000418450200051332 y la IP 192.0.2.34 usada en el portal."
+ expected:
+ - entity: PERSON
+ text: Mateo Ruiz
+ match: contains
+ - entity: LOCATION
+ text: Sevilla
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: mateo.ruiz@example.es
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +34 95 123 45 67
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4012 8888 8888 1881
+ match: exact
+ - entity: IBAN_CODE
+ text: ES9121000418450200051332
+ match: exact
+ - entity: IP_ADDRESS
+ text: 192.0.2.34
+ match: exact
+
+ - id: paragraph_fr_payment_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: fr
+ text: "Le dossier client mentionne Sophie Lambert, rattachee au bureau de Marseille, avec l'adresse sophie.lambert@example.fr. Le suivi contient aussi le telephone +33 4 91 12 34 56, la carte 5555 5555 5555 4444, l'IBAN FR1420041010050500013M02606 et l'adresse IP 10.44.12.8."
+ expected:
+ - entity: PERSON
+ text: Sophie Lambert
+ match: contains
+ - entity: LOCATION
+ text: Marseille
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: sophie.lambert@example.fr
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +33 4 91 12 34 56
+ match: contains
+ - entity: CREDIT_CARD
+ text: 5555 5555 5555 4444
+ match: exact
+ - entity: IBAN_CODE
+ text: FR1420041010050500013M02606
+ match: exact
+ - entity: IP_ADDRESS
+ text: 10.44.12.8
+ match: exact
+
+ - id: paragraph_it_booking_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: it
+ text: "Nel riepilogo prenotazione, Chiara Ferri risulta a Bologna e usa chiara.ferri@example.it. Il record include il telefono +39 051 123 4567, la carta 4242 4242 4242 4242, l'IBAN IT60X0542811101000000123456 e l'IP 172.20.4.9."
+ expected:
+ - entity: PERSON
+ text: Chiara Ferri
+ match: contains
+ - entity: LOCATION
+ text: Bologna
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: chiara.ferri@example.it
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +39 051 123 4567
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4242 4242 4242 4242
+ match: exact
+ - entity: IBAN_CODE
+ text: IT60X0542811101000000123456
+ match: exact
+ - entity: IP_ADDRESS
+ text: 172.20.4.9
+ match: exact
+
+ - id: paragraph_nl_account_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: nl
+ text: "Het accountoverzicht noemt Eva Bakker in Haarlem met e-mailadres eva.bakker@example.nl. Voor verificatie staan telefoon +31 23 123 4567, kaart 4111 1111 1111 1111, IBAN NL91ABNA0417164300 en IP-adres 8.8.8.8 in hetzelfde bericht."
+ expected:
+ - entity: PERSON
+ text: Eva Bakker
+ match: contains
+ - entity: LOCATION
+ text: Haarlem
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: eva.bakker@example.nl
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +31 23 123 4567
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+ - entity: IBAN_CODE
+ text: NL91ABNA0417164300
+ match: exact
+ - entity: IP_ADDRESS
+ text: 8.8.8.8
+ match: exact
+
+ - id: paragraph_pl_support_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: pl
+ text: "W zgłoszeniu serwisowym Marta Zielinska podaje Kraków jako lokalizację i adres marta.zielinska@example.pl. W dalszej części są telefon +48 12 123 45 67, karta 5555 5555 5555 4444, IBAN PL61109010140000071219812874 oraz IP 1.0.0.1."
+ expected:
+ - entity: PERSON
+ text: Marta Zielinska
+ match: contains
+ - entity: LOCATION
+ text: Kraków
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: marta.zielinska@example.pl
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +48 12 123 45 67
+ match: contains
+ - entity: CREDIT_CARD
+ text: 5555 5555 5555 4444
+ match: exact
+ - entity: IBAN_CODE
+ text: PL61109010140000071219812874
+ match: exact
+ - entity: IP_ADDRESS
+ text: 1.0.0.1
+ match: exact
+
+ - id: paragraph_pt_invoice_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: pt
+ text: "A fatura pendente pertence a Bruno Costa no Porto, com contacto bruno.costa@example.pt. O mesmo paragrafo inclui telefone +351 22 123 4567, cartao 4242 4242 4242 4242, IBAN PT50000201231234567890154 e IP 45.33.32.156."
+ expected:
+ - entity: PERSON
+ text: Bruno Costa
+ match: contains
+ - entity: LOCATION
+ text: Porto
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: bruno.costa@example.pt
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +351 22 123 4567
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4242 4242 4242 4242
+ match: exact
+ - entity: IBAN_CODE
+ text: PT50000201231234567890154
+ match: exact
+ - entity: IP_ADDRESS
+ text: 45.33.32.156
+ match: exact
+
+ - id: paragraph_ro_contract_case
+ suite: multilingual-paragraphs
+ category: eval
+ split: test
+ language: ro
+ text: "Contractul intern il mentioneaza pe Mihai Stan din Cluj-Napoca si adresa mihai.stan@example.ro. Pentru validare apar telefonul +40 264 123 456, cardul 4012 8888 8888 1881, IBAN RO49AAAA1B31007593840000 si IP 185.199.109.153."
+ expected:
+ - entity: PERSON
+ text: Mihai Stan
+ match: contains
+ - entity: LOCATION
+ text: Cluj-Napoca
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: mihai.stan@example.ro
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +40 264 123 456
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4012 8888 8888 1881
+ match: exact
+ - entity: IBAN_CODE
+ text: RO49AAAA1B31007593840000
+ match: exact
+ - entity: IP_ADDRESS
+ text: 185.199.109.153
+ match: exact
--- /dev/null
+cases:
+ - id: sentence_en_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: en
+ text: "Emma Carter in London uses emma.carter@example.com, phone +44 20 7946 0958, card 4111 1111 1111 1111, IBAN GB82WEST12345698765432, and IP 203.0.113.42."
+ expected:
+ - entity: PERSON
+ text: Emma Carter
+ match: contains
+ - entity: LOCATION
+ text: London
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: emma.carter@example.com
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +44 20 7946 0958
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+ - entity: IBAN_CODE
+ text: GB82WEST12345698765432
+ match: exact
+ - entity: IP_ADDRESS
+ text: 203.0.113.42
+ match: exact
+
+ - id: sentence_de_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: de
+ text: "Lena Fischer in Berlin nutzt lena.fischer@example.de, Telefon +49 30 12345678, Karte 4012 8888 8888 1881, IBAN DE89370400440532013000 und IP 198.51.100.17."
+ expected:
+ - entity: PERSON
+ text: Lena Fischer
+ match: contains
+ - entity: LOCATION
+ text: Berlin
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: lena.fischer@example.de
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +49 30 12345678
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4012 8888 8888 1881
+ match: exact
+ - entity: IBAN_CODE
+ text: DE89370400440532013000
+ match: exact
+ - entity: IP_ADDRESS
+ text: 198.51.100.17
+ match: exact
+
+ - id: sentence_es_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: es
+ text: "Lucia Romero en Madrid usa lucia.romero@example.es, telefono +34 91 123 45 67, tarjeta 5555 5555 5555 4444, IBAN ES9121000418450200051332 e IP 192.0.2.88."
+ expected:
+ - entity: PERSON
+ text: Lucia Romero
+ match: contains
+ - entity: LOCATION
+ text: Madrid
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: lucia.romero@example.es
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +34 91 123 45 67
+ match: contains
+ - entity: CREDIT_CARD
+ text: 5555 5555 5555 4444
+ match: exact
+ - entity: IBAN_CODE
+ text: ES9121000418450200051332
+ match: exact
+ - entity: IP_ADDRESS
+ text: 192.0.2.88
+ match: exact
+
+ - id: sentence_fr_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: fr
+ text: "Camille Durand a Paris utilise camille.durand@example.fr, telephone +33 1 42 68 53 00, carte 4242 4242 4242 4242, IBAN FR1420041010050500013M02606 et IP 10.24.8.5."
+ expected:
+ - entity: PERSON
+ text: Camille Durand
+ match: contains
+ - entity: LOCATION
+ text: Paris
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: camille.durand@example.fr
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +33 1 42 68 53 00
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4242 4242 4242 4242
+ match: exact
+ - entity: IBAN_CODE
+ text: FR1420041010050500013M02606
+ match: exact
+ - entity: IP_ADDRESS
+ text: 10.24.8.5
+ match: exact
+
+ - id: sentence_it_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: it
+ text: "Marco Conti a Torino usa marco.conti@example.it, telefono +39 06 1234 5678, carta 4111 1111 1111 1111, IBAN IT60X0542811101000000123456 e IP 172.16.4.20."
+ expected:
+ - entity: PERSON
+ text: Marco Conti
+ match: contains
+ - entity: LOCATION
+ text: Torino
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: marco.conti@example.it
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +39 06 1234 5678
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+ - entity: IBAN_CODE
+ text: IT60X0542811101000000123456
+ match: exact
+ - entity: IP_ADDRESS
+ text: 172.16.4.20
+ match: exact
+
+ - id: sentence_nl_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: nl
+ text: "Daan Jansen in Utrecht gebruikt daan.jansen@example.nl, telefoon +31 20 123 4567, kaart 5555 5555 5555 4444, IBAN NL91ABNA0417164300 en IP 8.8.4.4."
+ expected:
+ - entity: PERSON
+ text: Daan Jansen
+ match: contains
+ - entity: LOCATION
+ text: Utrecht
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: daan.jansen@example.nl
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +31 20 123 4567
+ match: contains
+ - entity: CREDIT_CARD
+ text: 5555 5555 5555 4444
+ match: exact
+ - entity: IBAN_CODE
+ text: NL91ABNA0417164300
+ match: exact
+ - entity: IP_ADDRESS
+ text: 8.8.4.4
+ match: exact
+
+ - id: sentence_pl_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: pl
+ text: "Piotr Nowak w Warszawie używa piotr.nowak@example.pl, telefonu +48 22 123 45 67, karty 4111 1111 1111 1111, IBAN PL61109010140000071219812874 i IP 1.1.1.1."
+ expected:
+ - entity: PERSON
+ text: Piotr Nowak
+ match: contains
+ - entity: LOCATION
+ text: Warszawie
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: piotr.nowak@example.pl
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +48 22 123 45 67
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4111 1111 1111 1111
+ match: exact
+ - entity: IBAN_CODE
+ text: PL61109010140000071219812874
+ match: exact
+ - entity: IP_ADDRESS
+ text: 1.1.1.1
+ match: exact
+
+ - id: sentence_pt_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: pt
+ text: "Ana Pereira em Lisboa usa ana.pereira@example.pt, telefone +351 21 123 4567, cartao 4012 8888 8888 1881, IBAN PT50000201231234567890154 e IP 45.67.89.10."
+ expected:
+ - entity: PERSON
+ text: Ana Pereira
+ match: contains
+ - entity: LOCATION
+ text: Lisboa
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: ana.pereira@example.pt
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +351 21 123 4567
+ match: contains
+ - entity: CREDIT_CARD
+ text: 4012 8888 8888 1881
+ match: exact
+ - entity: IBAN_CODE
+ text: PT50000201231234567890154
+ match: exact
+ - entity: IP_ADDRESS
+ text: 45.67.89.10
+ match: exact
+
+ - id: sentence_ro_all_entities
+ suite: multilingual-sentences
+ category: eval
+ split: test
+ language: ro
+ text: "Elena Ionescu din Bucuresti foloseste elena.ionescu@example.ro, telefon +40 21 123 4567, card 5555 5555 5555 4444, IBAN RO49AAAA1B31007593840000 si IP 185.199.108.153."
+ expected:
+ - entity: PERSON
+ text: Elena Ionescu
+ match: contains
+ - entity: LOCATION
+ text: Bucuresti
+ match: contains
+ - entity: EMAIL_ADDRESS
+ text: elena.ionescu@example.ro
+ match: exact
+ - entity: PHONE_NUMBER
+ text: +40 21 123 4567
+ match: contains
+ - entity: CREDIT_CARD
+ text: 5555 5555 5555 4444
+ match: exact
+ - entity: IBAN_CODE
+ text: RO49AAAA1B31007593840000
+ match: exact
+ - entity: IP_ADDRESS
+ text: 185.199.108.153
+ match: exact
--- /dev/null
+cases:
+ - id: precision_paragraph_release_numbers
+ suite: precision-paragraphs
+ category: precision
+ split: test
+ language: en
+ gate: false
+ entities: [CREDIT_CARD, PHONE_NUMBER, IP_ADDRESS]
+ text: "The release checklist mentions build 4111-1111, ticket 5552671, version 12.4.8, shard 10.24, and rollout window 20:30. None of these strings is a payment card, phone number, or IP address."
+ expected: []
+
+ - id: precision_paragraph_email_lookalikes
+ suite: precision-paragraphs
+ category: precision
+ split: test
+ language: en
+ gate: false
+ entities: [EMAIL_ADDRESS]
+ text: "The onboarding guide says to contact support [at] example [dot] com, use the placeholder user@@example..com only in screenshots, and never paste a real mailbox into templates."
+ expected: []
+
+ - id: precision_paragraph_invalid_accounts
+ suite: precision-paragraphs
+ category: precision
+ split: test
+ language: de
+ gate: false
+ entities: [IBAN_CODE, CREDIT_CARD]
+ text: "Der Testlauf enthaelt die Beispielwerte DE001234, DE89370400440532013001 und 4111 1111 1111 1112. Diese Werte sind absichtlich ungueltig und sollen nicht als echte Konten oder Karten gelten."
+ expected: []
+
+ - id: precision_paragraph_operational_locations
+ suite: precision-paragraphs
+ category: precision
+ split: test
+ language: en
+ gate: false
+ entities: [LOCATION, PERSON]
+ text: "The deployment moved from eu-west-1 to us-east-1, the owner field stayed empty, and the runbook only refers to customer, administrator, reviewer, and approver roles."
+ expected: []
+
+ - id: precision_paragraph_multilingual_roles
+ suite: precision-paragraphs
+ category: precision
+ split: test
+ language: es
+ gate: false
+ entities: [PERSON, LOCATION]
+ text: "El cliente solicita una factura, el administrador revisa la cola y el equipo confirma que no hay nombres de personas ni ciudades reales en este parrafo."
+ expected: []
+
+ - id: precision_paragraph_log_noise
+ suite: precision-paragraphs
+ category: precision
+ split: test
+ language: en
+ gate: false
+ entities: [EMAIL_ADDRESS, PHONE_NUMBER, IP_ADDRESS, LOCATION, PERSON]
+ text: "Log sample: user=anonymous, host=local-dev, route=/api/v1/orders, status=204, latency=123ms, trace=abc123def456, region=staging-west, retry=0. The sample intentionally avoids direct personal data."
+ expected: []
--- /dev/null
+cases:
+ - id: precision_obfuscated_email
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [EMAIL_ADDRESS]
+ text: "Reach support at support [at] example [dot] com."
+ expected: []
+
+ - id: precision_malformed_email
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [EMAIL_ADDRESS]
+ text: "The placeholder alice@@example..com is not a valid mailbox."
+ expected: []
+
+ - id: precision_invalid_credit_card
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [CREDIT_CARD]
+ text: "The internal test number 4111 1111 1111 1112 is invalid."
+ expected: []
+
+ - id: precision_short_iban
+ suite: precision
+ category: precision
+ split: test
+ language: de
+ entities: [IBAN_CODE]
+ text: "Die Zeichenfolge DE001234 ist keine IBAN."
+ expected: []
+
+ - id: precision_invalid_iban_checksum
+ suite: precision
+ category: precision
+ split: test
+ language: de
+ entities: [IBAN_CODE]
+ text: "Die Beispielnummer DE89370400440532013001 hat keine gueltige Pruefsumme."
+ expected: []
+
+ - id: precision_invalid_iban_it_checksum
+ suite: precision
+ category: precision
+ split: test
+ language: it
+ entities: [IBAN_CODE]
+ text: "IBAN con cifra di controllo errata: IT60X0542811101000000123457."
+ expected: []
+
+ - id: precision_version_not_phone
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [PHONE_NUMBER]
+ text: "Release 12.4.8 shipped before ticket 5552671 was closed."
+ expected: []
+
+ - id: precision_order_id_not_phone
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [PHONE_NUMBER]
+ text: "Order 5552671 was closed after release 12.4.8."
+ expected: []
+
+ - id: precision_invalid_ip
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [IP_ADDRESS]
+ text: "The release number 999.12.3.456 is not an IP address."
+ expected: []
+
+ - id: precision_common_words_not_person
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [PERSON]
+ text: "The administrator role approved only read access."
+ expected: []
+
+ - id: precision_fr_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: fr
+ entities: [PERSON, LOCATION]
+ text: "Le client souhaite un rendez-vous la semaine prochaine."
+ expected: []
+
+ - id: precision_es_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: es
+ entities: [PERSON, LOCATION]
+ text: "El cliente solicita la factura del mes corriente."
+ expected: []
+
+ - id: precision_it_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: it
+ entities: [PERSON, LOCATION]
+ text: "Il cliente ha richiesto la fattura entro venerdi."
+ expected: []
+
+ - id: precision_nl_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: nl
+ entities: [PERSON, LOCATION]
+ text: "De klant vraagt om een snelle levering deze week."
+ expected: []
+
+ - id: precision_pt_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: pt
+ entities: [PERSON, LOCATION]
+ text: "O cliente pediu a fatura ate sexta-feira."
+ expected: []
+
+ - id: precision_cloud_region_not_location
+ suite: precision
+ category: precision
+ split: test
+ language: en
+ entities: [LOCATION]
+ text: "The workload moved from eu-west-1 to us-east-1 during the test."
+ expected: []
+
+ - id: precision_pl_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: pl
+ entities: [PERSON, LOCATION]
+ text: "Klient prosi o fakturę przed końcem tygodnia."
+ expected: []
+
+ - id: precision_pl_invalid_credit_card
+ suite: precision
+ category: precision
+ split: test
+ language: pl
+ entities: [CREDIT_CARD]
+ text: "Numer testowy 4111 1111 1111 1112 jest nieprawidłowy."
+ expected: []
+
+ - id: precision_pl_invalid_iban_checksum
+ suite: precision
+ category: precision
+ split: test
+ language: pl
+ entities: [IBAN_CODE]
+ text: "Ciąg PL00109010140000071219812874 ma błędną sumę kontrolną."
+ expected: []
+
+ - id: precision_pl_version_not_phone
+ suite: precision
+ category: precision
+ split: test
+ language: pl
+ entities: [PHONE_NUMBER]
+ text: "Wersja 12.4.8 została wydana przed zgłoszeniem 5552671."
+ expected: []
+
+ - id: precision_pl_cloud_region_not_location
+ suite: precision
+ category: precision
+ split: test
+ language: pl
+ entities: [LOCATION]
+ text: "Zadanie przeniesiono z eu-west-1 do us-east-1 podczas testu."
+ expected: []
+
+ - id: precision_ro_roles_not_person_location
+ suite: precision
+ category: precision
+ split: test
+ language: ro
+ entities: [PERSON, LOCATION]
+ gate: false
+ note: "Instance of a broader Presidio over-detection: non-name tokens (role nouns, emails, IBANs) are labelled PERSON/LOCATION at ~0.85 across several languages, e.g. 'Clientul' here. Tracked as a known false positive; report-only until recognizer precision improves."
+ text: "Clientul cere factura până la sfârșitul săptămânii."
+ expected: []
+
+ - id: precision_ro_invalid_credit_card
+ suite: precision
+ category: precision
+ split: test
+ language: ro
+ entities: [CREDIT_CARD]
+ text: "Numărul de test 4111 1111 1111 1112 este invalid."
+ expected: []
+
+ - id: precision_ro_invalid_iban_checksum
+ suite: precision
+ category: precision
+ split: test
+ language: ro
+ entities: [IBAN_CODE]
+ text: "Șirul RO00AAAA1B31007593840000 are o sumă de control greșită."
+ expected: []
+
+ - id: precision_ro_version_not_phone
+ suite: precision
+ category: precision
+ split: test
+ language: ro
+ entities: [PHONE_NUMBER]
+ text: "Versiunea 12.4.8 a fost lansată înainte de tichetul 5552671."
+ expected: []
+
+ - id: precision_ro_cloud_region_not_location
+ suite: precision
+ category: precision
+ split: test
+ language: ro
+ entities: [LOCATION]
+ text: "Sarcina a fost mutată din eu-west-1 în us-east-1 în timpul testului."
+ expected: []
import { z } from "zod";
+import { CONFIGURED_PII_ENTITIES, SUPPORTED_LANGUAGES } from "./taxonomy";
-// Schema for expected PII entity in test data
-export const ExpectedEntitySchema = z.object({
- type: z.string(),
- text: z.string(),
-});
-
-// Schema for a single test case
-export const TestCaseSchema = z.object({
- id: z.string(),
- text: z.string(),
- language: z.string(),
- expected: z.array(ExpectedEntitySchema),
- description: z.string().optional(),
-});
-
-export type ExpectedEntity = z.infer<typeof ExpectedEntitySchema>;
-export type TestCase = z.infer<typeof TestCaseSchema>;
-
-// Result of running a single test case
-export interface TestResult {
- id: string;
- text: string;
- language: string;
- passed: boolean;
- expected: ExpectedEntity[];
- detected: DetectedEntity[];
- falseNegatives: ExpectedEntity[]; // Expected but not detected
- falsePositives: DetectedEntity[]; // Detected but not expected
- truePositives: ExpectedEntity[]; // Correctly detected
-}
-
-export interface DetectedEntity {
- type: string;
+export const ConfiguredPiiEntitySchema = z.enum(CONFIGURED_PII_ENTITIES);
+export type ConfiguredPiiEntity = z.infer<typeof ConfiguredPiiEntitySchema>;
+
+export const SupportedLanguageSchema = z.enum(SUPPORTED_LANGUAGES);
+export type SupportedLanguage = z.infer<typeof SupportedLanguageSchema>;
+
+export const MatchModeSchema = z.enum(["exact", "contains", "overlap"]);
+export type MatchMode = z.infer<typeof MatchModeSchema>;
+
+export const CategorySchema = z.enum(["core", "precision", "eval", "hard"]);
+export type BenchmarkCategory = z.infer<typeof CategorySchema>;
+
+export const SplitSchema = z.enum(["dev", "test"]);
+export type BenchmarkSplit = z.infer<typeof SplitSchema>;
+
+export const ExpectedSpanSchema = z
+ .object({
+ entity: ConfiguredPiiEntitySchema,
+ text: z.string().min(1),
+ match: MatchModeSchema.default("contains"),
+ aliases: z.array(z.string().min(1)).default([]),
+ })
+ .strict();
+export type ExpectedSpan = z.infer<typeof ExpectedSpanSchema>;
+
+export const BenchmarkCaseSchema = z
+ .object({
+ id: z.string().min(1),
+ suite: z.string().min(1),
+ category: CategorySchema.default("core"),
+ split: SplitSchema.default("test"),
+ language: SupportedLanguageSchema,
+ text: z.string().min(1),
+ entities: z.array(ConfiguredPiiEntitySchema).optional(),
+ gate: z.boolean().optional(),
+ note: z.string().optional(),
+ expected: z.array(ExpectedSpanSchema).default([]),
+ })
+ .strict();
+export type BenchmarkCase = z.infer<typeof BenchmarkCaseSchema>;
+
+export const BenchmarkFileSchema = z
+ .object({
+ cases: z.array(BenchmarkCaseSchema),
+ })
+ .strict();
+
+export type Detection = {
+ entity: string;
text: string;
start: number;
end: number;
score: number;
-}
-
-// Aggregated metrics
-export interface AccuracyMetrics {
- total: number;
- passed: number;
- failed: number;
- precision: number; // TP / (TP + FP)
- recall: number; // TP / (TP + FN)
- f1: number; // 2 * (P * R) / (P + R)
- truePositives: number;
- falsePositives: number;
- falseNegatives: number;
-}
+};
+
+export type TestResult = {
+ case: BenchmarkCase;
+ passed: boolean;
+ gating: boolean;
+ detections: Detection[];
+ matched: Array<{ expected: ExpectedSpan; detection: Detection }>;
+ missing: ExpectedSpan[];
+ unexpected: Detection[];
+ error?: string;
+};
},
"files": {
"ignoreUnknown": false,
- "includes": ["src/**/*.ts"]
+ "includes": ["src/**/*.ts", "benchmarks/pii-accuracy/*.ts"]
},
"formatter": {
"enabled": true,
"build": "bun build src/index.ts --outdir dist --target bun --external lightningcss",
"test": "bun test",
"typecheck": "tsc --noEmit",
- "lint": "biome lint src",
- "format": "biome format src --write",
- "check": "biome check src",
+ "typecheck:benchmarks": "tsc --noEmit --project tsconfig.benchmarks.json",
+ "lint": "biome lint src benchmarks/pii-accuracy/*.ts",
+ "format": "biome format src benchmarks/pii-accuracy/*.ts --write",
+ "check": "biome check src benchmarks/pii-accuracy/*.ts",
"benchmark:accuracy": "bun run benchmarks/pii-accuracy/run.ts"
},
"dependencies": {
--- /dev/null
+{
+ "extends": "./tsconfig.json",
+ "compilerOptions": {
+ "rootDir": "."
+ },
+ "include": ["benchmarks/**/*.ts"],
+ "exclude": ["node_modules", "dist"]
+}
git.99rst.org / sgasser-llm-shield.git / commitdiff