# Text patterns that are never masked (protects against false positives)
# whitelist:
- # - "Company Name Inc."
+ # - pattern: "Company Name Inc."
+ # - pattern: 'TEST-\d+'
+ # regex: true
+
+ # Text or regex patterns that are always masked, even if the detector misses them
+ # denylist:
+ # - pattern: "ProjectX"
+ # type: PROJECT_NAME
+ # - pattern: 'CUST-\d{6}'
+ # type: CUSTOMER_ID
+ # regex: true
# PII Detection settings (detector service, /analyze contract)
pii_detection:
masking:
show_markers: false
marker_text: "[protected]"
+ whitelist:
+ - pattern: "Company Name Inc."
+ - pattern: 'TEST-\d+'
+ regex: true
+ denylist:
+ - pattern: "ProjectX"
+ type: PROJECT_NAME
+ - pattern: 'CUST-\d{6}'
+ type: CUSTOMER_ID
+ regex: true
```
| Option | Default | Description |
|--------|---------|-------------|
| `show_markers` | `false` | Add visual markers around unmasked values |
| `marker_text` | `[protected]` | Marker text if enabled |
+| `whitelist` | `[]` | Text patterns that are never masked; set `regex: true` for regex patterns |
+| `denylist` | `[]` | Text patterns that are always masked with the configured `type`; set `regex: true` for regex patterns |
## Response Headers
```yaml
masking:
whitelist:
- - "Acme Corp"
- - "Product XYZ"
+ - pattern: "Acme Corp"
+ - pattern: "Product XYZ"
+ - pattern: 'TEST-\d+'
+ regex: true
```
-Patterns match bidirectionally - detected text containing a whitelist entry (or vice versa) is excluded.
+A literal entry is matched as a substring: a detected value is left unmasked if it contains the entry, or the entry contains it. Set `regex: true` for JavaScript regex syntax — a regex entry must match the **entire** detected value, so `\d{4}` won't unmask a longer number that merely contains four digits.
+
+## Denylist
+
+Force specific text or regex patterns to be masked, even when the detector does not report them or PII detection is disabled. Each entry needs a `type`, which is used for the placeholder name.
+
+```yaml
+masking:
+ denylist:
+ - pattern: "ProjectX"
+ type: PROJECT_NAME
+ - pattern: 'CUST-\d{6}'
+ type: CUSTOMER_ID
+ regex: true
+```
+
+Patterns are matched literally by default. Set `regex: true` for JavaScript regex syntax — use single quotes in YAML when the pattern contains backslashes. A regex pattern must not match the empty string (use `\d+`, not `\d*`); empty-matching patterns are rejected at startup because they would mask nothing.
## Scan Roles
cleanupConfig(path);
}
});
+
+ test("accepts masking whitelist and denylist patterns", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+masking:
+ whitelist:
+ - "Acme Corp"
+ - pattern: 'TEST-\\d+'
+ regex: true
+ denylist:
+ - pattern: "ProjectX"
+ type: PROJECT_NAME
+ - pattern: 'CUST-\\d{6}'
+ type: CUSTOMER_ID
+ regex: true
+pii_detection:
+ detector_url: http://localhost:5002
+`);
+
+ try {
+ const config = loadConfig(path);
+
+ expect(config.masking.whitelist).toEqual([
+ { pattern: "You are Claude Code, Anthropic's official CLI for Claude.", regex: false },
+ { pattern: "Acme Corp", regex: false },
+ { pattern: "TEST-\\d+", regex: true },
+ ]);
+ expect(config.masking.denylist).toEqual([
+ { pattern: "ProjectX", type: "PROJECT_NAME", regex: false },
+ { pattern: "CUST-\\d{6}", type: "CUSTOMER_ID", regex: true },
+ ]);
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
+ test("rejects invalid masking whitelist regex patterns", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+masking:
+ whitelist:
+ - pattern: "[Acme"
+ regex: true
+pii_detection:
+ detector_url: http://localhost:5002
+`);
+
+ try {
+ expect(() => loadConfig(path)).toThrow("Invalid configuration");
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
+ test("rejects invalid masking denylist regex patterns", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+masking:
+ denylist:
+ - pattern: "[ProjectX"
+ type: PROJECT_NAME
+ regex: true
+pii_detection:
+ detector_url: http://localhost:5002
+`);
+
+ try {
+ expect(() => loadConfig(path)).toThrow("Invalid configuration");
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
+ test("rejects denylist regex patterns that match the empty string", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+masking:
+ denylist:
+ - pattern: 'x*'
+ type: NUM
+ regex: true
+pii_detection:
+ detector_url: http://localhost:5002
+`);
+
+ try {
+ expect(() => loadConfig(path)).toThrow("Invalid configuration");
+ } finally {
+ cleanupConfig(path);
+ }
+ });
});
base_url: z.string().url().default("https://chatgpt.com/backend-api/codex"),
});
-const DEFAULT_WHITELIST = ["You are Claude Code, Anthropic's official CLI for Claude."];
+const DEFAULT_WHITELIST = [
+ { pattern: "You are Claude Code, Anthropic's official CLI for Claude.", regex: false },
+];
+
+function validateRegexPattern(
+ pattern: string,
+ regex: boolean,
+ ctx: z.RefinementCtx,
+ message: string,
+): void {
+ if (!regex) return;
+
+ let compiled: RegExp;
+ try {
+ compiled = new RegExp(pattern, "g");
+ } catch {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ path: ["pattern"],
+ message,
+ });
+ return;
+ }
+
+ // Reject patterns that match the empty string: zero-length matches are skipped, so they mask nothing.
+ if (compiled.test("")) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ path: ["pattern"],
+ message: `${message} (must not match the empty string)`,
+ });
+ }
+}
+
+const WhitelistPatternSchema = z.union([
+ z
+ .string()
+ .min(1)
+ .transform((pattern) => ({ pattern, regex: false })),
+ z
+ .object({
+ pattern: z.string().min(1),
+ regex: z.boolean().default(false),
+ })
+ .superRefine((entry, ctx) => {
+ validateRegexPattern(entry.pattern, entry.regex, ctx, "Invalid whitelist regex pattern");
+ }),
+]);
+
+const DenylistPatternSchema = z
+ .object({
+ pattern: z.string().min(1),
+ type: z.string().min(1),
+ regex: z.boolean().default(false),
+ })
+ .superRefine((entry, ctx) => {
+ validateRegexPattern(entry.pattern, entry.regex, ctx, "Invalid denylist regex pattern");
+ });
const MaskingSchema = z.object({
show_markers: z.boolean().default(false),
marker_text: z.string().default("[protected]"),
whitelist: z
- .array(z.string())
+ .array(WhitelistPatternSchema)
.default([])
.transform((arr) => [...DEFAULT_WHITELIST, ...arr]),
+ denylist: z.array(DenylistPatternSchema).default([]),
});
const LanguageEnum = z.enum(SUPPORTED_LANGUAGES);
export type CodexProviderConfig = z.infer<typeof CodexProviderSchema>;
export type LocalProviderConfig = z.infer<typeof LocalProviderSchema>;
export type MaskingConfig = z.infer<typeof MaskingSchema>;
+export type WhitelistPattern = z.infer<typeof WhitelistPatternSchema>;
+export type DenylistPattern = z.infer<typeof DenylistPatternSchema>;
export type SecretsDetectionConfig = z.infer<typeof SecretsDetectionSchema>;
export type ServerConfig = z.infer<typeof ServerSchema>;
entity_type: string;
}
-function overlaps(a: Span, b: Span): boolean {
+export function overlaps(a: Span, b: Span): boolean {
return a.start < b.end && b.start < a.end;
}
expect(findPartialPlaceholderStart(text)).toBe(6);
});
- test("handles text ending with single bracket", () => {
- // Single [ is not a placeholder start, so should return -1
- expect(findPartialPlaceholderStart("Hello [")).toBe(-1);
+ test("buffers a trailing single bracket (first half of a split [[)", () => {
+ expect(findPartialPlaceholderStart("Hello [")).toBe(6);
+ });
+
+ test("buffers a trailing single bracket after a complete placeholder", () => {
+ expect(findPartialPlaceholderStart("[[PERSON_1]] [")).toBe(13);
+ });
+
+ test("buffers an incomplete placeholder whose closing ]] is split", () => {
+ expect(findPartialPlaceholderStart("Hi [[PERSON_1]")).toBe(3);
});
});
* Returns the position where it's safe to split, or -1 if entire string is safe
*/
export function findPartialPlaceholderStart(text: string): number {
- const placeholderStart = text.lastIndexOf(PLACEHOLDER_DELIMITERS.start);
+ const { start, end } = PLACEHOLDER_DELIMITERS;
+ const placeholderStart = text.lastIndexOf(start);
- if (placeholderStart === -1) {
- return -1; // No potential placeholder, entire string is safe
+ // An opened "[[" with no closing "]]" after it is an incomplete placeholder.
+ if (placeholderStart !== -1 && !text.slice(placeholderStart).includes(end)) {
+ return placeholderStart;
}
- // Check if there's a complete placeholder after the last [[
- const afterStart = text.slice(placeholderStart);
- const hasCompletePlaceholder = afterStart.includes(PLACEHOLDER_DELIMITERS.end);
-
- if (hasCompletePlaceholder) {
- return -1; // Placeholder is complete, entire string is safe
+ // A trailing "[" may be the first half of a "[[" that completes in the next chunk.
+ if (text.endsWith(start.slice(0, 1))) {
+ return text.length - 1;
}
- return placeholderStart; // Return position where partial placeholder starts
+ return -1; // Entire string is safe to emit
}
import { afterEach, describe, expect, mock, test } from "bun:test";
+import { getConfig } from "../config";
import { openaiExtractor } from "../masking/extractors/openai";
import type { OpenAIMessage, OpenAIRequest } from "../providers/openai/types";
-import { filterWhitelistedEntities, PIIDetector } from "./detect";
+import {
+ filterWhitelistedEntities,
+ findDenylistedEntities,
+ mergeDenylistEntities,
+ PIIDetector,
+} from "./detect";
const originalFetch = globalThis.fetch;
// First message (empty string) has no entities
expect(result.spanEntities[0]).toHaveLength(0);
});
+
+ test("adds denylist entities when detector returns none", async () => {
+ const config = getConfig();
+ const previousDenylist = config.masking.denylist;
+ config.masking.denylist = [{ pattern: "ProjectX", type: "PROJECT_NAME", regex: false }];
+ mockDetector({});
+
+ try {
+ const detector = new PIIDetector();
+ const request = createRequest([{ role: "user", content: "Launch ProjectX" }]);
+
+ const result = await detector.analyzeRequest(request, openaiExtractor);
+
+ expect(result.hasPII).toBe(true);
+ expect(result.spanEntities[0]).toEqual([
+ { entity_type: "PROJECT_NAME", start: 7, end: 15, score: 1 },
+ ]);
+ } finally {
+ config.masking.denylist = previousDenylist;
+ }
+ });
+
+ test("applies denylist when PII detection is disabled", async () => {
+ const config = getConfig();
+ const previousEnabled = config.pii_detection.enabled;
+ const previousDenylist = config.masking.denylist;
+ config.pii_detection.enabled = false;
+ config.masking.denylist = [{ pattern: "ProjectX", type: "PROJECT_NAME", regex: false }];
+ const fetchMock = mock(async () => {
+ throw new Error("Detector should not be called");
+ });
+ globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+ try {
+ const detector = new PIIDetector();
+ const request = createRequest([{ role: "user", content: "Launch ProjectX" }]);
+
+ const result = await detector.analyzeRequest(request, openaiExtractor);
+
+ expect(result.hasPII).toBe(true);
+ expect(fetchMock).not.toHaveBeenCalled();
+ } finally {
+ config.pii_detection.enabled = previousEnabled;
+ config.masking.denylist = previousDenylist;
+ }
+ });
+
+ test("does not let a denylist substring shrink an overlapping detector entity", async () => {
+ const config = getConfig();
+ const previousDenylist = config.masking.denylist;
+ config.masking.denylist = [{ pattern: "ProjectX", type: "PROJECT_NAME", regex: false }];
+ mockDetector({
+ "ProjectX@corp.com": [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 23, score: 0.95 }],
+ });
+
+ try {
+ const detector = new PIIDetector();
+ const request = createRequest([{ role: "user", content: "Email ProjectX@corp.com" }]);
+
+ const result = await detector.analyzeRequest(request, openaiExtractor);
+
+ expect(result.spanEntities[0]).toEqual([
+ { entity_type: "EMAIL_ADDRESS", start: 6, end: 23, score: 1 },
+ ]);
+ } finally {
+ config.masking.denylist = previousDenylist;
+ }
+ });
+
+ test("skips detection entirely when disabled and no denylist is configured", async () => {
+ const config = getConfig();
+ const previousEnabled = config.pii_detection.enabled;
+ const previousDenylist = config.masking.denylist;
+ config.pii_detection.enabled = false;
+ config.masking.denylist = [];
+ const fetchMock = mock(async () => {
+ throw new Error("Detector should not be called");
+ });
+ globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+ try {
+ const detector = new PIIDetector();
+ const request = createRequest([{ role: "user", content: "Launch ProjectX" }]);
+
+ const result = await detector.analyzeRequest(request, openaiExtractor);
+
+ expect(result.hasPII).toBe(false);
+ expect(result.spanEntities).toEqual([]);
+ expect(fetchMock).not.toHaveBeenCalled();
+ } finally {
+ config.pii_detection.enabled = previousEnabled;
+ config.masking.denylist = previousDenylist;
+ }
+ });
});
describe("detectPII", () => {
test("filters entities matching whitelist pattern", () => {
const text = "You are Claude Code, Anthropic's official CLI for Claude.";
const entities = [{ entity_type: "PERSON", start: 8, end: 14, score: 0.9 }];
- const whitelist = ["You are Claude Code, Anthropic's official CLI for Claude."];
+ const whitelist = [
+ { pattern: "You are Claude Code, Anthropic's official CLI for Claude.", regex: false },
+ ];
const result = filterWhitelistedEntities(text, entities, whitelist);
{ entity_type: "PERSON", start: 8, end: 16, score: 0.9 },
{ entity_type: "EMAIL_ADDRESS", start: 20, end: 36, score: 0.95 },
];
- const whitelist = ["Claude"];
+ const whitelist = [{ pattern: "Claude", regex: false }];
const result = filterWhitelistedEntities(text, entities, whitelist);
test("filters when entity text is contained in whitelist pattern", () => {
const text = "Hello Claude, how are you?";
const entities = [{ entity_type: "PERSON", start: 6, end: 12, score: 0.85 }];
- const whitelist = ["You are Claude Code"];
+ const whitelist = [{ pattern: "You are Claude Code", regex: false }];
const result = filterWhitelistedEntities(text, entities, whitelist);
expect(result).toHaveLength(2);
});
+
+ test("filters entities matching regex whitelist pattern", () => {
+ const text = "Reference TEST-1234 is public";
+ const entities = [{ entity_type: "CUSTOMER_ID", start: 10, end: 19, score: 0.9 }];
+ const whitelist = [{ pattern: "TEST-\\d+", regex: true }];
+
+ const result = filterWhitelistedEntities(text, entities, whitelist);
+
+ expect(result).toHaveLength(0);
+ });
+
+ test("does not filter when a regex whitelist only partially matches the entity", () => {
+ const text = "card 1234567890123456 end";
+ const entities = [{ entity_type: "CREDIT_CARD", start: 5, end: 21, score: 0.99 }];
+ const whitelist = [{ pattern: "\\d{4}", regex: true }];
+
+ const result = filterWhitelistedEntities(text, entities, whitelist);
+
+ expect(result).toHaveLength(1);
+ });
+ });
+
+ describe("findDenylistedEntities", () => {
+ test("finds literal denylist patterns", () => {
+ const result = findDenylistedEntities("ProjectX uses ProjectX-API", [
+ { pattern: "ProjectX", type: "PROJECT_NAME", regex: false },
+ ]);
+
+ expect(result).toEqual([
+ { entity_type: "PROJECT_NAME", start: 0, end: 8, score: 1 },
+ { entity_type: "PROJECT_NAME", start: 14, end: 22, score: 1 },
+ ]);
+ });
+
+ test("finds regex denylist patterns", () => {
+ const result = findDenylistedEntities("Customers CUST-123456 and CUST-654321", [
+ { pattern: "CUST-\\d{6}", type: "CUSTOMER_ID", regex: true },
+ ]);
+
+ expect(result).toEqual([
+ { entity_type: "CUSTOMER_ID", start: 10, end: 21, score: 1 },
+ { entity_type: "CUSTOMER_ID", start: 26, end: 37, score: 1 },
+ ]);
+ });
+
+ test("matches regex syntax literally unless regex is enabled", () => {
+ const result = findDenylistedEntities("Internal [ProjectX", [
+ { pattern: "[ProjectX", type: "PROJECT_NAME", regex: false },
+ ]);
+
+ expect(result).toEqual([{ entity_type: "PROJECT_NAME", start: 9, end: 18, score: 1 }]);
+ });
+
+ test("matches regex patterns containing escaped non-syntax characters", () => {
+ const result = findDenylistedEntities("Customer CUST-123456 onboarded", [
+ { pattern: "CUST\\-\\d{6}", type: "CUSTOMER_ID", regex: true },
+ ]);
+
+ expect(result).toEqual([{ entity_type: "CUSTOMER_ID", start: 9, end: 20, score: 1 }]);
+ });
+
+ test("ignores matches that fall inside a known placeholder", () => {
+ const result = findDenylistedEntities(
+ "conn [[CONNECTION_STRING_1]] ProjectX",
+ [
+ { pattern: "\\d+", type: "NUM", regex: true },
+ { pattern: "ProjectX", type: "PROJECT_NAME", regex: false },
+ ],
+ ["[[CONNECTION_STRING_1]]"],
+ );
+
+ expect(result).toEqual([{ entity_type: "PROJECT_NAME", start: 29, end: 37, score: 1 }]);
+ });
+ });
+
+ describe("mergeDenylistEntities", () => {
+ test("returns detector entities unchanged when there is no denylist", () => {
+ const detected = [{ entity_type: "EMAIL_ADDRESS", start: 0, end: 16, score: 0.9 }];
+
+ expect(mergeDenylistEntities(detected, [])).toBe(detected);
+ });
+
+ test("adds non-overlapping denylist matches", () => {
+ const detected = [{ entity_type: "EMAIL_ADDRESS", start: 0, end: 5, score: 0.9 }];
+ const denylisted = [{ entity_type: "PROJECT_NAME", start: 10, end: 18, score: 1 }];
+
+ expect(mergeDenylistEntities(detected, denylisted)).toEqual([
+ { entity_type: "EMAIL_ADDRESS", start: 0, end: 5, score: 0.9 },
+ { entity_type: "PROJECT_NAME", start: 10, end: 18, score: 1 },
+ ]);
+ });
+
+ test("keeps the full detector span when a denylist match is contained within it", () => {
+ const detected = [{ entity_type: "EMAIL_ADDRESS", start: 0, end: 17, score: 0.95 }];
+ const denylisted = [{ entity_type: "PROJECT_NAME", start: 0, end: 8, score: 1 }];
+
+ expect(mergeDenylistEntities(detected, denylisted)).toEqual([
+ { entity_type: "EMAIL_ADDRESS", start: 0, end: 17, score: 1 },
+ ]);
+ });
+
+ test("unions a partial overlap so no covered region is left unmasked", () => {
+ const detected = [{ entity_type: "PERSON", start: 0, end: 10, score: 0.9 }];
+ const denylisted = [{ entity_type: "PROJECT_NAME", start: 5, end: 15, score: 1 }];
+
+ expect(mergeDenylistEntities(detected, denylisted)).toEqual([
+ { entity_type: "PERSON", start: 0, end: 15, score: 1 },
+ ]);
+ });
+
+ test("returns denylist-only matches when the detector found nothing", () => {
+ const denylisted = [{ entity_type: "PROJECT_NAME", start: 0, end: 8, score: 1 }];
+
+ expect(mergeDenylistEntities([], denylisted)).toEqual([
+ { entity_type: "PROJECT_NAME", start: 0, end: 8, score: 1 },
+ ]);
+ });
});
});
-import { getConfig } from "../config";
+import { type DenylistPattern, getConfig, type WhitelistPattern } from "../config";
import { HEALTH_CHECK_TIMEOUT_MS } from "../constants/timeouts";
+import { overlaps, resolveConflicts } from "../masking/conflict-resolver";
import type { RequestExtractor } from "../masking/types";
import { getLanguageDetector, type SupportedLanguage } from "../services/language-detector";
score: number;
}
+// Denylist matches are exact (operator-configured), so they carry full confidence.
+const DENYLIST_MATCH_SCORE = 1;
+
+function findLiteralMatches(text: string, pattern: string, type: string): PIIEntity[] {
+ const matches: PIIEntity[] = [];
+ let index = text.indexOf(pattern);
+
+ while (index !== -1) {
+ matches.push({
+ entity_type: type,
+ start: index,
+ end: index + pattern.length,
+ score: DENYLIST_MATCH_SCORE,
+ });
+ index = text.indexOf(pattern, index + pattern.length);
+ }
+
+ return matches;
+}
+
+function findRegexMatches(text: string, pattern: string, type: string): PIIEntity[] {
+ const regex = new RegExp(pattern, "g");
+ const matches: PIIEntity[] = [];
+
+ for (const match of text.matchAll(regex)) {
+ if (match.index === undefined || match[0].length === 0) continue;
+ matches.push({
+ entity_type: type,
+ start: match.index,
+ end: match.index + match[0].length,
+ score: DENYLIST_MATCH_SCORE,
+ });
+ }
+
+ return matches;
+}
+
+function placeholderSpans(
+ text: string,
+ placeholders: readonly string[],
+): Array<{ start: number; end: number }> {
+ const spans: Array<{ start: number; end: number }> = [];
+ for (const placeholder of placeholders) {
+ let index = text.indexOf(placeholder);
+ while (index !== -1) {
+ spans.push({ start: index, end: index + placeholder.length });
+ index = text.indexOf(placeholder, index + placeholder.length);
+ }
+ }
+ return spans;
+}
+
+export function findDenylistedEntities(
+ text: string,
+ denylist: DenylistPattern[],
+ knownPlaceholders: readonly string[] = [],
+): PIIEntity[] {
+ if (denylist.length === 0 || !text) return [];
+
+ const matches = denylist.flatMap(({ pattern, type, regex }) =>
+ regex ? findRegexMatches(text, pattern, type) : findLiteralMatches(text, pattern, type),
+ );
+ if (matches.length === 0 || knownPlaceholders.length === 0) return matches;
+
+ // Drop matches inside an already-masked placeholder; re-masking its internals would corrupt the earlier mask.
+ const masked = placeholderSpans(text, knownPlaceholders);
+ if (masked.length === 0) return matches;
+ return matches.filter((m) => !masked.some((p) => overlaps(m, p)));
+}
+
+// Additive merge: a denylist match extends coverage but never shrinks an overlapping detector span; overlaps become their union and the detector type wins.
+export function mergeDenylistEntities(detected: PIIEntity[], denylisted: PIIEntity[]): PIIEntity[] {
+ if (denylisted.length === 0) return detected;
+
+ const resolvedDetector = resolveConflicts(detected);
+ const tagged = [
+ ...resolvedDetector.map((e) => ({ e, forced: false })),
+ ...denylisted.map((e) => ({ e, forced: true })),
+ ].sort((a, b) => a.e.start - b.e.start);
+
+ const result: { e: PIIEntity; forced: boolean }[] = [];
+ for (const item of tagged) {
+ const last = result[result.length - 1];
+ if (last && overlaps(item.e, last.e)) {
+ last.e = {
+ entity_type: last.forced && !item.forced ? item.e.entity_type : last.e.entity_type,
+ start: last.e.start,
+ end: Math.max(last.e.end, item.e.end),
+ score: Math.max(last.e.score, item.e.score),
+ };
+ last.forced = last.forced && item.forced;
+ } else {
+ result.push({ e: { ...item.e }, forced: item.forced });
+ }
+ }
+
+ return result.map((r) => r.e);
+}
+
export function filterWhitelistedEntities(
text: string,
entities: PIIEntity[],
- whitelist: string[],
+ whitelist: WhitelistPattern[],
): PIIEntity[] {
if (whitelist.length === 0) return entities;
return entities.filter((entity) => {
const detectedText = text.slice(entity.start, entity.end);
- return !whitelist.some(
- (pattern) => pattern.includes(detectedText) || detectedText.includes(pattern),
- );
+ return !whitelist.some(({ pattern, regex }) => {
+ if (regex) {
+ // Anchor to the whole entity so a partial match can't un-mask a larger detected span.
+ return new RegExp(`^(?:${pattern})$`).test(detectedText);
+ }
+ return pattern.includes(detectedText) || detectedText.includes(pattern);
+ });
});
}
async analyzeRequest<TRequest, TResponse>(
request: TRequest,
extractor: RequestExtractor<TRequest, TResponse>,
+ knownPlaceholders: readonly string[] = [],
): Promise<PIIDetectionResult> {
const startTime = Date.now();
const config = getConfig();
+ // Pure pass-through: detection off and no denylist, so skip extraction and language detection.
+ if (!config.pii_detection.enabled && config.masking.denylist.length === 0) {
+ return {
+ hasPII: false,
+ spanEntities: [],
+ allEntities: [],
+ scanTimeMs: 0,
+ language: config.pii_detection.fallback_language,
+ languageFallback: true,
+ };
+ }
+
// Extract all text spans from request
const spans = extractor.extractTexts(request);
? new Set(config.pii_detection.scan_roles)
: null;
const whitelist = config.masking.whitelist;
+ const denylist = config.masking.denylist;
const spanEntities: PIIEntity[][] = await Promise.all(
spans.map(async (span) => {
+ if (!span.text) return [];
+ const denylistedEntities = findDenylistedEntities(span.text, denylist, knownPlaceholders);
+
if (scanRoles && span.role && !scanRoles.has(span.role)) {
- return [];
+ return mergeDenylistEntities([], denylistedEntities);
}
- if (!span.text) return [];
- const entities = await this.detectPII(span.text, langResult.language);
- return filterWhitelistedEntities(span.text, entities, whitelist);
+
+ const detectedEntities = config.pii_detection.enabled
+ ? await this.detectPII(span.text, langResult.language)
+ : [];
+ const filteredEntities = filterWhitelistedEntities(span.text, detectedEntities, whitelist);
+ return mergeDenylistEntities(filteredEntities, denylistedEntities);
}),
);
show_markers: false,
marker_text: "[protected]",
whitelist: [],
+ denylist: [],
};
const configWithMarkers: MaskingConfig = {
show_markers: true,
marker_text: "[protected]",
whitelist: [],
+ denylist: [],
};
/** Helper to create a minimal request from messages */
show_markers: false,
marker_text: "[protected]",
whitelist: [],
+ denylist: [],
};
/**
show_markers: false,
marker_text: "[protected]",
whitelist: [],
+ denylist: [],
};
/**
expect(result).toContain("a@b.com");
});
+ test("buffers a placeholder split between the two opening brackets", async () => {
+ const context = createMaskingContext();
+ context.mapping["[[EMAIL_ADDRESS_1]]"] = "a@b.com";
+
+ const chunks = [
+ `data: {"choices":[{"delta":{"content":"Hello ["}}]}\n\n`,
+ `data: {"choices":[{"delta":{"content":"[EMAIL_ADDRESS_1]] world"}}]}\n\n`,
+ ];
+ const source = createSSEStream(chunks);
+
+ const unmaskedStream = createUnmaskingStream(source, context, defaultConfig);
+ const result = await consumeStream(unmaskedStream);
+
+ expect(result).toContain("a@b.com");
+ expect(result).not.toContain("[[EMAIL_ADDRESS_1]]");
+ });
+
+ test("buffers a placeholder split between the two closing brackets", async () => {
+ const context = createMaskingContext();
+ context.mapping["[[EMAIL_ADDRESS_1]]"] = "a@b.com";
+
+ const chunks = [
+ `data: {"choices":[{"delta":{"content":"Hello [[EMAIL_ADDRESS_1]"}}]}\n\n`,
+ `data: {"choices":[{"delta":{"content":"] world"}}]}\n\n`,
+ ];
+ const source = createSSEStream(chunks);
+
+ const unmaskedStream = createUnmaskingStream(source, context, defaultConfig);
+ const result = await consumeStream(unmaskedStream);
+
+ expect(result).toContain("a@b.com");
+ expect(result).not.toContain("[[EMAIL_ADDRESS_1]");
+ });
+
test("flushes remaining buffer on stream end", async () => {
const context = createMaskingContext();
context.mapping["[[EMAIL_ADDRESS_1]]"] = "test@test.com";
import { unmaskSecretsResponse } from "../secrets/mask";
import { logRequest } from "../services/logger";
import { detectPII, maskPII, type PIIDetectResult } from "../services/pii";
-import { processSecretsRequest, type SecretsProcessResult } from "../services/secrets";
+import {
+ processSecretsRequest,
+ type SecretsProcessResult,
+ secretPlaceholders,
+} from "../services/secrets";
import {
createLogData,
errorFormats,
request = secretsResult.request;
}
- // Step 2: Detect PII (skip if disabled)
+ // Step 2: Detect PII and configured denylist terms
let piiResult: PIIDetectResult;
- if (!config.pii_detection.enabled) {
- piiResult = {
- detection: {
- hasPII: false,
- spanEntities: [],
- allEntities: [],
- scanTimeMs: 0,
- language: "en",
- languageFallback: false,
- },
- hasPII: false,
- };
- } else {
- try {
- piiResult = await detectPII(request, anthropicExtractor);
- } catch (error) {
- console.error("PII detection error:", error);
- return respondDetectionError(c, request, secretsResult, startTime);
- }
+ try {
+ piiResult = await detectPII(request, anthropicExtractor, secretPlaceholders(secretsResult));
+ } catch (error) {
+ console.error("PII detection error:", error);
+ return respondDetectionError(c, request, secretsResult, startTime);
}
// Step 3: Route mode - send to local if PII or secrets detected
import { describe, expect, mock, test } from "bun:test";
import { Hono } from "hono";
-import { filterWhitelistedEntities, type PIIEntity } from "../pii/detect";
+import {
+ filterWhitelistedEntities,
+ findDenylistedEntities,
+ mergeDenylistEntities,
+ type PIIEntity,
+} from "../pii/detect";
// Mock the PII detector to avoid needing the detector running
const mockDetectPII = mock<(text: string, language: string) => Promise<PIIEntity[]>>(() =>
healthCheck: mock(() => Promise.resolve(true)),
}),
filterWhitelistedEntities,
+ findDenylistedEntities,
+ mergeDenylistEntities,
}));
// Mock the logger to avoid database operations
const baseConfig = realConfig.getConfig();
const testConfig = {
...baseConfig,
+ // Pin detection on so the detector mock is consumed regardless of config.yaml.
+ pii_detection: { ...baseConfig.pii_detection, enabled: true },
secrets_detection: {
...baseConfig.secrets_detection,
enabled: true,
expect(body.entities[0].type).toBe("EMAIL_ADDRESS");
});
+ test("masks configured denylist patterns", async () => {
+ const previousDenylist = testConfig.masking.denylist;
+ testConfig.masking.denylist = [
+ { pattern: "ProjectX", type: "PROJECT_NAME", regex: false },
+ { pattern: "CUST-\\d{6}", type: "CUSTOMER_ID", regex: true },
+ ];
+ mockDetectPII.mockResolvedValueOnce([]);
+
+ try {
+ const res = await app.request("/api/mask", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ text: "ProjectX customer CUST-123456" }),
+ });
+
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as {
+ masked: string;
+ context: Record<string, string>;
+ entities: { type: string; placeholder: string }[];
+ };
+ expect(body.masked).toBe("[[PROJECT_NAME_1]] customer [[CUSTOMER_ID_1]]");
+ expect(body.context["[[PROJECT_NAME_1]]"]).toBe("ProjectX");
+ expect(body.context["[[CUSTOMER_ID_1]]"]).toBe("CUST-123456");
+ expect(body.entities).toEqual([
+ { type: "PROJECT_NAME", placeholder: "[[PROJECT_NAME_1]]" },
+ { type: "CUSTOMER_ID", placeholder: "[[CUSTOMER_ID_1]]" },
+ ]);
+ } finally {
+ testConfig.masking.denylist = previousDenylist;
+ }
+ });
+
+ test("denylist match inside a detected entity does not leak the rest of it", async () => {
+ const previousDenylist = testConfig.masking.denylist;
+ testConfig.masking.denylist = [{ pattern: "ProjectX", type: "PROJECT_NAME", regex: false }];
+ mockDetectPII.mockResolvedValueOnce([
+ { entity_type: "EMAIL_ADDRESS", start: 6, end: 23, score: 0.95 },
+ ]);
+
+ try {
+ const res = await app.request("/api/mask", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ text: "Email ProjectX@corp.com" }),
+ });
+
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as {
+ masked: string;
+ context: Record<string, string>;
+ entities: { type: string; placeholder: string }[];
+ };
+ expect(body.masked).toBe("Email [[EMAIL_ADDRESS_1]]");
+ expect(body.context["[[EMAIL_ADDRESS_1]]"]).toBe("ProjectX@corp.com");
+ expect(body.entities).toEqual([
+ { type: "EMAIL_ADDRESS", placeholder: "[[EMAIL_ADDRESS_1]]" },
+ ]);
+ } finally {
+ testConfig.masking.denylist = previousDenylist;
+ }
+ });
+
test("respects startFrom counters", async () => {
mockDetectPII.mockResolvedValueOnce([
{ entity_type: "EMAIL_ADDRESS", start: 0, end: 16, score: 0.9 },
expect(body.entities.some((e) => e.type === "EMAIL_ADDRESS")).toBe(false);
});
+ test("denylist does not corrupt an existing secret placeholder", async () => {
+ const previousDenylist = testConfig.masking.denylist;
+ testConfig.masking.denylist = [{ pattern: "\\d+", type: "NUM", regex: true }];
+ mockDetectPII.mockResolvedValueOnce([]);
+
+ try {
+ const res = await app.request("/api/mask", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ text: "Connection: postgres://admin:S3cretPass@db.example.com:5432/appdb",
+ }),
+ });
+
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as {
+ masked: string;
+ context: Record<string, string>;
+ entities: { type: string }[];
+ };
+ expect(body.masked).toContain("[[CONNECTION_STRING_1]]");
+ expect(body.masked).not.toContain("[[NUM");
+ expect(body.masked).not.toContain("STRING_[[");
+ expect(body.entities.some((e) => e.type === "NUM")).toBe(false);
+ expect(body.context["[[CONNECTION_STRING_1]]"]).toContain("postgres://");
+ } finally {
+ testConfig.masking.denylist = previousDenylist;
+ }
+ });
+
test("returns 400 for malformed JSON", async () => {
const res = await app.request("/api/mask", {
method: "POST",
import { z } from "zod";
import { getConfig, type SecretsDetectionConfig } from "../config";
import { createPlaceholderContext, type PlaceholderContext } from "../masking/context";
-import { filterWhitelistedEntities, getPIIDetector } from "../pii/detect";
+import {
+ filterWhitelistedEntities,
+ findDenylistedEntities,
+ getPIIDetector,
+ mergeDenylistEntities,
+} from "../pii/detect";
import { mask as maskPII } from "../pii/mask";
import { detectSecrets } from "../secrets/detect";
import { maskSecrets } from "../secrets/mask";
try {
const piiStartTime = Date.now();
const detector = getPIIDetector();
- const piiEntities = await detector.detectPII(maskedText, language);
+ const piiEntities = config.pii_detection.enabled
+ ? await detector.detectPII(maskedText, language)
+ : [];
scanTimeMs = Date.now() - piiStartTime;
// Apply whitelist filtering
piiEntities,
config.masking.whitelist,
);
+ const entitiesToMask = mergeDenylistEntities(
+ filteredEntities,
+ findDenylistedEntities(maskedText, config.masking.denylist, Object.keys(context.mapping)),
+ );
// Capture counters before masking to track new entities
const countersBefore = { ...context.counters };
- const piiResult = maskPII(maskedText, filteredEntities, context);
+ const piiResult = maskPII(maskedText, entitiesToMask, context);
maskedText = piiResult.masked;
allEntities.push(...extractEntities(countersBefore, piiResult.context));
// Collect unique entity types for logging
- for (const entity of filteredEntities) {
+ for (const entity of entitiesToMask) {
if (!piiEntityTypes.includes(entity.entity_type)) {
piiEntityTypes.push(entity.entity_type);
}
} from "../secrets/mask";
import { logRequest } from "../services/logger";
import { detectPII, maskPII, type PIIDetectResult } from "../services/pii";
-import { processSecretsRequest, type SecretsProcessResult } from "../services/secrets";
+import {
+ processSecretsRequest,
+ type SecretsProcessResult,
+ secretPlaceholders,
+} from "../services/secrets";
import {
createLogData,
errorFormats,
}
let piiResult: PIIDetectResult;
- if (!config.pii_detection.enabled) {
- piiResult = {
- detection: {
- hasPII: false,
- spanEntities: [],
- allEntities: [],
- scanTimeMs: 0,
- language: config.pii_detection.fallback_language,
- languageFallback: false,
- },
- hasPII: false,
- };
- } else {
- try {
- piiResult = await detectPII(request, codexExtractor);
- } catch (error) {
- console.error("PII detection error:", error);
- return respondDetectionError(c, request, startTime);
- }
+ try {
+ piiResult = await detectPII(request, codexExtractor, secretPlaceholders(secretsResult));
+ } catch (error) {
+ console.error("PII detection error:", error);
+ return respondDetectionError(c, request, startTime);
}
const shouldBlockRouteMode =
import { unmaskSecretsResponse } from "../secrets/mask";
import { logRequest } from "../services/logger";
import { detectPII, maskPII, type PIIDetectResult } from "../services/pii";
-import { processSecretsRequest, type SecretsProcessResult } from "../services/secrets";
+import {
+ processSecretsRequest,
+ type SecretsProcessResult,
+ secretPlaceholders,
+} from "../services/secrets";
import { extractTextContent } from "../utils/content";
import {
createLogData,
request = secretsResult.request;
}
- // Step 2: Detect PII (skip if disabled)
+ // Step 2: Detect PII and configured denylist terms
let piiResult: PIIDetectResult;
- if (!config.pii_detection.enabled) {
- piiResult = {
- detection: {
- hasPII: false,
- spanEntities: [],
- allEntities: [],
- scanTimeMs: 0,
- language: "en",
- languageFallback: false,
- },
- hasPII: false,
- };
- } else {
- try {
- piiResult = await detectPII(request, openaiExtractor);
- } catch (error) {
- console.error("PII detection error:", error);
- return respondDetectionError(c, request, startTime);
- }
+ try {
+ piiResult = await detectPII(request, openaiExtractor, secretPlaceholders(secretsResult));
+ } catch (error) {
+ console.error("PII detection error:", error);
+ return respondDetectionError(c, request, startTime);
}
// Step 3: Process based on mode
export async function detectPII<TRequest, TResponse>(
request: TRequest,
extractor: RequestExtractor<TRequest, TResponse>,
+ // Required (no default) so a new route can't silently skip the secrets placeholders.
+ knownPlaceholders: readonly string[],
): Promise<PIIDetectResult> {
const detector = getPIIDetector();
- const detection = await detector.analyzeRequest(request, extractor);
+ const detection = await detector.analyzeRequest(request, extractor, knownPlaceholders);
return {
detection,
masked: boolean;
}
+/** Placeholder strings already inserted by secrets masking (so later PII/denylist passes skip them). */
+export function secretPlaceholders<TRequest>(result: SecretsProcessResult<TRequest>): string[] {
+ return result.maskingContext ? Object.keys(result.maskingContext.mapping) : [];
+}
+
/**
* Process a request for secrets detection
*/
git.99rst.org / sgasser-llm-shield.git / commitdiff