From: Stefan Gasser Date: Wed, 21 Jan 2026 17:31:02 +0000 (+0100) Subject: Refactor logging interfaces for simpler data structures X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=8d387c329fa816fe80de4236bd8f17dd36a6a7b9;p=sgasser-llm-shield.git Refactor logging interfaces for simpler data structures - Simplify PIILogData: entityTypes string[] instead of allEntities objects - Simplify SecretsLogData: types string[] instead of matches objects - Move mapping logic into toPIILogData/toSecretsLogData converters - Update api.ts to use createLogData() instead of manual construction --- diff --git a/src/routes/anthropic.ts b/src/routes/anthropic.ts index 0153e8c..23da843 100644 --- a/src/routes/anthropic.ts +++ b/src/routes/anthropic.ts @@ -263,7 +263,7 @@ function respondBlocked( provider: "anthropic", model: request.model, startTime, - secrets: { detected: true, matches: secretTypes.map((t) => ({ type: t })), masked: false }, + secrets: { detected: true, types: secretTypes, masked: false }, statusCode: 400, errorMessage: `Request blocked: detected secret material (${secretTypes.join(",")})`, }), diff --git a/src/routes/api.ts b/src/routes/api.ts index e544b66..79adbb0 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -8,23 +8,14 @@ import { Hono } from "hono"; import { z } from "zod"; import { getConfig, type SecretsDetectionConfig } from "../config"; -import { resolveConflicts, resolveOverlaps } from "../masking/conflict-resolver"; -import { - createPlaceholderContext, - incrementAndGenerate, - type PlaceholderContext, - replaceWithPlaceholders, -} from "../masking/context"; -import { - generatePlaceholder as generatePlaceholderFromFormat, - generateSecretPlaceholder, - PII_PLACEHOLDER_FORMAT, -} from "../masking/placeholders"; -import type { PIIEntity } from "../pii/detect"; +import { createPlaceholderContext, type PlaceholderContext } from "../masking/context"; import { getPIIDetector } from "../pii/detect"; -import { detectSecrets, type SecretLocation } from "../secrets/detect"; +import { mask as maskPII } from "../pii/mask"; +import { detectSecrets } from "../secrets/detect"; +import { maskSecrets } from "../secrets/mask"; import { getLanguageDetector, type SupportedLanguage } from "../services/language-detector"; import { logRequest } from "../services/logger"; +import { createLogData } from "./utils"; export const apiRoutes = new Hono(); @@ -53,79 +44,27 @@ interface MaskResponse { } /** - * Generates a PII placeholder + * Extracts entities from context by comparing counters before/after masking */ -function generatePIIPlaceholder(entityType: string, context: PlaceholderContext): string { - return incrementAndGenerate(entityType, context, (type, count) => - generatePlaceholderFromFormat(PII_PLACEHOLDER_FORMAT, type, count), - ); -} - -/** - * Generates a secrets placeholder - */ -function generateSecretsPlaceholder(secretType: string, context: PlaceholderContext): string { - return incrementAndGenerate(secretType, context, generateSecretPlaceholder); -} - -/** - * Masks text with PII entities - */ -function maskWithPII( - text: string, - entities: PIIEntity[], - context: PlaceholderContext, -): { masked: string; entities: MaskEntity[] } { - if (entities.length === 0) { - return { masked: text, entities: [] }; - } - - const maskEntities: MaskEntity[] = []; - - const masked = replaceWithPlaceholders( - text, - entities, - context, - (e) => e.entity_type, - (type, ctx) => { - const placeholder = generatePIIPlaceholder(type, ctx); - maskEntities.push({ type, placeholder }); - return placeholder; - }, - resolveConflicts, - ); - - return { masked, entities: maskEntities }; -} - -/** - * Masks text with secret locations - */ -function maskWithSecrets( - text: string, - locations: SecretLocation[], +function extractEntities( + countersBefore: Record, context: PlaceholderContext, -): { masked: string; entities: MaskEntity[] } { - if (locations.length === 0) { - return { masked: text, entities: [] }; +): MaskEntity[] { + const entities: MaskEntity[] = []; + + for (const [type, count] of Object.entries(context.counters)) { + const startCount = countersBefore[type] || 0; + // Add entities for each new placeholder created + for (let i = startCount + 1; i <= count; i++) { + // Find the placeholder in the mapping + const placeholder = Object.keys(context.mapping).find((p) => p.includes(`${type}_${i}]`)); + if (placeholder) { + entities.push({ type, placeholder }); + } + } } - const maskEntities: MaskEntity[] = []; - - const masked = replaceWithPlaceholders( - text, - locations, - context, - (loc) => loc.type, - (type, ctx) => { - const placeholder = generateSecretsPlaceholder(type, ctx); - maskEntities.push({ type, placeholder }); - return placeholder; - }, - resolveOverlaps, - ); - - return { masked, entities: maskEntities }; + return entities; } /** @@ -208,9 +147,11 @@ apiRoutes.post("/mask", async (c) => { ); }); - const piiResult = maskWithPII(maskedText, filteredEntities, context); + // Capture counters before masking to track new entities + const countersBefore = { ...context.counters }; + const piiResult = maskPII(maskedText, filteredEntities, context); maskedText = piiResult.masked; - allEntities.push(...piiResult.entities); + allEntities.push(...extractEntities(countersBefore, piiResult.context)); // Collect unique entity types for logging for (const entity of filteredEntities) { @@ -221,20 +162,14 @@ apiRoutes.post("/mask", async (c) => { } catch (error) { // Log the error logRequest( - { - timestamp: new Date().toISOString(), - mode: "mask", + createLogData({ provider: "api", model: "mask", - piiDetected: false, - entities: [], - latencyMs: Date.now() - startTime, - scanTimeMs: 0, - language, - languageFallback, + startTime, + pii: { hasPII: false, entityTypes: [], language, languageFallback, scanTimeMs: 0 }, statusCode: 503, errorMessage: error instanceof Error ? error.message : "PII detection failed", - }, + }), userAgent, ); @@ -266,9 +201,11 @@ apiRoutes.post("/mask", async (c) => { const secretsResult = detectSecrets(maskedText, secretsConfig); if (secretsResult.locations && secretsResult.locations.length > 0) { - const secretsMaskResult = maskWithSecrets(maskedText, secretsResult.locations, context); + // Capture counters before masking to track new entities + const countersBefore = { ...context.counters }; + const secretsMaskResult = maskSecrets(maskedText, secretsResult.locations, context); maskedText = secretsMaskResult.masked; - allEntities.push(...secretsMaskResult.entities); + allEntities.push(...extractEntities(countersBefore, secretsMaskResult.context)); // Collect unique secret types for logging for (const match of secretsResult.matches) { @@ -280,20 +217,20 @@ apiRoutes.post("/mask", async (c) => { } catch (error) { // Log the error logRequest( - { - timestamp: new Date().toISOString(), - mode: "mask", + createLogData({ provider: "api", model: "mask", - piiDetected: piiEntityTypes.length > 0, - entities: piiEntityTypes, - latencyMs: Date.now() - startTime, - scanTimeMs, - language, - languageFallback, + startTime, + pii: { + hasPII: piiEntityTypes.length > 0, + entityTypes: piiEntityTypes, + language, + languageFallback, + scanTimeMs, + }, statusCode: 503, errorMessage: error instanceof Error ? error.message : "Secrets detection failed", - }, + }), userAgent, ); @@ -312,22 +249,22 @@ apiRoutes.post("/mask", async (c) => { // Log successful request logRequest( - { - timestamp: new Date().toISOString(), - mode: "mask", + createLogData({ provider: "api", model: "mask", - piiDetected: piiEntityTypes.length > 0, - entities: piiEntityTypes, - latencyMs: Date.now() - startTime, - scanTimeMs, - language, - languageFallback, + startTime, + pii: { + hasPII: piiEntityTypes.length > 0, + entityTypes: piiEntityTypes, + language, + languageFallback, + scanTimeMs, + }, + secrets: + secretTypes.length > 0 ? { detected: true, types: secretTypes, masked: true } : undefined, maskedContent: config.logging.log_masked_content ? maskedText : undefined, - secretsDetected: secretTypes.length > 0, - secretsTypes: secretTypes.length > 0 ? secretTypes : undefined, statusCode: 200, - }, + }), userAgent, ); diff --git a/src/routes/openai.ts b/src/routes/openai.ts index d4c347c..51ca97d 100644 --- a/src/routes/openai.ts +++ b/src/routes/openai.ts @@ -206,7 +206,7 @@ function respondBlocked( provider: "openai", model: body.model || "unknown", startTime, - secrets: { detected: true, matches: secretTypes.map((t) => ({ type: t })), masked: false }, + secrets: { detected: true, types: secretTypes, masked: false }, statusCode: 400, errorMessage: secretsResult.blockedReason, }), diff --git a/src/routes/utils.ts b/src/routes/utils.ts index 44732bf..68d51a6 100644 --- a/src/routes/utils.ts +++ b/src/routes/utils.ts @@ -137,7 +137,7 @@ export function setBlockedHeaders(c: Context, secretTypes: string[]): void { */ export interface PIILogData { hasPII: boolean; - allEntities: { entity_type: string }[]; + entityTypes: string[]; language: string; languageFallback: boolean; detectedLanguage?: string; @@ -149,7 +149,7 @@ export interface PIILogData { */ export interface SecretsLogData { detected?: boolean; - matches?: { type: string }[]; + types?: string[]; masked: boolean; } @@ -159,7 +159,7 @@ export interface SecretsLogData { export function toPIILogData(piiResult: PIIDetectResult): PIILogData { return { hasPII: piiResult.hasPII, - allEntities: piiResult.detection.allEntities, + entityTypes: [...new Set(piiResult.detection.allEntities.map((e) => e.entity_type))], language: piiResult.detection.language, languageFallback: piiResult.detection.languageFallback, detectedLanguage: piiResult.detection.detectedLanguage, @@ -187,7 +187,7 @@ export function toSecretsLogData( if (!secretsResult.detection) return undefined; return { detected: secretsResult.detection.detected, - matches: secretsResult.detection.matches, + types: secretsResult.detection.matches.map((m) => m.type), masked: secretsResult.masked, }; } @@ -231,7 +231,7 @@ export function createLogData(options: CreateLogDataOptions): RequestLogData { provider, model: model || "unknown", piiDetected: pii?.hasPII ?? false, - entities: pii ? [...new Set(pii.allEntities.map((e) => e.entity_type))] : [], + entities: pii?.entityTypes ?? [], latencyMs: Date.now() - startTime, scanTimeMs: pii?.scanTimeMs ?? 0, language: pii?.language ?? config.pii_detection.fallback_language, @@ -239,7 +239,7 @@ export function createLogData(options: CreateLogDataOptions): RequestLogData { detectedLanguage: pii?.detectedLanguage, maskedContent, secretsDetected: secrets?.detected, - secretsTypes: secrets?.matches?.map((m) => m.type), + secretsTypes: secrets?.types, statusCode, errorMessage, }; diff --git a/src/services/logger.ts b/src/services/logger.ts index bd1a952..ddf2d00 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -32,7 +32,7 @@ export interface Stats { total_requests: number; pii_requests: number; pii_percentage: number; - openai_requests: number; + proxy_requests: number; local_requests: number; api_requests: number; avg_scan_time_ms: number; @@ -170,9 +170,11 @@ export class Logger { .prepare(`SELECT COUNT(*) as count FROM request_logs WHERE pii_detected = 1`) .get() as { count: number }; - // Upstream vs Local vs API - const openaiResult = this.db - .prepare(`SELECT COUNT(*) as count FROM request_logs WHERE provider = 'openai'`) + // Proxy (OpenAI + Anthropic) vs Local vs API + const proxyResult = this.db + .prepare( + `SELECT COUNT(*) as count FROM request_logs WHERE provider IN ('openai', 'anthropic')`, + ) .get() as { count: number }; const localResult = this.db .prepare(`SELECT COUNT(*) as count FROM request_logs WHERE provider = 'local'`) @@ -210,7 +212,7 @@ export class Logger { total_requests: total, pii_requests: pii, pii_percentage: total > 0 ? Math.round((pii / total) * 100 * 10) / 10 : 0, - openai_requests: openaiResult.count, + proxy_requests: proxyResult.count, local_requests: localResult.count, api_requests: apiResult.count, avg_scan_time_ms: Math.round(scanTimeResult.avg || 0), diff --git a/src/views/dashboard/page.tsx b/src/views/dashboard/page.tsx index 963434f..9f438d9 100644 --- a/src/views/dashboard/page.tsx +++ b/src/views/dashboard/page.tsx @@ -258,9 +258,9 @@ const StatsGrid: FC = () => ( @@ -463,15 +463,15 @@ async function fetchStats() { } if (data.mode === 'route') { - document.getElementById('openai-requests').textContent = data.openai_requests.toLocaleString(); + document.getElementById('proxy-requests').textContent = data.proxy_requests.toLocaleString(); document.getElementById('local-requests').textContent = data.local_requests.toLocaleString(); - const total = data.openai_requests + data.local_requests; - const openaiPct = total > 0 ? Math.round((data.openai_requests / total) * 100) : 50; - const localPct = 100 - openaiPct; + const total = data.proxy_requests + data.local_requests; + const proxyPct = total > 0 ? Math.round((data.proxy_requests / total) * 100) : 50; + const localPct = 100 - proxyPct; document.getElementById('provider-split').innerHTML = - '
' + openaiPct + '%
' + + '
' + proxyPct + '%
' + '
' + localPct + '%
'; }