From: Stefan Gasser Date: Tue, 23 Jun 2026 09:39:04 +0000 (+0200) Subject: Clean up unused exports and route helpers (#113) X-Git-Tag: v0.7.0~6 X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=b899dc15fbd371792048a79d90a6112685ca0a10;p=sgasser-llm-shield.git Clean up unused exports and route helpers (#113) * Remove unused postcss dependency * Drop unused secret and masking exports * Extract streaming header helper * Trim redundant JSDoc comments * Document Codex login setup * Revert "Document Codex login setup" This reverts commit a60ef991da233710e3785a964a6a11b6540f8de3. --- diff --git a/bun.lock b/bun.lock index 1ee7c79..bbe20c2 100644 --- a/bun.lock +++ b/bun.lock @@ -8,7 +8,6 @@ "@hono/zod-validator": "^0.7.6", "hono": "^4.12.23", "hono-tailwind": "^2.2.0", - "postcss": "^8.5.15", "tailwindcss": "^4.3.0", "yaml": "^2.9.0", "zod": "^3.25.76", diff --git a/package.json b/package.json index 5e71562..c15a31b 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ "@hono/zod-validator": "^0.7.6", "hono": "^4.12.23", "hono-tailwind": "^2.2.0", - "postcss": "^8.5.15", "tailwindcss": "^4.3.0", "yaml": "^2.9.0", "zod": "^3.25.76" diff --git a/src/masking/conflict-resolver.ts b/src/masking/conflict-resolver.ts index 32d4f9a..4be6641 100644 --- a/src/masking/conflict-resolver.ts +++ b/src/masking/conflict-resolver.ts @@ -1,17 +1,11 @@ // Conflict resolution based on Microsoft Presidio's logic // https://github.com/microsoft/presidio/blob/main/presidio-anonymizer/presidio_anonymizer/anonymizer_engine.py -/** - * Base interface for items with position (used by both PII and secrets) - */ export interface Span { start: number; end: number; } -/** - * Extended interface for PII entities with confidence scores - */ export interface EntityWithScore extends Span { score: number; entity_type: string; @@ -76,7 +70,6 @@ function removeConflicting(entities: T[]): T[] { return result; } -/** For PII entities with scores. Merges same-type overlaps, removes cross-type conflicts. */ export function resolveConflicts(entities: T[]): T[] { if (entities.length <= 1) return [...entities]; @@ -96,10 +89,7 @@ export function resolveConflicts(entities: T[]): T[] return removeConflicting(afterMerge); } -/** - * Simple conflict resolution for items without scores (secrets) - * Keeps non-overlapping spans, longer span wins ties. - */ +// Secrets have no confidence score, so overlapping ties keep the longer span. export function resolveOverlaps(items: T[]): T[] { if (items.length <= 1) return [...items]; diff --git a/src/masking/context.ts b/src/masking/context.ts index 63f8b02..6648d0e 100644 --- a/src/masking/context.ts +++ b/src/masking/context.ts @@ -1,37 +1,12 @@ -/** - * Placeholder context and text transformation utilities - */ - import { findPartialPlaceholderStart } from "../masking/placeholders"; import type { Span } from "./conflict-resolver"; -/** - * Generic context for placeholder-based transformations - * Used by both PII masking and secrets masking - */ export interface PlaceholderContext { - /** Maps placeholder -> original value */ mapping: Record; - /** Maps original value -> placeholder (for deduplication) */ reverseMapping: Record; - /** Counter per type for sequential numbering */ counters: Record; } -/** - * Result of masking text with placeholders - * Used by both PII masking and secrets masking - */ -export interface MaskResult { - /** Text with sensitive data replaced by placeholders */ - masked: string; - /** Context for unmasking (maps placeholders to original values) */ - context: PlaceholderContext; -} - -/** - * Creates a new placeholder context - */ export function createPlaceholderContext(): PlaceholderContext { return { mapping: {}, @@ -40,11 +15,6 @@ export function createPlaceholderContext(): PlaceholderContext { }; } -/** - * Increments counter for type and generates placeholder using format function - * - * Shared counter logic for both PII masking and secrets masking. - */ export function incrementAndGenerate( type: string, context: PlaceholderContext, @@ -55,15 +25,6 @@ export function incrementAndGenerate( return format(type, count); } -/** - * Restores placeholders in text with original values - * - * Generic function used by both PII unmasking and secrets unmasking. - * - * @param text - Text containing placeholders - * @param context - Context with placeholder mappings - * @param formatValue - Optional function to format restored values (e.g., add markers) - */ export function restorePlaceholders( text: string, context: PlaceholderContext, @@ -84,19 +45,6 @@ export function restorePlaceholders( return result; } -/** - * Replaces items in text with placeholders - * - * Generic function used by both PII masking and secrets masking. - * Handles: conflict resolution, placeholder assignment, and replacement. - * - * @param text - Text to process - * @param items - Items with start/end positions to replace - * @param context - Placeholder context for tracking mappings - * @param getType - Function to get the type string from an item - * @param generatePlaceholder - Function to generate placeholder for a type - * @param resolveConflicts - Function to resolve overlapping items - */ export function replaceWithPlaceholders( text: string, items: T[], @@ -145,16 +93,6 @@ export function replaceWithPlaceholders( return result; } -/** - * Processes a stream chunk, buffering partial placeholders - * - * Generic function used by both PII unmasking and secrets unmasking. - * - * @param buffer - Previous buffer content - * @param newChunk - New chunk to process - * @param context - Placeholder context - * @param restore - Function to restore placeholders in text - */ export function processStreamChunk( buffer: string, newChunk: string, @@ -183,13 +121,6 @@ export function processStreamChunk( }; } -/** - * Flushes remaining buffer at end of stream - * - * @param buffer - Remaining buffer content - * @param context - Placeholder context - * @param restore - Function to restore placeholders in text - */ export function flushBuffer( buffer: string, context: PlaceholderContext, diff --git a/src/pii/mask.ts b/src/pii/mask.ts index d318293..befdb11 100644 --- a/src/pii/mask.ts +++ b/src/pii/mask.ts @@ -1,7 +1,3 @@ -/** - * PII masking - */ - import type { MaskingConfig } from "../config"; import { resolveConflicts } from "../masking/conflict-resolver"; import { incrementAndGenerate } from "../masking/context"; @@ -22,33 +18,21 @@ import type { PIIDetectionResult, PIIEntity } from "./detect"; export { createMaskingContext, type PlaceholderContext } from "../masking/service"; -/** - * Result of masking operation - */ export interface MaskResult { masked: string; context: PlaceholderContext; } -/** - * Generates a placeholder for a PII entity type - */ function generatePlaceholder(entityType: string, context: PlaceholderContext): string { return incrementAndGenerate(entityType, context, (type, count) => generatePlaceholderFromFormat(PII_PLACEHOLDER_FORMAT, type, count), ); } -/** - * Creates formatValue function from masking config - */ function getFormatValue(config: MaskingConfig): ((original: string) => string) | undefined { return config.show_markers ? (original: string) => `${config.marker_text}${original}` : undefined; } -/** - * Masks PII entities in text, replacing them with placeholders - */ export function mask( text: string, entities: PIIEntity[], @@ -72,16 +56,10 @@ export function mask( }; } -/** - * Unmasks text by replacing placeholders with original values - */ export function unmask(text: string, context: PlaceholderContext, config: MaskingConfig): string { return unmaskText(text, context, getFormatValue(config)); } -/** - * Streaming unmask helper - processes chunks and unmasks when complete placeholders are found - */ export function unmaskStreamChunk( buffer: string, newChunk: string, @@ -91,9 +69,6 @@ export function unmaskStreamChunk( return unmaskChunk(buffer, newChunk, context, getFormatValue(config)); } -/** - * Flushes remaining buffer at end of stream - */ export function flushMaskingBuffer( buffer: string, context: PlaceholderContext, @@ -102,19 +77,11 @@ export function flushMaskingBuffer( return flushBuffer(buffer, context, getFormatValue(config)); } -/** - * Result of masking a request - */ export interface MaskRequestResult { - /** The masked request */ request: TRequest; - /** Masking context for unmasking response */ context: PlaceholderContext; } -/** - * Masks PII in a request using an extractor - */ export function maskRequest( request: TRequest, detection: PIIDetectionResult, @@ -153,9 +120,6 @@ function maskSpansWithEntities( ); } -/** - * Unmasks a response using a request extractor - */ export function unmaskResponse( response: TResponse, context: PlaceholderContext, diff --git a/src/providers/anthropic/stream-transformer.ts b/src/providers/anthropic/stream-transformer.ts index 87c972a..53984f5 100644 --- a/src/providers/anthropic/stream-transformer.ts +++ b/src/providers/anthropic/stream-transformer.ts @@ -1,12 +1,5 @@ -/** - * Anthropic SSE stream transformer for unmasking PII and secrets - * - * Anthropic uses a different SSE format than OpenAI: - * - event: message_start / content_block_start / content_block_delta / etc. - * - data: {...} - * - * Text content comes in content_block_delta events with delta.type === "text_delta" - */ +// Anthropic SSE differs from OpenAI: event lines identify message/content events, +// and text arrives as content_block_delta data with delta.type === "text_delta". import type { MaskingConfig } from "../../config"; import type { PlaceholderContext } from "../../masking/context"; @@ -14,9 +7,6 @@ import { flushMaskingBuffer, unmaskStreamChunk } from "../../pii/mask"; import { flushSecretsMaskingBuffer, unmaskSecretsStreamChunk } from "../../secrets/mask"; import type { ContentBlockDeltaEvent, TextDelta } from "./types"; -/** - * Creates a transform stream that unmasks Anthropic SSE content - */ export function createAnthropicUnmaskingStream( source: ReadableStream, piiContext: PlaceholderContext | undefined, diff --git a/src/routes/anthropic.ts b/src/routes/anthropic.ts index 4cbaa76..6e0876f 100644 --- a/src/routes/anthropic.ts +++ b/src/routes/anthropic.ts @@ -1,14 +1,3 @@ -/** - * Anthropic-compatible messages route - * - * Flow: - * 1. Validate request - * 2. Process secrets (detect, maybe block, mask, or route_local) - * 3. Detect PII - * 4. Route mode: if PII found, send to local provider - * 5. Mask mode: mask PII if found, send to Anthropic, unmask response - */ - import { zValidator } from "@hono/zod-validator"; import type { Context } from "hono"; import { Hono } from "hono"; @@ -42,6 +31,7 @@ import { handleProviderError, setBlockedHeaders, setResponseHeaders, + setStreamingHeaders, toPIIHeaderData, toPIILogData, toSecretsHeaderData, @@ -50,9 +40,6 @@ import { export const anthropicRoutes = new Hono(); -/** - * POST /v1/messages - Anthropic-compatible messages endpoint - */ anthropicRoutes.post( "/v1/messages", zValidator("json", AnthropicRequestSchema, (result, c) => { @@ -162,11 +149,6 @@ anthropicRoutes.post( }, ); -/** - * Proxy all other requests to Anthropic - * - * Transparent header forwarding - all auth headers from client are passed through. - */ anthropicRoutes.all("/*", async (c) => { const config = getConfig(); @@ -327,9 +309,7 @@ async function sendToLocal(c: Context, originalRequest: AnthropicRequest, opts: ); if (result.isStreaming) { - c.header("Content-Type", "text/event-stream"); - c.header("Cache-Control", "no-cache"); - c.header("Connection", "keep-alive"); + setStreamingHeaders(c); return c.body(result.response as ReadableStream); } @@ -417,9 +397,7 @@ function respondStreaming( secretsContext: PlaceholderContext | undefined, ) { const config = getConfig(); - c.header("Content-Type", "text/event-stream"); - c.header("Cache-Control", "no-cache"); - c.header("Connection", "keep-alive"); + setStreamingHeaders(c); if (piiMaskingContext || secretsContext) { const unmaskingStream = createAnthropicUnmaskingStream( diff --git a/src/routes/codex.ts b/src/routes/codex.ts index dae9585..771bfdd 100644 --- a/src/routes/codex.ts +++ b/src/routes/codex.ts @@ -34,6 +34,7 @@ import { handleProviderError, setBlockedHeaders, setResponseHeaders, + setStreamingHeaders, toPIIHeaderData, toPIILogData, toSecretsHeaderData, @@ -51,13 +52,6 @@ const CodexResponsesRequestSchema = z }) .passthrough(); -/** - * POST /responses - * - * Inspected Codex Responses route. This mirrors the OpenAI/Anthropic protected - * endpoints: detect secrets/PII, mask before upstream, unmask streamed response, - * and log the request in the dashboard. - */ codexRoutes.post( "/responses", zValidator("json", CodexResponsesRequestSchema, (result, c) => { @@ -115,10 +109,6 @@ codexRoutes.post( }, ); -/** - * Wildcard pass-through proxy for /models and any future Codex endpoints that do - * not carry prompt content. - */ codexRoutes.all("/*", (c) => { const config = getConfig(); const normalizedBaseUrl = config.providers.codex.base_url.replace(/\/$/, ""); @@ -370,9 +360,7 @@ function respondStreaming( secretsContext?: PlaceholderContext, maskingConfig = getConfig().masking, ) { - c.header("Content-Type", "text/event-stream"); - c.header("Cache-Control", "no-cache"); - c.header("Connection", "keep-alive"); + setStreamingHeaders(c); if (piiContext || secretsContext) { return c.body(createCodexUnmaskingStream(stream, piiContext, maskingConfig, secretsContext)); diff --git a/src/routes/openai.ts b/src/routes/openai.ts index 1f66250..2e64441 100644 --- a/src/routes/openai.ts +++ b/src/routes/openai.ts @@ -1,16 +1,3 @@ -/** - * OpenAI-compatible chat completion route - * - * Flow: - * 1. Validate request - * 2. Process secrets (detect, maybe block or mask) - * 3. Detect PII - * 4. Based on mode: - * - mask: mask PII, send to OpenAI, unmask response - * - route: send to local (if PII) or OpenAI (if clean) - * 5. Return response - */ - import { zValidator } from "@hono/zod-validator"; import type { Context } from "hono"; import { Hono } from "hono"; @@ -43,6 +30,7 @@ import { handleProviderError, setBlockedHeaders, setResponseHeaders, + setStreamingHeaders, toPIIHeaderData, toPIILogData, toSecretsHeaderData, @@ -51,9 +39,6 @@ import { export const openaiRoutes = new Hono(); -/** - * POST /v1/chat/completions - */ openaiRoutes.post( "/v1/chat/completions", zValidator("json", OpenAIRequestSchema, (result, c) => { @@ -130,9 +115,6 @@ openaiRoutes.post( }, ); -/** - * Wildcard proxy for /models, /embeddings, /audio/*, /images/*, etc. - */ openaiRoutes.all("/*", (c) => { const config = getConfig(); const { baseUrl } = getOpenAIInfo(config.providers.openai); @@ -337,9 +319,7 @@ async function sendToLocal(c: Context, originalRequest: OpenAIRequest, opts: Loc ); if (result.isStreaming) { - c.header("Content-Type", "text/event-stream"); - c.header("Cache-Control", "no-cache"); - c.header("Connection", "keep-alive"); + setStreamingHeaders(c); return c.body(result.response as ReadableStream); } @@ -371,9 +351,7 @@ function respondStreaming( secretsContext?: PlaceholderContext, maskingConfig?: MaskingConfig, ) { - c.header("Content-Type", "text/event-stream"); - c.header("Cache-Control", "no-cache"); - c.header("Connection", "keep-alive"); + setStreamingHeaders(c); if (piiContext || secretsContext) { const stream = createUnmaskingStream( diff --git a/src/routes/utils.ts b/src/routes/utils.ts index 5aed03a..2b1616e 100644 --- a/src/routes/utils.ts +++ b/src/routes/utils.ts @@ -1,10 +1,3 @@ -/** - * Shared route utilities - * - * Common utilities for route handlers including error formatting, - * response headers, and logging helpers. - */ - import type { Context } from "hono"; import { getConfig } from "../config"; import { ProviderError } from "../providers/errors"; @@ -17,9 +10,6 @@ import type { SecretsProcessResult } from "../services/secrets"; // Error Response Types & Formatting // ============================================================================ -/** - * Error response format for OpenAI - */ export interface OpenAIErrorResponse { error: { message: string; @@ -29,9 +19,6 @@ export interface OpenAIErrorResponse { }; } -/** - * Error response format for Anthropic - */ export interface AnthropicErrorResponse { type: "error"; error: { @@ -40,9 +27,6 @@ export interface AnthropicErrorResponse { }; } -/** - * Format adapters for different API schemas - */ export const errorFormats = { openai: { error( @@ -88,9 +72,6 @@ export interface SecretsHeaderData { masked: boolean; } -/** - * Set common PasteGuard response headers - */ export function setResponseHeaders( c: Context, mode: string, @@ -114,39 +95,33 @@ export function setResponseHeaders( } } -/** - * Set headers for blocked request (secrets detected) - */ export function setBlockedHeaders(c: Context, secretTypes: string[]): void { c.header("X-PasteGuard-Secrets-Detected", "true"); c.header("X-PasteGuard-Secrets-Types", secretTypes.join(",")); } +export function setStreamingHeaders(c: Context): void { + c.header("Content-Type", "text/event-stream"); + c.header("Cache-Control", "no-cache"); + c.header("Connection", "keep-alive"); +} + // ============================================================================ // Logging Helpers // ============================================================================ -/** - * PII detection result for logging - */ export interface PIILogData { hasPII: boolean; entityTypes: string[]; scanTimeMs: number; } -/** - * Secrets detection result for logging - */ export interface SecretsLogData { detected?: boolean; types?: string[]; masked: boolean; } -/** - * Convert PIIDetectResult to PIILogData - */ export function toPIILogData(piiResult: PIIDetectResult): PIILogData { return { hasPII: piiResult.hasPII, @@ -155,18 +130,12 @@ export function toPIILogData(piiResult: PIIDetectResult): PIILogData { }; } -/** - * Convert PIIDetectResult to PIIHeaderData - */ export function toPIIHeaderData(piiResult: PIIDetectResult): PIIHeaderData { return { hasPII: piiResult.hasPII, }; } -/** - * Convert SecretsProcessResult to SecretsLogData - */ export function toSecretsLogData( secretsResult: SecretsProcessResult, ): SecretsLogData | undefined { @@ -178,9 +147,6 @@ export function toSecretsLogData( }; } -/** - * Convert SecretsProcessResult to SecretsHeaderData - */ export function toSecretsHeaderData( secretsResult: SecretsProcessResult, ): SecretsHeaderData | undefined { @@ -204,9 +170,6 @@ export interface CreateLogDataOptions { errorMessage?: string; } -/** - * Create log data object for request logging - */ export function createLogData(options: CreateLogDataOptions): RequestLogData { const config = getConfig(); const { provider, model, startTime, pii, secrets, maskedContent, statusCode, errorMessage } = @@ -245,13 +208,6 @@ export interface ProviderErrorContext { userAgent: string | null; } -/** - * Handle provider errors with logging - * - * Returns the appropriate response for the error type. - * For ProviderError, returns the original error body. - * For other errors, returns a formatted error response. - */ export function handleProviderError( c: Context, error: unknown, diff --git a/src/secrets/detect.ts b/src/secrets/detect.ts index 2f2a760..295d155 100644 --- a/src/secrets/detect.ts +++ b/src/secrets/detect.ts @@ -10,23 +10,11 @@ import type { export type { MessageSecretsResult, - SecretEntityType, SecretLocation, - SecretsDetectionResult, - SecretsMatch, } from "./patterns/types"; -/** - * Detects secret material (e.g. private keys, API keys, tokens) in text - * - * Uses the pattern registry to scan for various secret types: - * - Private keys: OpenSSH, PEM (RSA, generic, encrypted) - * - API keys: OpenAI, AWS, GitHub - * - Tokens: JWT, Bearer - * - Environment variables: Passwords, secrets, connection strings - * - * Respects max_scan_chars limit for performance. - */ +// Scans private keys, API keys, tokens, env secrets, and connection strings. +// Respects max_scan_chars to cap regex work on large prompts. export function detectSecrets( text: string, config: SecretsDetectionConfig, @@ -67,9 +55,6 @@ export function detectSecrets( }; } -/** - * Detects secrets in a request using an extractor - */ export function detectSecretsInRequest( request: TRequest, config: SecretsDetectionConfig, @@ -79,10 +64,7 @@ export function detectSecretsInRequest( return detectSecretsInSpans(spans, config); } -/** - * Detects secrets in text spans (low-level) - */ -export function detectSecretsInSpans( +function detectSecretsInSpans( spans: TextSpan[], config: SecretsDetectionConfig, ): MessageSecretsResult { diff --git a/src/secrets/patterns/index.ts b/src/secrets/patterns/index.ts index 5505ea2..aa29eb6 100644 --- a/src/secrets/patterns/index.ts +++ b/src/secrets/patterns/index.ts @@ -16,6 +16,3 @@ export const patternDetectors: PatternDetector[] = [ tokensDetector, envVarsDetector, ]; - -export type { PatternDetector, SecretEntityType, SecretsDetectionResult } from "./types"; -export { detectPattern } from "./utils"; diff --git a/src/services/logger.ts b/src/services/logger.ts index edcb42b..4bc4164 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -27,9 +27,6 @@ export interface RequestLog { error_message: string | null; } -/** - * Statistics summary - */ export interface Stats { total_requests: number; pii_requests: number; @@ -58,9 +55,6 @@ export function normalizeRequestSource( return "api"; } -/** - * SQLite-based logger for request tracking - */ export class Logger { private db: Database; private retentionDays: number; @@ -161,9 +155,6 @@ export class Logger { ); } - /** - * Gets recent logs - */ getLogs(limit: number = 100, offset: number = 0): RequestLog[] { const stmt = this.db.prepare(` SELECT @@ -196,9 +187,6 @@ export class Logger { })); } - /** - * Gets statistics - */ getStats(): Stats { // Total requests const totalResult = this.db.prepare(`SELECT COUNT(*) as count FROM request_logs`).get() as { @@ -267,9 +255,6 @@ export class Logger { }; } - /** - * Gets entity breakdown - */ getEntityStats(): Array<{ entity: string; count: number }> { const logs = this.db .prepare(` @@ -294,9 +279,6 @@ export class Logger { .sort((a, b) => b.count - a.count); } - /** - * Cleans up old logs based on retention policy - */ cleanup(): number { if (this.retentionDays <= 0) { return 0; // Keep forever @@ -314,9 +296,6 @@ export class Logger { return result.changes; } - /** - * Closes database connection - */ close(): void { this.db.close(); }