## Tech Stack
-[Bun](https://bun.sh) · [Hono](https://hono.dev) · [GLiNER](https://github.com/urchade/GLiNER) + [python-stdnum](https://arthurdejong.org/python-stdnum/) ([`detector/`](detector/)) · SQLite
+[Bun](https://bun.sh) · [Hono](https://hono.dev) · [GLiNER](https://github.com/urchade/GLiNER) + [python-stdnum](https://arthurdejong.org/python-stdnum/) ([`detector/`](detector/)) · SQLite or Postgres
## Contributing
"@hono/zod-validator": "^0.7.6",
"hono": "^4.12.23",
"hono-tailwind": "^2.2.0",
+ "kysely": "^0.29.2",
+ "pg": "^8.22.0",
"tailwindcss": "^4.3.0",
"yaml": "^2.9.0",
"zod": "^3.25.76",
"devDependencies": {
"@biomejs/biome": "^2.4.16",
"@types/bun": "latest",
+ "@types/pg": "^8.20.0",
"typescript": "^5.9.3",
},
},
"@types/node": ["@types/node@25.0.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA=="],
+ "@types/pg": ["@types/pg@8.20.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow=="],
+
"bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
"jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="],
+ "kysely": ["kysely@0.29.2", "", {}, "sha512-s6WVJyEZrbm6jhBpiKHsGHyePMrVQKJ85wZCFCr9W4QHv6WTjWIrdvTmO9hDEA3bNK0xkrE2DqrHsXMLWuZpQg=="],
+
"lightningcss": ["lightningcss@1.30.2", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.30.2", "lightningcss-darwin-arm64": "1.30.2", "lightningcss-darwin-x64": "1.30.2", "lightningcss-freebsd-x64": "1.30.2", "lightningcss-linux-arm-gnueabihf": "1.30.2", "lightningcss-linux-arm64-gnu": "1.30.2", "lightningcss-linux-arm64-musl": "1.30.2", "lightningcss-linux-x64-gnu": "1.30.2", "lightningcss-linux-x64-musl": "1.30.2", "lightningcss-win32-arm64-msvc": "1.30.2", "lightningcss-win32-x64-msvc": "1.30.2" } }, "sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ=="],
"lightningcss-android-arm64": ["lightningcss-android-arm64@1.30.2", "", { "os": "android", "cpu": "arm64" }, "sha512-BH9sEdOCahSgmkVhBLeU7Hc9DWeZ1Eb6wNS6Da8igvUwAe0sqROHddIlvU06q3WyXVEOYDZ6ykBZQnjTbmo4+A=="],
"nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="],
+ "pg": ["pg@8.22.0", "", { "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", "pg-protocol": "^1.15.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA=="],
+
+ "pg-cloudflare": ["pg-cloudflare@1.4.0", "", {}, "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A=="],
+
+ "pg-connection-string": ["pg-connection-string@2.14.0", "", {}, "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg=="],
+
+ "pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="],
+
+ "pg-pool": ["pg-pool@3.14.0", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw=="],
+
+ "pg-protocol": ["pg-protocol@1.15.0", "", {}, "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ=="],
+
+ "pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="],
+
+ "pgpass": ["pgpass@1.0.5", "", { "dependencies": { "split2": "^4.1.0" } }, "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug=="],
+
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
"postcss": ["postcss@8.5.15", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A=="],
+ "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="],
+
+ "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="],
+
+ "postgres-date": ["postgres-date@1.0.7", "", {}, "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="],
+
+ "postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="],
+
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
+ "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
+
"tailwindcss": ["tailwindcss@4.3.0", "", {}, "sha512-y6nxMGB1nMW9R6k96e5gdIFzcfL/gTJRNaqGes1YvkLnPVXzWgbqFF2yLC0T8G774n24cx3Pe8XrKoniCOAH+Q=="],
"tapable": ["tapable@2.3.0", "", {}, "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg=="],
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
+ "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="],
+
"yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="],
"zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
# Logging settings
logging:
- # SQLite database for request logs
+ # Storage backend for request logs: "sqlite" (default) or "postgres"
+ driver: sqlite
+
+ # SQLite database for request logs (used when driver is "sqlite")
database: ./data/pasteguard.db
+ # Postgres connection string (required when driver is "postgres")
+ # postgres_url: ${POSTGRES_URL:-postgres://pasteguard:pasteguard@localhost:5432/pasteguard}
+
# Log retention in days (0 = keep forever)
retention_days: 30
```yaml
logging:
+ driver: sqlite
database: ./data/pasteguard.db
retention_days: 30
log_masked_content: true
| Option | Default | Description |
|--------|---------|-------------|
-| `database` | `./data/pasteguard.db` | SQLite database path |
+| `driver` | `sqlite` | Storage backend: `sqlite` or `postgres` |
+| `database` | `./data/pasteguard.db` | SQLite database path (used when `driver` is `sqlite`) |
+| `postgres_url` | — | Postgres connection string (required when `driver` is `postgres`) |
| `retention_days` | `30` | Days to keep logs. `0` = forever |
| `log_masked_content` | `true` | Store request text in the dashboard after masking |
## Database
-Logs are stored in SQLite:
+Both backends share the same schema; the dashboard works identically on either. The schema is
+created and migrated automatically on startup.
+
+### SQLite (default)
```yaml
logging:
+ driver: sqlite
database: ./data/pasteguard.db
```
- ./data:/pasteguard/data
```
+### Postgres
+
+Use Postgres for shared or higher-volume deployments:
+
+```yaml
+logging:
+ driver: postgres
+ postgres_url: ${POSTGRES_URL:-postgres://pasteguard:pasteguard@localhost:5432/pasteguard}
+```
+
+`postgres_url` is required when `driver` is `postgres`; startup fails with a clear error if it is
+missing. The value supports environment-variable substitution, so you can keep credentials out of
+the config file.
+
+Existing SQLite databases are not migrated to Postgres automatically — switching `driver` starts
+logging into the new backend. An existing SQLite database created by an older PasteGuard version is
+upgraded in place the first time the new version starts.
+
## Retention
Logs older than `retention_days` are automatically deleted:
"@hono/zod-validator": "^0.7.6",
"hono": "^4.12.23",
"hono-tailwind": "^2.2.0",
+ "kysely": "^0.29.2",
+ "pg": "^8.22.0",
"tailwindcss": "^4.3.0",
"yaml": "^2.9.0",
"zod": "^3.25.76"
"devDependencies": {
"@biomejs/biome": "^2.4.16",
"@types/bun": "latest",
+ "@types/pg": "^8.20.0",
"typescript": "^5.9.3"
},
"keywords": [
}
});
+ test("defaults request logging to SQLite", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+`);
+
+ try {
+ const config = loadConfig(path);
+
+ expect(config.logging.driver).toBe("sqlite");
+ expect(config.logging.database).toBe("./data/pasteguard.db");
+ expect(config.logging.postgres_url).toBeUndefined();
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
+ test("accepts Postgres request logging config", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+logging:
+ driver: postgres
+ postgres_url: \${POSTGRES_URL:-postgres://pasteguard:pasteguard@localhost:5432/pasteguard}
+`);
+
+ try {
+ const config = loadConfig(path);
+
+ expect(config.logging.driver).toBe("postgres");
+ expect(config.logging.postgres_url).toBe(
+ "postgres://pasteguard:pasteguard@localhost:5432/pasteguard",
+ );
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
+ test("requires postgres_url for Postgres request logging", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+logging:
+ driver: postgres
+`);
+
+ try {
+ expect(() => loadConfig(path)).toThrow("Invalid configuration");
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
test("rejects invalid phone region codes", () => {
const path = writeConfig(`
mode: mask
request_timeout: z.coerce.number().int().min(0).default(600),
});
-const LoggingSchema = z.object({
- database: z.string().default("./data/pasteguard.db"),
- retention_days: z.coerce.number().int().min(0).default(30),
- log_masked_content: z.boolean().default(true),
-});
+const LoggingSchema = z
+ .object({
+ driver: z.enum(["sqlite", "postgres"]).default("sqlite"),
+ database: z.string().default("./data/pasteguard.db"),
+ postgres_url: z.string().optional(),
+ retention_days: z.coerce.number().int().min(0).default(30),
+ log_masked_content: z.boolean().default(true),
+ })
+ .refine((logging) => logging.driver !== "postgres" || Boolean(logging.postgres_url), {
+ path: ["postgres_url"],
+ message: "logging.postgres_url is required when logging.driver is 'postgres'",
+ });
const DashboardAuthSchema = z.object({
username: z.string(),
};
// Startup validation
-validateStartup().then(() => {
+validateStartup().then(async () => {
printStartupBanner(config, host, port);
- const stopCleanup = startCleanupScheduler(config);
+ const stopCleanup = await startCleanupScheduler(config);
setupGracefulShutdown(stopCleanup);
});
`);
}
-function startCleanupScheduler(config: ReturnType<typeof getConfig>): () => void {
+async function startCleanupScheduler(config: ReturnType<typeof getConfig>): Promise<() => void> {
let cleanupInterval: ReturnType<typeof setInterval> | null = null;
if (config.logging.retention_days > 0) {
// Run cleanup on startup
try {
- const deleted = logger.cleanup();
+ const deleted = await logger.cleanup();
if (deleted > 0) {
console.log(
`Log cleanup: removed ${deleted} entries older than ${config.logging.retention_days} days`,
// Schedule daily cleanup
cleanupInterval = setInterval(
() => {
- try {
- const count = logger.cleanup();
- if (count > 0) {
- console.log(
- `Log cleanup: removed ${count} entries older than ${config.logging.retention_days} days`,
- );
+ void (async () => {
+ try {
+ const count = await logger.cleanup();
+ if (count > 0) {
+ console.log(
+ `Log cleanup: removed ${count} entries older than ${config.logging.retention_days} days`,
+ );
+ }
+ } catch (error) {
+ console.error("Log cleanup failed:", error);
}
- } catch (error) {
- console.error("Log cleanup failed:", error);
- }
+ })();
},
24 * 60 * 60 * 1000,
);
function shutdown() {
console.log("\nShutting down...");
stopCleanup();
- try {
- getLogger().close();
- } catch {
- // Logger might not be initialized
- }
- process.exit(0);
+ void (async () => {
+ try {
+ await getLogger().close();
+ } catch {
+ // Logger might not be initialized
+ }
+ process.exit(0);
+ })();
}
process.on("SIGTERM", shutdown);
--- /dev/null
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { type Config, loadConfig } from "../config";
+import { createLogDatabase, type LogKysely, migrateLogDatabase } from "./db";
+
+function legacyConfig(): { config: Config; dbPath: string; dir: string } {
+ const dir = mkdtempSync(join(tmpdir(), "pasteguard-db-test-"));
+ const path = join(dir, "config.yaml");
+ const dbPath = join(dir, "legacy.db");
+ writeFileSync(
+ path,
+ `
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+logging:
+ driver: sqlite
+ database: ${dbPath}
+`,
+ );
+ return { config: loadConfig(path), dbPath, dir };
+}
+
+// Simulates a database created by an older PasteGuard version: request_logs
+// exists but predates the source/secrets/status columns and the Kysely migrator.
+function createLegacyDatabase(dbPath: string): void {
+ const raw = new Database(dbPath);
+ raw.run(`CREATE TABLE request_logs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ timestamp TEXT NOT NULL,
+ mode TEXT NOT NULL DEFAULT 'route',
+ provider TEXT NOT NULL,
+ model TEXT NOT NULL,
+ pii_detected INTEGER NOT NULL DEFAULT 0,
+ entities TEXT,
+ latency_ms INTEGER NOT NULL,
+ scan_time_ms INTEGER NOT NULL DEFAULT 0,
+ prompt_tokens INTEGER,
+ completion_tokens INTEGER,
+ user_agent TEXT,
+ masked_content TEXT,
+ created_at TEXT DEFAULT CURRENT_TIMESTAMP
+ )`);
+ raw.run(
+ `INSERT INTO request_logs (timestamp, mode, provider, model, pii_detected, entities, latency_ms, scan_time_ms)
+ VALUES ('2026-06-28T10:00:00.000Z', 'mask', 'openai', 'old-model', 1, 'EMAIL_ADDRESS', 50, 5)`,
+ );
+ raw.close();
+}
+
+async function columnNames(db: LogKysely): Promise<Set<string>> {
+ const tables = await db.introspection.getTables({ withInternalKyselyTables: true });
+ const table = tables.find((t) => t.name === "request_logs");
+ return new Set((table?.columns ?? []).map((c) => c.name));
+}
+
+describe("migrateLogDatabase legacy SQLite upgrade", () => {
+ test("adds missing columns, backfills source, and baselines the migrator", async () => {
+ const { config, dbPath, dir } = legacyConfig();
+ createLegacyDatabase(dbPath);
+
+ try {
+ const { db, driver } = createLogDatabase(config);
+ await migrateLogDatabase(db, driver);
+
+ // Columns the legacy schema was missing are now present.
+ const columns = await columnNames(db);
+ for (const column of [
+ "source",
+ "secrets_detected",
+ "secrets_types",
+ "status_code",
+ "error_message",
+ ]) {
+ expect(columns.has(column)).toBe(true);
+ }
+
+ // The pre-existing row's NULL source was backfilled from provider.
+ const [row] = await db.selectFrom("request_logs").select(["source", "provider"]).execute();
+ expect(row.source).toBe("openai");
+ expect(row.provider).toBe("openai");
+
+ // The Kysely migrator is baselined so 0001 is treated as already applied.
+ const migrations = await db
+ .selectFrom("kysely_migration" as never)
+ .select("name" as never)
+ .execute();
+ expect(migrations.map((m) => (m as { name: string }).name)).toContain("0001_request_logs");
+
+ await db.destroy();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test("is idempotent across repeated startups", async () => {
+ const { config, dbPath, dir } = legacyConfig();
+ createLegacyDatabase(dbPath);
+
+ try {
+ for (let startup = 0; startup < 3; startup++) {
+ const { db, driver } = createLogDatabase(config);
+ await migrateLogDatabase(db, driver);
+ await db.destroy();
+ }
+
+ // The original row survives and is still the only one.
+ const { db } = createLogDatabase(config);
+ const rows = await db.selectFrom("request_logs").select("id").execute();
+ expect(rows).toHaveLength(1);
+ await db.destroy();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
--- /dev/null
+import { Database, type SQLQueryBindings } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import {
+ type ColumnType,
+ type Generated,
+ Kysely,
+ PostgresDialect,
+ type SqliteDatabase,
+ SqliteDialect,
+ type SqliteStatement,
+ sql,
+} from "kysely";
+import { type Migration, type MigrationProvider, Migrator } from "kysely/migration";
+import { Pool } from "pg";
+import type { Config } from "../config";
+
+export type LoggingDriver = "sqlite" | "postgres";
+
+export interface RequestLogsTable {
+ id: Generated<number>;
+ timestamp: string;
+ mode: string;
+ provider: string;
+ source: string | null;
+ model: string;
+ pii_detected: number;
+ entities: string | null;
+ latency_ms: number;
+ scan_time_ms: number;
+ prompt_tokens: number | null;
+ completion_tokens: number | null;
+ user_agent: string | null;
+ masked_content: string | null;
+ secrets_detected: number | null;
+ secrets_types: string | null;
+ status_code: number | null;
+ error_message: string | null;
+ created_at: ColumnType<string | null, string | undefined, never>;
+}
+
+export interface LogDatabase {
+ request_logs: RequestLogsTable;
+}
+
+interface MigrationDatabase extends LogDatabase {
+ kysely_migration: {
+ name: string;
+ timestamp: string;
+ };
+}
+
+export type LogKysely = Kysely<LogDatabase>;
+
+class BunSqliteDatabase implements SqliteDatabase {
+ constructor(private readonly db: Database) {}
+
+ close(): void {
+ this.db.close();
+ }
+
+ prepare(query: string): SqliteStatement {
+ const statement = this.db.prepare<unknown, SQLQueryBindings[]>(query);
+ const reader = /^(select|pragma|with)\b/i.test(query.trim());
+
+ return {
+ get reader() {
+ return reader;
+ },
+ all(parameters: ReadonlyArray<unknown>) {
+ return statement.all(...(parameters as SQLQueryBindings[]));
+ },
+ run(parameters: ReadonlyArray<unknown>) {
+ const result = statement.run(...(parameters as SQLQueryBindings[]));
+ return {
+ changes: result.changes,
+ lastInsertRowid: result.lastInsertRowid,
+ };
+ },
+ iterate(parameters: ReadonlyArray<unknown>) {
+ return statement.iterate(...(parameters as SQLQueryBindings[]));
+ },
+ };
+ }
+}
+
+export function createLogDatabase(config: Config): { db: LogKysely; driver: LoggingDriver } {
+ if (config.logging.driver === "postgres") {
+ return {
+ driver: "postgres",
+ db: new Kysely<LogDatabase>({
+ dialect: new PostgresDialect({
+ pool: new Pool({ connectionString: config.logging.postgres_url! }),
+ }),
+ }),
+ };
+ }
+
+ const dbPath = config.logging.database;
+ const dir = dirname(dbPath);
+ if (dir && dir !== ".") {
+ mkdirSync(dir, { recursive: true });
+ }
+
+ return {
+ driver: "sqlite",
+ db: new Kysely<LogDatabase>({
+ dialect: new SqliteDialect({
+ database: new BunSqliteDatabase(new Database(dbPath)),
+ }),
+ }),
+ };
+}
+
+export async function migrateLogDatabase(db: LogKysely, driver: LoggingDriver): Promise<void> {
+ await baselineExistingRequestLogs(db);
+
+ const migrator = new Migrator({
+ db,
+ provider: new InlineMigrationProvider({
+ "0001_request_logs": createRequestLogsMigration(driver),
+ }),
+ });
+
+ const result = await migrator.migrateToLatest();
+ if (result.error) {
+ throw result.error;
+ }
+}
+
+class InlineMigrationProvider implements MigrationProvider {
+ constructor(private readonly migrations: Record<string, Migration>) {}
+
+ async getMigrations(): Promise<Record<string, Migration>> {
+ return this.migrations;
+ }
+}
+
+function createRequestLogsMigration(driver: LoggingDriver): Migration {
+ return {
+ async up(db) {
+ let createTable = db.schema.createTable("request_logs").ifNotExists();
+
+ createTable =
+ driver === "postgres"
+ ? createTable.addColumn("id", "serial", (column) => column.primaryKey())
+ : createTable.addColumn("id", "integer", (column) => column.primaryKey().autoIncrement());
+
+ await createTable
+ .addColumn("timestamp", "text", (column) => column.notNull())
+ .addColumn("mode", "text", (column) => column.notNull().defaultTo("route"))
+ .addColumn("provider", "text", (column) => column.notNull())
+ .addColumn("source", "text")
+ .addColumn("model", "text", (column) => column.notNull())
+ .addColumn("pii_detected", "integer", (column) => column.notNull().defaultTo(0))
+ .addColumn("entities", "text")
+ .addColumn("latency_ms", "integer", (column) => column.notNull())
+ .addColumn("scan_time_ms", "integer", (column) => column.notNull().defaultTo(0))
+ .addColumn("prompt_tokens", "integer")
+ .addColumn("completion_tokens", "integer")
+ .addColumn("user_agent", "text")
+ .addColumn("masked_content", "text")
+ .addColumn("secrets_detected", "integer")
+ .addColumn("secrets_types", "text")
+ .addColumn("status_code", "integer")
+ .addColumn("error_message", "text")
+ .addColumn("created_at", "text", (column) => column.defaultTo(sql`CURRENT_TIMESTAMP`))
+ .execute();
+
+ await createRequestLogIndexes(db);
+ },
+ };
+}
+
+async function baselineExistingRequestLogs(db: LogKysely): Promise<void> {
+ const tables = await db.introspection.getTables({ withInternalKyselyTables: true });
+ const requestLogsTable = tables.find((table) => table.name === "request_logs");
+ const migrationTable = tables.find((table) => table.name === "kysely_migration");
+
+ if (!requestLogsTable || migrationTable) {
+ return;
+ }
+
+ const columns = new Set(requestLogsTable.columns.map((column) => column.name));
+ await addLegacyColumnIfMissing(db, columns, "source", "text");
+ await addLegacyColumnIfMissing(db, columns, "secrets_detected", "integer");
+ await addLegacyColumnIfMissing(db, columns, "secrets_types", "text");
+ await addLegacyColumnIfMissing(db, columns, "status_code", "integer");
+ await addLegacyColumnIfMissing(db, columns, "error_message", "text");
+
+ await db
+ .updateTable("request_logs")
+ .set({ source: sql<string>`provider` })
+ .where("source", "is", null)
+ .execute();
+ await createRequestLogIndexes(db);
+ await createKyselyMigrationBaseline(db as unknown as Kysely<MigrationDatabase>);
+}
+
+async function addLegacyColumnIfMissing(
+ db: LogKysely,
+ columns: Set<string>,
+ name: string,
+ type: "integer" | "text",
+): Promise<void> {
+ if (columns.has(name)) {
+ return;
+ }
+
+ await db.schema.alterTable("request_logs").addColumn(name, type).execute();
+ columns.add(name);
+}
+
+async function createRequestLogIndexes(db: LogKysely): Promise<void> {
+ await db.schema
+ .createIndex("idx_timestamp")
+ .ifNotExists()
+ .on("request_logs")
+ .column("timestamp")
+ .execute();
+ await db.schema
+ .createIndex("idx_provider")
+ .ifNotExists()
+ .on("request_logs")
+ .column("provider")
+ .execute();
+ await db.schema
+ .createIndex("idx_pii_detected")
+ .ifNotExists()
+ .on("request_logs")
+ .column("pii_detected")
+ .execute();
+}
+
+async function createKyselyMigrationBaseline(db: Kysely<MigrationDatabase>): Promise<void> {
+ await db.schema
+ .createTable("kysely_migration")
+ .ifNotExists()
+ .addColumn("name", "varchar(255)", (column) => column.notNull().primaryKey())
+ .addColumn("timestamp", "varchar(255)", (column) => column.notNull())
+ .execute();
+
+ await db
+ .insertInto("kysely_migration")
+ .values({
+ name: "0001_request_logs",
+ timestamp: new Date().toISOString(),
+ })
+ .execute();
+}
import { describe, expect, test } from "bun:test";
-import { normalizeRequestSource } from "./logger";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { sql } from "kysely";
+import { type Config, loadConfig } from "../config";
+import { createLogDatabase } from "./db";
+import { Logger, normalizeRequestSource, type RequestLog } from "./logger";
+
+function writeConfig(): { path: string; dir: string } {
+ const dir = mkdtempSync(join(tmpdir(), "pasteguard-logger-test-"));
+ const path = join(dir, "config.yaml");
+ const database = join(dir, "pasteguard.db");
+ writeFileSync(
+ path,
+ `
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+logging:
+ driver: sqlite
+ database: ${database}
+ retention_days: 30
+`,
+ );
+ return { path, dir };
+}
+
+function createLog(overrides: Partial<Omit<RequestLog, "id">> = {}): Omit<RequestLog, "id"> {
+ return {
+ timestamp: new Date().toISOString(),
+ mode: "mask",
+ provider: "openai",
+ source: "openai",
+ model: "gpt-test",
+ pii_detected: true,
+ entities: "EMAIL_ADDRESS,PERSON",
+ latency_ms: 120,
+ scan_time_ms: 12,
+ prompt_tokens: 10,
+ completion_tokens: 20,
+ user_agent: "test-agent",
+ masked_content: "hello [[EMAIL_ADDRESS_1]]",
+ secrets_detected: 1,
+ secrets_types: "API_KEY_SK",
+ status_code: 200,
+ error_message: null,
+ ...overrides,
+ };
+}
describe("normalizeRequestSource", () => {
test("uses provider as source for provider-backed requests", () => {
expect(normalizeRequestSource("api", "browser-extension")).toBe("browser_extension");
});
});
+
+describe("Logger SQLite backend", () => {
+ test("logs dashboard rows, stats, and entity stats", async () => {
+ const { path, dir } = writeConfig();
+
+ try {
+ const config = loadConfig(path);
+ const logger = new Logger({ config });
+
+ await logger.log(createLog());
+ await logger.log(
+ createLog({
+ timestamp: new Date(Date.now() - 5_000).toISOString(),
+ provider: "api",
+ source: "browser_extension",
+ pii_detected: false,
+ entities: "",
+ prompt_tokens: null,
+ completion_tokens: null,
+ secrets_detected: 0,
+ secrets_types: null,
+ }),
+ );
+
+ const logs = await logger.getLogs();
+ expect(logs).toHaveLength(2);
+ expect(logs[0].source).toBe("openai");
+ expect(logs[0].pii_detected).toBe(1);
+ expect(logs[0].secrets_detected).toBe(1);
+
+ const stats = await logger.getStats();
+ expect(stats.total_requests).toBe(2);
+ expect(stats.pii_requests).toBe(1);
+ expect(stats.browser_extension_requests).toBe(1);
+ expect(stats.total_tokens).toBe(30);
+ expect(stats.avg_scan_time_ms).toBe(12);
+
+ const entityStats = await logger.getEntityStats();
+ expect(entityStats).toEqual([
+ { entity: "EMAIL_ADDRESS", count: 1 },
+ { entity: "PERSON", count: 1 },
+ ]);
+
+ await logger.close();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test("cleans up rows outside retention", async () => {
+ const { path, dir } = writeConfig();
+
+ try {
+ const baseConfig = loadConfig(path);
+ const config = {
+ ...baseConfig,
+ logging: {
+ ...baseConfig.logging,
+ retention_days: 1,
+ },
+ };
+ const logger = new Logger({ config });
+
+ await logger.log(
+ createLog({
+ timestamp: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
+ }),
+ );
+ await logger.log(createLog());
+
+ expect(await logger.cleanup()).toBe(1);
+ expect(await logger.getLogs()).toHaveLength(1);
+
+ await logger.close();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test("keeps rows forever when retention is disabled", async () => {
+ const { path, dir } = writeConfig();
+
+ try {
+ const baseConfig = loadConfig(path);
+ const config = {
+ ...baseConfig,
+ logging: {
+ ...baseConfig.logging,
+ retention_days: 0,
+ },
+ };
+ const logger = new Logger({ config });
+
+ await logger.log(
+ createLog({
+ timestamp: new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toISOString(),
+ }),
+ );
+
+ expect(await logger.cleanup()).toBe(0);
+ expect(await logger.getLogs()).toHaveLength(1);
+
+ await logger.close();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test("paginates logs by timestamp with limit and offset", async () => {
+ const { path, dir } = writeConfig();
+
+ try {
+ const config = loadConfig(path);
+ const logger = new Logger({ config });
+
+ await logger.log(
+ createLog({ model: "oldest", timestamp: new Date(Date.now() - 2_000).toISOString() }),
+ );
+ await logger.log(
+ createLog({ model: "middle", timestamp: new Date(Date.now() - 1_000).toISOString() }),
+ );
+ await logger.log(createLog({ model: "newest", timestamp: new Date().toISOString() }));
+
+ const firstPage = await logger.getLogs(1, 0);
+ expect(firstPage).toHaveLength(1);
+ expect(firstPage[0].model).toBe("newest");
+
+ const secondPage = await logger.getLogs(1, 1);
+ expect(secondPage).toHaveLength(1);
+ expect(secondPage[0].model).toBe("middle");
+
+ await logger.close();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
+
+describe("Postgres logging config", () => {
+ test("selects the Postgres Kysely backend", () => {
+ const configPath = writeConfig();
+
+ try {
+ const baseConfig = loadConfig(configPath.path);
+ const { driver, db } = createLogDatabase({
+ ...baseConfig,
+ logging: {
+ ...baseConfig.logging,
+ driver: "postgres",
+ postgres_url: "postgres://pasteguard:pasteguard@localhost:5432/pasteguard",
+ },
+ });
+
+ expect(driver).toBe("postgres");
+ void db.destroy();
+ } finally {
+ rmSync(configPath.dir, { recursive: true, force: true });
+ }
+ });
+});
+
+// Live Postgres round-trip. Skipped unless PASTEGUARD_TEST_POSTGRES_URL points at a
+// throwaway database, e.g.
+// PASTEGUARD_TEST_POSTGRES_URL=postgres://test:test@localhost:5432/test bun test
+const POSTGRES_URL = process.env.PASTEGUARD_TEST_POSTGRES_URL;
+
+function postgresConfig(): Config {
+ const { path, dir } = writeConfig();
+ try {
+ const baseConfig = loadConfig(path);
+ return {
+ ...baseConfig,
+ logging: {
+ ...baseConfig.logging,
+ driver: "postgres",
+ postgres_url: POSTGRES_URL,
+ },
+ };
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+async function resetPostgres(config: Config): Promise<void> {
+ const { db } = createLogDatabase(config);
+ await sql`DROP TABLE IF EXISTS request_logs, kysely_migration, kysely_migration_lock CASCADE`.execute(
+ db,
+ );
+ await db.destroy();
+}
+
+describe.skipIf(!POSTGRES_URL)("Logger Postgres backend", () => {
+ test("migrates, logs rows, aggregates stats, and cleans up against Postgres", async () => {
+ const config = postgresConfig();
+ await resetPostgres(config);
+
+ const logger = new Logger({ config });
+
+ try {
+ await logger.log(createLog());
+ await logger.log(
+ createLog({
+ timestamp: new Date(Date.now() - 5_000).toISOString(),
+ provider: "api",
+ source: "browser_extension",
+ pii_detected: false,
+ entities: "",
+ prompt_tokens: null,
+ completion_tokens: null,
+ secrets_detected: 0,
+ secrets_types: null,
+ }),
+ );
+
+ const logs = await logger.getLogs();
+ expect(logs).toHaveLength(2);
+ expect(logs[0].source).toBe("openai");
+ expect(logs[0].pii_detected).toBe(1);
+ expect(logs[0].secrets_detected).toBe(1);
+ // Postgres returns counts as strings/bigints; ensure they are coerced to numbers.
+ expect(typeof logs[0].id).toBe("number");
+
+ const stats = await logger.getStats();
+ expect(stats.total_requests).toBe(2);
+ expect(stats.pii_requests).toBe(1);
+ expect(stats.browser_extension_requests).toBe(1);
+ expect(stats.total_tokens).toBe(30);
+ expect(stats.avg_scan_time_ms).toBe(12);
+
+ const entityStats = await logger.getEntityStats();
+ expect(entityStats).toEqual([
+ { entity: "EMAIL_ADDRESS", count: 1 },
+ { entity: "PERSON", count: 1 },
+ ]);
+
+ // Inserting an old row and cleaning up should delete exactly that row.
+ await logger.log(
+ createLog({
+ timestamp: new Date(Date.now() - 60 * 24 * 60 * 60 * 1000).toISOString(),
+ }),
+ );
+ expect(await logger.cleanup()).toBe(1);
+ expect(await logger.getLogs()).toHaveLength(2);
+ } finally {
+ await logger.close();
+ await resetPostgres(config);
+ }
+ });
+});
-import { Database } from "bun:sqlite";
-import { mkdirSync } from "node:fs";
-import { getConfig } from "../config";
+import { type Selectable, type SelectQueryBuilder, sql } from "kysely";
+import { type Config, getConfig } from "../config";
+import {
+ createLogDatabase,
+ type LogDatabase,
+ type LogKysely,
+ migrateLogDatabase,
+ type RequestLogsTable,
+} from "./db";
import { shouldLogMaskedContent } from "./log-content";
export type RequestProvider = "openai" | "anthropic" | "codex" | "local" | "api";
provider: RequestProvider;
source: RequestSource;
model: string;
- pii_detected: boolean;
+ pii_detected: boolean | 0 | 1;
entities: string;
latency_ms: number;
scan_time_ms: number;
requests_last_hour: number;
}
+type CountRow = { count: number | string | bigint };
+type AverageRow = { avg: number | string | null };
+type TotalRow = { total: number | string | bigint | null };
+type CountQuery = SelectQueryBuilder<LogDatabase, "request_logs", { count: number }>;
+
export function normalizeRequestSource(
provider: RequestProvider,
sourceHeader?: string | null,
return "api";
}
+function toNumber(value: number | string | bigint | null | undefined): number {
+ if (value === null || value === undefined) return 0;
+ return Number(value);
+}
+
+function toStoredFlag(value: boolean | 0 | 1): 0 | 1 {
+ return value === true || value === 1 ? 1 : 0;
+}
+
+// The dashboard row shape is whatever getLogs selects: every request_logs
+// column except the unused created_at. Derive it from the schema so the two
+// can't drift.
+type RequestLogRow = Omit<Selectable<RequestLogsTable>, "created_at">;
+
+function normalizeLogRow(row: RequestLogRow): RequestLog {
+ const provider = row.provider as RequestProvider;
+
+ return {
+ id: toNumber(row.id),
+ timestamp: row.timestamp,
+ mode: row.mode as "route" | "mask",
+ provider,
+ source: (row.source as RequestSource | null) || normalizeRequestSource(provider),
+ model: row.model,
+ pii_detected: toStoredFlag(row.pii_detected as 0 | 1),
+ entities: row.entities ?? "",
+ latency_ms: toNumber(row.latency_ms),
+ scan_time_ms: toNumber(row.scan_time_ms),
+ prompt_tokens: row.prompt_tokens === null ? null : toNumber(row.prompt_tokens),
+ completion_tokens: row.completion_tokens === null ? null : toNumber(row.completion_tokens),
+ user_agent: row.user_agent,
+ masked_content: row.masked_content,
+ secrets_detected: row.secrets_detected === null ? null : toNumber(row.secrets_detected),
+ secrets_types: row.secrets_types,
+ status_code: row.status_code === null ? null : toNumber(row.status_code),
+ error_message: row.error_message,
+ };
+}
+
+function buildStats(values: {
+ total: number | string | bigint;
+ pii: number | string | bigint;
+ proxy: number | string | bigint;
+ local: number | string | bigint;
+ api: number | string | bigint;
+ browserExtension: number | string | bigint;
+ avgScanTime: number | string | null;
+ totalTokens: number | string | bigint | null;
+ requestsLastHour: number | string | bigint;
+}): Stats {
+ const total = toNumber(values.total);
+ const pii = toNumber(values.pii);
+
+ return {
+ total_requests: total,
+ pii_requests: pii,
+ pii_percentage: total > 0 ? Math.round((pii / total) * 100 * 10) / 10 : 0,
+ proxy_requests: toNumber(values.proxy),
+ local_requests: toNumber(values.local),
+ api_requests: toNumber(values.api),
+ browser_extension_requests: toNumber(values.browserExtension),
+ avg_scan_time_ms: Math.round(toNumber(values.avgScanTime)),
+ total_tokens: toNumber(values.totalTokens),
+ requests_last_hour: toNumber(values.requestsLastHour),
+ };
+}
+
export class Logger {
- private db: Database;
+ private db: LogKysely;
+ private ready: Promise<void>;
private retentionDays: number;
- constructor() {
- const config = getConfig();
+ constructor(options: { config?: Config; db?: LogKysely } = {}) {
+ const config = options.config ?? getConfig();
this.retentionDays = config.logging.retention_days;
- // Ensure data directory exists
- const dbPath = config.logging.database;
- const dir = dbPath.substring(0, dbPath.lastIndexOf("/"));
- if (dir) {
- mkdirSync(dir, { recursive: true });
+ if (options.db) {
+ this.db = options.db;
+ this.ready = Promise.resolve();
+ } else {
+ const { db, driver } = createLogDatabase(config);
+ this.db = db;
+ this.ready = migrateLogDatabase(db, driver);
}
-
- this.db = new Database(dbPath);
- this.initializeDatabase();
}
- private initializeDatabase(): void {
- this.db.run(`
- CREATE TABLE IF NOT EXISTS request_logs (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- timestamp TEXT NOT NULL,
- mode TEXT NOT NULL DEFAULT 'route',
- provider TEXT NOT NULL,
- source TEXT,
- model TEXT NOT NULL,
- pii_detected INTEGER NOT NULL DEFAULT 0,
- entities TEXT,
- latency_ms INTEGER NOT NULL,
- scan_time_ms INTEGER NOT NULL DEFAULT 0,
- prompt_tokens INTEGER,
- completion_tokens INTEGER,
- user_agent TEXT,
- masked_content TEXT,
- secrets_detected INTEGER,
- secrets_types TEXT,
- created_at TEXT DEFAULT CURRENT_TIMESTAMP
- )
- `);
-
- // Migrate existing databases: add missing columns
- const columns = this.db.prepare("PRAGMA table_info(request_logs)").all() as Array<{
- name: string;
- }>;
- if (!columns.find((c) => c.name === "secrets_detected")) {
- this.db.run("ALTER TABLE request_logs ADD COLUMN secrets_detected INTEGER");
- this.db.run("ALTER TABLE request_logs ADD COLUMN secrets_types TEXT");
- }
- if (!columns.find((c) => c.name === "status_code")) {
- this.db.run("ALTER TABLE request_logs ADD COLUMN status_code INTEGER");
- this.db.run("ALTER TABLE request_logs ADD COLUMN error_message TEXT");
- }
- if (!columns.find((c) => c.name === "source")) {
- this.db.run("ALTER TABLE request_logs ADD COLUMN source TEXT");
- this.db.run("UPDATE request_logs SET source = provider WHERE source IS NULL");
- }
-
- // Create indexes for performance
- this.db.run(`
- CREATE INDEX IF NOT EXISTS idx_timestamp ON request_logs(timestamp)
- `);
- this.db.run(`
- CREATE INDEX IF NOT EXISTS idx_provider ON request_logs(provider)
- `);
- this.db.run(`
- CREATE INDEX IF NOT EXISTS idx_pii_detected ON request_logs(pii_detected)
- `);
+ async log(entry: Omit<RequestLog, "id">): Promise<void> {
+ await this.ready;
+ await this.db
+ .insertInto("request_logs")
+ .values({
+ timestamp: entry.timestamp,
+ mode: entry.mode,
+ provider: entry.provider,
+ source: entry.source,
+ model: entry.model,
+ pii_detected: toStoredFlag(entry.pii_detected),
+ entities: entry.entities,
+ latency_ms: entry.latency_ms,
+ scan_time_ms: entry.scan_time_ms,
+ prompt_tokens: entry.prompt_tokens,
+ completion_tokens: entry.completion_tokens,
+ user_agent: entry.user_agent,
+ masked_content: entry.masked_content,
+ secrets_detected: entry.secrets_detected ?? null,
+ secrets_types: entry.secrets_types ?? null,
+ status_code: entry.status_code ?? null,
+ error_message: entry.error_message ?? null,
+ })
+ .execute();
}
- log(entry: Omit<RequestLog, "id">): void {
- const stmt = this.db.prepare(`
- INSERT INTO request_logs
- (timestamp, mode, provider, source, model, pii_detected, entities, latency_ms, scan_time_ms, prompt_tokens, completion_tokens, user_agent, masked_content, secrets_detected, secrets_types, status_code, error_message)
- VALUES
- (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `);
-
- stmt.run(
- entry.timestamp,
- entry.mode,
- entry.provider,
- entry.source,
- entry.model,
- entry.pii_detected ? 1 : 0,
- entry.entities,
- entry.latency_ms,
- entry.scan_time_ms,
- entry.prompt_tokens,
- entry.completion_tokens,
- entry.user_agent,
- entry.masked_content,
- entry.secrets_detected ?? null,
- entry.secrets_types ?? null,
- entry.status_code ?? null,
- entry.error_message ?? null,
- );
+ async getLogs(limit: number = 100, offset: number = 0): Promise<RequestLog[]> {
+ await this.ready;
+ const logs = await this.db
+ .selectFrom("request_logs")
+ .select([
+ "id",
+ "timestamp",
+ "mode",
+ "provider",
+ "source",
+ "model",
+ "pii_detected",
+ "entities",
+ "latency_ms",
+ "scan_time_ms",
+ "prompt_tokens",
+ "completion_tokens",
+ "user_agent",
+ "masked_content",
+ "secrets_detected",
+ "secrets_types",
+ "status_code",
+ "error_message",
+ ])
+ .orderBy("timestamp", "desc")
+ .limit(limit)
+ .offset(offset)
+ .execute();
+
+ return logs.map(normalizeLogRow);
}
- getLogs(limit: number = 100, offset: number = 0): RequestLog[] {
- const stmt = this.db.prepare(`
- SELECT
- id,
- timestamp,
- mode,
- provider,
- source,
- model,
- pii_detected,
- entities,
- latency_ms,
- scan_time_ms,
- prompt_tokens,
- completion_tokens,
- user_agent,
- masked_content,
- secrets_detected,
- secrets_types,
- status_code,
- error_message
- FROM request_logs
- ORDER BY timestamp DESC
- LIMIT ? OFFSET ?
- `);
-
- return (stmt.all(limit, offset) as RequestLog[]).map((log) => ({
- ...log,
- source: log.source || normalizeRequestSource(log.provider),
- }));
- }
+ async getStats(): Promise<Stats> {
+ await this.ready;
+ const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString();
- getStats(): Stats {
- // Total requests
- const totalResult = this.db.prepare(`SELECT COUNT(*) as count FROM request_logs`).get() as {
- count: number;
- };
-
- // PII requests
- const piiResult = this.db
- .prepare(`SELECT COUNT(*) as count FROM request_logs WHERE pii_detected = 1`)
- .get() as { count: number };
-
- // Proxy (OpenAI + Anthropic + Codex) vs Local vs API
- const proxyResult = this.db
- .prepare(
- `SELECT COUNT(*) as count FROM request_logs WHERE provider IN ('openai', 'anthropic', 'codex')`,
+ const [totalResult, piiResult, proxyResult, localResult, apiResult, browserExtensionResult] =
+ await Promise.all([
+ this.count(),
+ this.count((qb) => qb.where("pii_detected", "=", 1)),
+ this.count((qb) => qb.where("provider", "in", ["openai", "anthropic", "codex"])),
+ this.count((qb) => qb.where("provider", "=", "local")),
+ this.count((qb) =>
+ qb.where("provider", "=", "api").where("source", "!=", "browser_extension"),
+ ),
+ this.count((qb) => qb.where("source", "=", "browser_extension")),
+ ]);
+
+ const [scanTimeResult] = await this.db
+ .selectFrom("request_logs")
+ .select((eb) => eb.fn.avg<number>("scan_time_ms").as("avg"))
+ .execute();
+
+ const [tokensResult] = await this.db
+ .selectFrom("request_logs")
+ .select(
+ sql<number>`COALESCE(SUM(COALESCE(${sql.ref("prompt_tokens")}, 0) + COALESCE(${sql.ref(
+ "completion_tokens",
+ )}, 0)), 0)`.as("total"),
)
- .get() as { count: number };
- const localResult = this.db
- .prepare(`SELECT COUNT(*) as count FROM request_logs WHERE provider = 'local'`)
- .get() as { count: number };
- const apiResult = this.db
- .prepare(
- `SELECT COUNT(*) as count FROM request_logs WHERE provider = 'api' AND source != 'browser_extension'`,
- )
- .get() as { count: number };
- const browserExtensionResult = this.db
- .prepare(`SELECT COUNT(*) as count FROM request_logs WHERE source = 'browser_extension'`)
- .get() as { count: number };
-
- // Average scan time
- const scanTimeResult = this.db
- .prepare(`SELECT AVG(scan_time_ms) as avg FROM request_logs`)
- .get() as { avg: number | null };
-
- // Total tokens
- const tokensResult = this.db
- .prepare(`
- SELECT COALESCE(SUM(COALESCE(prompt_tokens, 0) + COALESCE(completion_tokens, 0)), 0) as total
- FROM request_logs
- `)
- .get() as { total: number };
-
- // Requests last hour
- const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString();
- const hourResult = this.db
- .prepare(`
- SELECT COUNT(*) as count FROM request_logs
- WHERE timestamp >= ?
- `)
- .get(oneHourAgo) as { count: number };
-
- const total = totalResult.count;
- const pii = piiResult.count;
-
- return {
- total_requests: total,
- pii_requests: pii,
- pii_percentage: total > 0 ? Math.round((pii / total) * 100 * 10) / 10 : 0,
- proxy_requests: proxyResult.count,
- local_requests: localResult.count,
- api_requests: apiResult.count,
- browser_extension_requests: browserExtensionResult.count,
- avg_scan_time_ms: Math.round(scanTimeResult.avg || 0),
- total_tokens: tokensResult.total,
- requests_last_hour: hourResult.count,
- };
+ .execute();
+
+ const hourResult = await this.count((qb) => qb.where("timestamp", ">=", oneHourAgo));
+
+ return buildStats({
+ total: totalResult.count,
+ pii: piiResult.count,
+ proxy: proxyResult.count,
+ local: localResult.count,
+ api: apiResult.count,
+ browserExtension: browserExtensionResult.count,
+ avgScanTime: (scanTimeResult as AverageRow).avg,
+ totalTokens: (tokensResult as TotalRow).total,
+ requestsLastHour: hourResult.count,
+ });
}
- getEntityStats(): Array<{ entity: string; count: number }> {
- const logs = this.db
- .prepare(`
- SELECT entities FROM request_logs WHERE entities IS NOT NULL AND entities != ''
- `)
- .all() as Array<{ entities: string }>;
+ async getEntityStats(): Promise<Array<{ entity: string; count: number }>> {
+ await this.ready;
+ const logs = await this.db
+ .selectFrom("request_logs")
+ .select("entities")
+ .where("entities", "is not", null)
+ .where("entities", "!=", "")
+ .execute();
const entityCounts = new Map<string, number>();
for (const log of logs) {
- const entities = log.entities
+ const entities = (log.entities ?? "")
.split(",")
.map((e) => e.trim())
.filter(Boolean);
.sort((a, b) => b.count - a.count);
}
- cleanup(): number {
+ async cleanup(): Promise<number> {
+ await this.ready;
+
if (this.retentionDays <= 0) {
- return 0; // Keep forever
+ return 0;
}
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - this.retentionDays);
- const result = this.db
- .prepare(`
- DELETE FROM request_logs WHERE timestamp < ?
- `)
- .run(cutoffDate.toISOString());
+ const result = await this.db
+ .deleteFrom("request_logs")
+ .where("timestamp", "<", cutoffDate.toISOString())
+ .executeTakeFirst();
- return result.changes;
+ return toNumber(result.numDeletedRows);
}
- close(): void {
- this.db.close();
+ async close(): Promise<void> {
+ await this.ready;
+ await this.db.destroy();
+ }
+
+ private async count(applyWhere?: (qb: CountQuery) => CountQuery): Promise<CountRow> {
+ let query = this.db
+ .selectFrom("request_logs")
+ .select((eb) => eb.fn.countAll<number>().as("count"));
+
+ if (applyWhere) {
+ query = applyWhere(query);
+ }
+
+ const [result] = await query.execute();
+ return result as CountRow;
}
}
-// Singleton instance
let loggerInstance: Logger | null = null;
export function getLogger(): Logger {
secretsMasked: data.secretsMasked,
});
- // Only log secret types if configured to do so
const shouldLogSecretTypes =
config.secrets_detection.log_detected_types && data.secretsTypes?.length;
- logger.log({
- timestamp: data.timestamp,
- mode: data.mode,
- provider: data.provider,
- source: data.source ?? normalizeRequestSource(data.provider),
- model: data.model,
- pii_detected: data.piiDetected,
- entities: data.entities.join(","),
- latency_ms: data.latencyMs,
- scan_time_ms: data.scanTimeMs,
- prompt_tokens: data.promptTokens ?? null,
- completion_tokens: data.completionTokens ?? null,
- user_agent: userAgent,
- masked_content: shouldLogContent ? (data.maskedContent ?? null) : null,
- secrets_detected: data.secretsDetected !== undefined ? (data.secretsDetected ? 1 : 0) : null,
- secrets_types: shouldLogSecretTypes ? data.secretsTypes!.join(",") : null,
- status_code: data.statusCode ?? null,
- error_message: data.errorMessage ?? null,
- });
+ void logger
+ .log({
+ timestamp: data.timestamp,
+ mode: data.mode,
+ provider: data.provider,
+ source: data.source ?? normalizeRequestSource(data.provider),
+ model: data.model,
+ pii_detected: data.piiDetected,
+ entities: data.entities.join(","),
+ latency_ms: data.latencyMs,
+ scan_time_ms: data.scanTimeMs,
+ prompt_tokens: data.promptTokens ?? null,
+ completion_tokens: data.completionTokens ?? null,
+ user_agent: userAgent,
+ masked_content: shouldLogContent ? (data.maskedContent ?? null) : null,
+ secrets_detected:
+ data.secretsDetected !== undefined ? (data.secretsDetected ? 1 : 0) : null,
+ secrets_types: shouldLogSecretTypes ? data.secretsTypes!.join(",") : null,
+ status_code: data.statusCode ?? null,
+ error_message: data.errorMessage ?? null,
+ })
+ .catch((error) => {
+ console.error("Failed to log request:", error);
+ });
} catch (error) {
console.error("Failed to log request:", error);
}
/**
* GET /api/logs - Get recent request logs
*/
-dashboardRoutes.get("/api/logs", zValidator("query", LogsQuerySchema), (c) => {
+dashboardRoutes.get("/api/logs", zValidator("query", LogsQuerySchema), async (c) => {
const { limit, offset } = c.req.valid("query");
const logger = getLogger();
- const logs = logger.getLogs(limit, offset);
+ const logs = await logger.getLogs(limit, offset);
return c.json({
logs,
/**
* GET /api/stats - Get statistics
*/
-dashboardRoutes.get("/api/stats", (c) => {
+dashboardRoutes.get("/api/stats", async (c) => {
const config = getConfig();
const logger = getLogger();
- const stats = logger.getStats();
- const entityStats = logger.getEntityStats();
+ const stats = await logger.getStats();
+ const entityStats = await logger.getEntityStats();
return c.json({
...stats,
git.99rst.org / sgasser-llm-shield.git / commitdiff