From: Stefan Gasser Date: Tue, 23 Jun 2026 07:42:46 +0000 (+0200) Subject: Remove language detection from PII flow (#108) X-Git-Tag: v0.7.0~9 X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=fb560252e7cba1400286d93c86f3249d968e4fe6;p=sgasser-llm-shield.git Remove language detection from PII flow (#108) * Remove language detection from PII flow * Format detector tests * Fix detector phone region typing --- diff --git a/README.md b/README.md index c68b0cb..0b432cf 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Both detected and masked in real time, including streaming responses. ## How detection works -Detection runs as a separate service that PasteGuard calls over HTTP, so you can run it wherever you like. It mixes two things: exact checks with checksums (IBANs, credit cards, emails, phones, IPs) and a small AI model ([GLiNER](https://github.com/urchade/GLiNER)) for names and places. It works the same in any language. +Detection runs as a separate service that PasteGuard calls over HTTP, so you can run it wherever you like. It mixes two things: exact checks with checksums (IBANs, credit cards, emails, phones, IPs) and a small AI model ([GLiNER](https://github.com/urchade/GLiNER)) for names and places. It works the same in any language. National-format phone numbers are validated against configured phone regions; international `+` numbers work globally. Code, Docker image, and tests are in [`detector/`](detector/). diff --git a/bun.lock b/bun.lock index 434b383..1ee7c79 100644 --- a/bun.lock +++ b/bun.lock @@ -6,7 +6,6 @@ "name": "pasteguard", "dependencies": { "@hono/zod-validator": "^0.7.6", - "eld": "^2.0.3", "hono": "^4.12.23", "hono-tailwind": "^2.2.0", "postcss": "^8.5.15", @@ -92,8 +91,6 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - "eld": ["eld@2.0.3", "", {}, "sha512-KiwFrycPPJ2XgKb7MGsJ3m3grUFXq6+55R8ZuS9fj+Qv0WRFgeEBhmorHO/YwHV9VqhdJURKFGJ+CY24sEikOw=="], - "enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="], "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], diff --git a/config.example.yaml b/config.example.yaml index 7f2933c..accc236 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -71,18 +71,25 @@ pii_detection: # for local dev (bun run dev against the docker-compose detector service). detector_url: ${DETECTOR_URL:-http://localhost:5002} - # Detection is multilingual and language-agnostic — one model finds entities - # regardless of the language. This list is only a hint (e.g. phone-number - # region) and the language reported in headers; it does not gate detection. - languages: - - en - # - de - # - fr - # - es - # - it - - # Fallback language if detected language is not in the list above - fallback_language: en + # Regions for national-format phone numbers; +international numbers work globally. + phone_regions: + - US + - GB + - DE + - AT + - CH + - IT + - FR + - BE + - LU + - ES + - NL + - PT + - BR + - PL + - RO + - MD + - IN score_threshold: 0.7 # Minimum confidence score (0.0 - 1.0) diff --git a/detector/detector/app.py b/detector/detector/app.py index e89c5e1..3729ab4 100644 --- a/detector/detector/app.py +++ b/detector/detector/app.py @@ -1,13 +1,3 @@ -"""The /analyze service PasteGuard speaks (src/pii/detect.ts): - POST /analyze {text, language, entities?, score_threshold?} - -> [{entity_type, start, end, score}] (offsets into text) - GET /health -> 200 - -The service is language-agnostic: every language returns 200 with a (possibly -empty) array, so the PasteGuard language probe always treats configured -languages as supported. -""" - from __future__ import annotations from bisect import bisect_left @@ -35,7 +25,7 @@ def _utf16_mapper(text: str): class AnalyzeRequest(BaseModel): text: str - language: str = "" + phone_regions: list[str] | None = None entities: list[str] | None = None score_threshold: float = 0.0 @@ -64,9 +54,7 @@ def health() -> dict[str, str]: @app.post("/analyze", response_model=list[Entity]) def analyze(req: AnalyzeRequest) -> list[Entity]: - deterministic = detect_deterministic(req.text, req.language) - # NER already applied per-label floors, so merge must not re-drop by the - # global threshold; deterministic spans are 1.0, so a 0.0 floor is safe. + deterministic = detect_deterministic(req.text, req.phone_regions) fuzzy = detect_gliner(req.text, req.score_threshold) spans = merge(deterministic, fuzzy, req.entities, 0.0) to_u16 = _utf16_mapper(req.text) diff --git a/detector/detector/deterministic.py b/detector/detector/deterministic.py index 35fad55..dbc1a49 100644 --- a/detector/detector/deterministic.py +++ b/detector/detector/deterministic.py @@ -27,31 +27,25 @@ from .entities import ( overlaps, ) -# Default phonenumbers region per request language, so national-format numbers -# (not only +international) are matched. -LANG_TO_REGION = { - "de": "DE", - "it": "IT", - "fr": "FR", - "es": "ES", - "en": "US", - "nl": "NL", - "pt": "PT", - "pl": "PL", - "ro": "RO", -} -EXTRA_LANGUAGE_REGIONS = { - # English benchmark and real traffic may contain either US or UK national - # formats. The primary region stays US; GB is an additional pass. - "en": ["GB"], - # Conservative CLDR-backed language regions where national phone formats - # are common in same-language text. - "de": ["AT", "CH"], - "fr": ["BE", "CH", "LU"], - "nl": ["BE"], - "pt": ["BR"], - "ro": ["MD"], -} +DEFAULT_PHONE_REGIONS = [ + "US", + "GB", + "DE", + "AT", + "CH", + "IT", + "FR", + "BE", + "LU", + "ES", + "NL", + "PT", + "BR", + "PL", + "RO", + "MD", + "IN", +] # `\w` is Unicode-aware so accented names (müller@, andré.) match in full; # structure rejects leading/trailing/consecutive dots. @@ -150,40 +144,50 @@ def _credit_card(text: str) -> list[Span]: return out -def _phone(text: str, language: str) -> list[Span]: - language_key = (language or "").lower() - region = LANG_TO_REGION.get(language_key) - regions = [region] if region else [] - regions.extend(EXTRA_LANGUAGE_REGIONS.get(language_key, [])) +def _phone_regions(phone_regions: list[str] | None) -> list[str]: + if phone_regions: + normalized = [] + seen = set() + for region in phone_regions: + region = (region or "").upper() + if not re.fullmatch(r"[A-Z]{2}", region) or region in seen: + continue + normalized.append(region) + seen.add(region) + return normalized + + return DEFAULT_PHONE_REGIONS + + +def _phone(text: str, phone_regions: list[str] | None = None) -> list[Span]: + regions: list[str | None] = [] + regions.extend(_phone_regions(phone_regions)) + if not regions: + regions.append(None) out: list[Span] = [] - # VALID leniency: only well-formed, assignable numbers. POSSIBLE would flag - # long invoice/ID digit runs as phones. - for candidate_region in regions or [None]: + for candidate_region in regions: for match in phonenumbers.PhoneNumberMatcher( text, candidate_region, leniency=phonenumbers.Leniency.VALID ): span = Span(PHONE_NUMBER, match.start, match.end, 1.0) if not any(s.start == span.start and s.end == span.end for s in out): out.append(span) - return out + out.sort(key=lambda s: (s.start, -(s.end - s.start))) + return [s for i, s in enumerate(out) if not any(overlaps(s, prev) for prev in out[:i])] -# Priority order: most specific first. A later detector's span is dropped if it -# overlaps an already-accepted one. -def detect_deterministic(text: str, language: str = "") -> list[Span]: +def detect_deterministic(text: str, phone_regions: list[str] | None = None) -> list[Span]: if not text: return [] ordered: list[Span] = [] ordered += _email(text) - # IPv6 before IPv4 so an IPv4-mapped/embedded IPv6 (e.g. ::ffff:192.168.0.1) - # is claimed in full, not truncated to its IPv4 tail by overlap resolution. ordered += _ipv6(text) ordered += _ipv4(text) ordered += _iban(text) ordered += _vat(text) ordered += _credit_card(text) - ordered += _phone(text, language) + ordered += _phone(text, phone_regions) accepted: list[Span] = [] for span in ordered: diff --git a/detector/tests/test_analyze.py b/detector/tests/test_analyze.py index a168d9c..698de32 100644 --- a/detector/tests/test_analyze.py +++ b/detector/tests/test_analyze.py @@ -8,7 +8,7 @@ import pytest from fastapi.testclient import TestClient import detector.app as appmod -from detector.entities import LOCATION, PERSON, Span +from detector.entities import LOCATION, PERSON, PHONE_NUMBER, Span @pytest.fixture @@ -38,7 +38,7 @@ def test_health(client): def test_response_shape_and_offsets(client): text = "IBAN IT60X0542811101000000123456" - r = client.post("/analyze", json={"text": text, "language": "it", "score_threshold": 0.7}) + r = client.post("/analyze", json={"text": text, "score_threshold": 0.7}) assert r.status_code == 200 body = r.json() assert body and all({"entity_type", "start", "end", "score"} == set(e) for e in body) @@ -49,10 +49,8 @@ def test_response_shape_and_offsets(client): def test_routing_iban_in_german_text(client): - # An Italian IBAN inside German text, language=de — the deterministic layer - # is language-agnostic, so it must still be found. text = "Der Mandant Mario Rossi (IT60X0542811101000000123456) zahlt." - r = client.post("/analyze", json={"text": text, "language": "de", "score_threshold": 0.7}) + r = client.post("/analyze", json={"text": text, "score_threshold": 0.7}) types = {e["entity_type"] for e in r.json()} assert "IBAN_CODE" in types assert "PERSON" in types # from the (stubbed) multilingual NER @@ -64,7 +62,6 @@ def test_entity_filter(client): "/analyze", json={ "text": text, - "language": "it", "entities": ["IBAN_CODE"], "score_threshold": 0.7, }, @@ -73,24 +70,37 @@ def test_entity_filter(client): assert types == {"IBAN_CODE"} +def test_phone_regions_control_national_formats(client): + text = "English ticket text with Indian callback 98765 43210." + r = client.post( + "/analyze", + json={ + "text": text, + "phone_regions": ["IN"], + "entities": ["PHONE_NUMBER"], + "score_threshold": 0.7, + }, + ) + types = {e["entity_type"] for e in r.json()} + assert PHONE_NUMBER in types + + def test_score_threshold_drops_fuzzy_keeps_deterministic(client): text = "Mario Rossi, IBAN IT60X0542811101000000123456" - r = client.post("/analyze", json={"text": text, "language": "it", "score_threshold": 0.99}) + r = client.post("/analyze", json={"text": text, "score_threshold": 0.99}) types = {e["entity_type"] for e in r.json()} assert "IBAN_CODE" in types # deterministic, score 1.0 assert "PERSON" not in types # fuzzy 0.95 < 0.99 -def test_language_probe_never_errors(client): - # The PasteGuard probe: any language must return 200 with an array, never - # an error, so every configured language is treated as supported. - r = client.post("/analyze", json={"text": "test", "language": "xx", "entities": ["PERSON"]}) +def test_minimal_request_never_errors(client): + r = client.post("/analyze", json={"text": "test", "entities": ["PERSON"]}) assert r.status_code == 200 assert isinstance(r.json(), list) def test_empty_text(client): - r = client.post("/analyze", json={"text": "", "language": "de"}) + r = client.post("/analyze", json={"text": ""}) assert r.status_code == 200 assert r.json() == [] @@ -99,7 +109,7 @@ def test_utf16_offsets_with_astral_char(client): # An emoji (astral, 2 UTF-16 code units) before the email must shift the # returned offsets so PasteGuard's JS text.slice lands on the email. text = "Hi 😀 mail@x.com" - r = client.post("/analyze", json={"text": text, "language": "en", "score_threshold": 0.7}) + r = client.post("/analyze", json={"text": text, "score_threshold": 0.7}) email = next(e for e in r.json() if e["entity_type"] == "EMAIL_ADDRESS") # JS code units: "Hi " (3) + emoji (2) + " " (1) = 6 assert email["start"] == 6 diff --git a/detector/tests/test_deterministic.py b/detector/tests/test_deterministic.py index 607a256..9ce051c 100644 --- a/detector/tests/test_deterministic.py +++ b/detector/tests/test_deterministic.py @@ -13,8 +13,10 @@ from detector.entities import ( ) -def types_texts(text, language=""): - return [(s.entity_type, text[s.start : s.end]) for s in detect_deterministic(text, language)] +def types_texts(text, phone_regions=None): + return [ + (s.entity_type, text[s.start : s.end]) for s in detect_deterministic(text, phone_regions) + ] # --- IBAN --- @@ -26,27 +28,27 @@ def test_iban_plain(): def test_iban_spaced_keeps_spacing_in_span(): text = "Bonifico IBAN IT60 X054 2811 1010 0000 0123 456 entro lunedì" - assert (IBAN_CODE, "IT60 X054 2811 1010 0000 0123 456") in types_texts(text, "it") + assert (IBAN_CODE, "IT60 X054 2811 1010 0000 0123 456") in types_texts(text) def test_iban_german(): assert (IBAN_CODE, "DE89 3704 0044 0532 0130 00") in types_texts( - "auf IBAN DE89 3704 0044 0532 0130 00", "de" + "auf IBAN DE89 3704 0044 0532 0130 00" ) def test_iban_invalid_checksum_rejected(): - assert types_texts("IBAN errato: IT60X0542811101000000123457", "it") == [] + assert types_texts("IBAN errato: IT60X0542811101000000123457") == [] def test_iban_lowercase(): assert (IBAN_CODE, "de89 3704 0044 0532 0130 00") in types_texts( - "bitte auf iban de89 3704 0044 0532 0130 00", "de" + "bitte auf iban de89 3704 0044 0532 0130 00" ) def test_iban_lowercase_does_not_bleed_into_following_word(): - spans = detect_deterministic("iban de89 3704 0044 0532 0130 00 grazie", "it") + spans = detect_deterministic("iban de89 3704 0044 0532 0130 00 grazie") iban = next(s for s in spans if s.entity_type == IBAN_CODE) text = "iban de89 3704 0044 0532 0130 00 grazie" assert "grazie" not in text[iban.start : iban.end] @@ -54,37 +56,37 @@ def test_iban_lowercase_does_not_bleed_into_following_word(): def test_iban_does_not_bleed_into_following_word(): # The lowercase word after the IBAN must not be swallowed. - spans = detect_deterministic("IBAN IT60 X054 2811 1010 0000 0123 456 entro", "it") + spans = detect_deterministic("IBAN IT60 X054 2811 1010 0000 0123 456 entro") iban = next(s for s in spans if s.entity_type == IBAN_CODE) assert "entro" not in "IBAN IT60 X054 2811 1010 0000 0123 456 entro"[iban.start : iban.end] # --- VAT (EU, stdnum-validated) --- def test_vat_valid_multiple_countries(): - for v, lang in [ - ("DE136695976", "de"), - ("IT00743110157", "it"), - ("FR40303265045", "fr"), - ("ESA13585625", "es"), - ("ATU13585627", "de"), - ("BE0428759497", "nl"), - ("PL5260001246", "pl"), + for v in [ + "DE136695976", + "IT00743110157", + "FR40303265045", + "ESA13585625", + "ATU13585627", + "BE0428759497", + "PL5260001246", ]: - assert (VAT_CODE, v) in types_texts(f"VAT {v} on the invoice", lang) + assert (VAT_CODE, v) in types_texts(f"VAT {v} on the invoice") def test_vat_spaced_after_prefix(): - assert (VAT_CODE, "DE 136695976") in types_texts("USt-IdNr DE 136695976", "de") + assert (VAT_CODE, "DE 136695976") in types_texts("USt-IdNr DE 136695976") def test_vat_invalid_checksum_rejected(): - assert all(t != VAT_CODE for t, _ in types_texts("VAT DE136695977 is wrong", "de")) + assert all(t != VAT_CODE for t, _ in types_texts("VAT DE136695977 is wrong")) def test_vat_does_not_claim_iban(): # An IBAN must stay IBAN_CODE and never be mistagged as VAT. text = "auf IBAN DE89 3704 0044 0532 0130 00" - types = types_texts(text, "de") + types = types_texts(text) assert (IBAN_CODE, "DE89 3704 0044 0532 0130 00") in types assert all(t != VAT_CODE for t, _ in types) @@ -92,18 +94,18 @@ def test_vat_does_not_claim_iban(): def test_vat_label_prefix_not_absorbed(): # A 2-letter label before the VAT (e.g. "ID") must not be taken as the country prefix. for text in ["Tax ID DE136695976 here", "Steuer-ID DE136695976 anbei"]: - assert (VAT_CODE, "DE136695976") in types_texts(text, "en") + assert (VAT_CODE, "DE136695976") in types_texts(text) def test_vat_lowercase(): - assert (VAT_CODE, "de136695976") in types_texts("the vat is de136695976.", "en") - assert (VAT_CODE, "it00743110157") in types_texts("partita iva it00743110157.", "it") + assert (VAT_CODE, "de136695976") in types_texts("the vat is de136695976.") + assert (VAT_CODE, "it00743110157") in types_texts("partita iva it00743110157.") def test_vat_word_prefix_does_not_swallow_following_vat(): # A lowercase word that is also a country code ("es", "it") must not hide the real VAT. for text in ["es DE136695976", "it DE136695976"]: - assert (VAT_CODE, "DE136695976") in types_texts(text, "en") + assert (VAT_CODE, "DE136695976") in types_texts(text) # --- Email / IP --- @@ -187,79 +189,94 @@ def test_credit_card_invalid_luhn_rejected(): # --- Phone (VALID leniency: no FP on long ID digit runs) --- def test_phone_german_national(): - assert (PHONE_NUMBER, "0171-1234567") in types_texts("Telefon 0171-1234567", "de") + assert (PHONE_NUMBER, "0171-1234567") in types_texts("Telefon 0171-1234567", ["DE"]) def test_phone_international(): - assert (PHONE_NUMBER, "+49 171 1234567") in types_texts("Tel: +49 171 1234567", "de") + assert (PHONE_NUMBER, "+49 171 1234567") in types_texts("Tel: +49 171 1234567") + + +def test_phone_regions_control_national_formats(): + assert (PHONE_NUMBER, "98765 43210") in types_texts( + "Please call the customer on 98765 43210.", ["IN"] + ) + assert (PHONE_NUMBER, "06 6982") in types_texts( + "El contacto italiano atiende en 06 6982.", ["IT"] + ) + + +def test_phone_default_regions_keep_longest_overlap(): + types = types_texts("Please call the customer on 98765 43210.") + assert (PHONE_NUMBER, "98765 43210") in types + assert (PHONE_NUMBER, "43210") not in types def test_phone_no_false_positive_on_invoice_number(): - assert all(t != PHONE_NUMBER for t, _ in types_texts("Rechnung 2893081508152 vom", "de")) + assert all(t != PHONE_NUMBER for t, _ in types_texts("Rechnung 2893081508152 vom")) def test_phone_english_uk_national(): assert (PHONE_NUMBER, "0121 234 5678") in types_texts( - "The Birmingham callback number is 0121 234 5678.", "en" + "The Birmingham callback number is 0121 234 5678.", ["GB"] ) def test_phone_german_extra_regions(): assert (PHONE_NUMBER, "01 234567890") in types_texts( - "Die Wiener Kontaktnummer ist 01 234567890.", "de" + "Die Wiener Kontaktnummer ist 01 234567890.", ["AT"] ) assert (PHONE_NUMBER, "0848 800 800") in types_texts( - "Die Schweizer Kontaktnummer ist 0848 800 800.", "de" + "Die Schweizer Kontaktnummer ist 0848 800 800.", ["CH"] ) def test_phone_french_extra_regions(): assert (PHONE_NUMBER, "012 34 56 78") in types_texts( - "Le numéro belge du contact est 012 34 56 78.", "fr" + "Le numéro belge du contact est 012 34 56 78.", ["BE"] ) assert (PHONE_NUMBER, "0848 800 800") in types_texts( - "Le numéro suisse du contact est 0848 800 800.", "fr" + "Le numéro suisse du contact est 0848 800 800.", ["CH"] ) assert (PHONE_NUMBER, "27 12 34 56") in types_texts( - "Le numéro luxembourgeois du contact est 27 12 34 56.", "fr" + "Le numéro luxembourgeois du contact est 27 12 34 56.", ["LU"] ) def test_phone_dutch_belgium_region(): assert (PHONE_NUMBER, "012 34 56 78") in types_texts( - "Het Belgische telefoonnummer is 012 34 56 78.", "nl" + "Het Belgische telefoonnummer is 012 34 56 78.", ["BE"] ) def test_phone_portuguese_brazil_region(): assert (PHONE_NUMBER, "(11) 2345-6789") in types_texts( - "O número brasileiro do contato é (11) 2345-6789.", "pt" + "O número brasileiro do contato é (11) 2345-6789.", ["BR"] ) def test_phone_polish_and_romanian_primary_regions(): assert (PHONE_NUMBER, "12 345 67 89") in types_texts( - "Numer krajowy kontaktu to 12 345 67 89.", "pl" + "Numer krajowy kontaktu to 12 345 67 89.", ["PL"] ) assert (PHONE_NUMBER, "021 123 4567") in types_texts( - "Numărul național al contactului este 021 123 4567.", "ro" + "Numărul național al contactului este 021 123 4567.", ["RO"] ) def test_phone_romanian_moldova_region(): assert (PHONE_NUMBER, "022 212 345") in types_texts( - "Numărul de contact din Moldova este 022 212 345.", "ro" + "Numărul de contact din Moldova este 022 212 345.", ["MD"] ) # --- overlap / priority --- def test_no_overlapping_spans(): text = "IBAN IT60 X054 2811 1010 0000 0123 456, mail luca@example.it" - spans = detect_deterministic(text, "it") + spans = detect_deterministic(text) spans.sort(key=lambda s: s.start) for a, b in pairwise(spans): assert a.end <= b.start def test_empty_text(): - assert detect_deterministic("", "de") == [] + assert detect_deterministic("") == [] diff --git a/docs/api-reference/anthropic.mdx b/docs/api-reference/anthropic.mdx index a0210fc..5f5f2e1 100644 --- a/docs/api-reference/anthropic.mdx +++ b/docs/api-reference/anthropic.mdx @@ -116,8 +116,6 @@ PasteGuard adds headers to indicate PII and secrets handling: | `X-PasteGuard-Provider` | Provider used (`anthropic` or `local`) | | `X-PasteGuard-PII-Detected` | `true` if PII was found | | `X-PasteGuard-PII-Masked` | `true` if PII was masked (mask mode only) | -| `X-PasteGuard-Language` | Detected language code | -| `X-PasteGuard-Language-Fallback` | `true` if configured language was not available | | `X-PasteGuard-Secrets-Detected` | `true` if secrets were found | | `X-PasteGuard-Secrets-Types` | Comma-separated list of detected secret types | | `X-PasteGuard-Secrets-Masked` | `true` if secrets were masked | diff --git a/docs/api-reference/dashboard-api.mdx b/docs/api-reference/dashboard-api.mdx index 5c06c1b..a07766b 100644 --- a/docs/api-reference/dashboard-api.mdx +++ b/docs/api-reference/dashboard-api.mdx @@ -45,9 +45,6 @@ curl "http://localhost:3000/dashboard/api/logs?limit=100&offset=0" "prompt_tokens": 150, "completion_tokens": 200, "user_agent": "OpenAI-Python/1.0.0", - "language": "en", - "language_fallback": false, - "detected_language": "en", "masked_content": "Hello [[EMAIL_ADDRESS_1]]", "secrets_detected": 0, "secrets_types": null diff --git a/docs/api-reference/mask.mdx b/docs/api-reference/mask.mdx index 53598a0..3d4ebd2 100644 --- a/docs/api-reference/mask.mdx +++ b/docs/api-reference/mask.mdx @@ -37,7 +37,6 @@ X-PasteGuard-Source: browser-extension | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `text` | string | Yes | Text to scan and mask | -| `language` | string | No | Language code for PII detection (auto-detected if not provided) | | `startFrom` | object | No | Counter values to continue numbering from previous calls | | `detect` | array | No | What to detect: `["pii"]`, `["secrets"]`, or `["pii", "secrets"]` (default: both) | @@ -57,9 +56,7 @@ X-PasteGuard-Source: browser-extension "entities": [ { "type": "EMAIL_ADDRESS", "placeholder": "[[EMAIL_ADDRESS_1]]" }, { "type": "PHONE_NUMBER", "placeholder": "[[PHONE_NUMBER_1]]" } - ], - "language": "en", - "languageFallback": false + ] } ``` @@ -69,8 +66,6 @@ X-PasteGuard-Source: browser-extension | `context` | Mapping of placeholder to original value (for client-side unmasking) | | `counters` | Final counter values per entity type | | `entities` | List of detected entities with their placeholders | -| `language` | Language used for PII detection | -| `languageFallback` | Whether the configured fallback language was used (auto-detection failed) | ## Detection Options diff --git a/docs/api-reference/openai.mdx b/docs/api-reference/openai.mdx index 0cb496d..caaa8ee 100644 --- a/docs/api-reference/openai.mdx +++ b/docs/api-reference/openai.mdx @@ -117,8 +117,6 @@ PasteGuard adds headers to indicate PII and secrets handling: | `X-PasteGuard-Provider` | Provider used (`openai` or `local`) | | `X-PasteGuard-PII-Detected` | `true` if PII was found | | `X-PasteGuard-PII-Masked` | `true` if PII was masked (mask mode only) | -| `X-PasteGuard-Language` | Detected language code | -| `X-PasteGuard-Language-Fallback` | `true` if configured language was not available | | `X-PasteGuard-Secrets-Detected` | `true` if secrets were found | | `X-PasteGuard-Secrets-Types` | Comma-separated list of detected secret types | | `X-PasteGuard-Secrets-Masked` | `true` if secrets were masked | diff --git a/docs/api-reference/status.mdx b/docs/api-reference/status.mdx index 15aeabe..0a5d4e8 100644 --- a/docs/api-reference/status.mdx +++ b/docs/api-reference/status.mdx @@ -81,8 +81,7 @@ curl http://localhost:3000/info } }, "pii_detection": { - "languages": ["en"], - "fallback_language": "en", + "phone_regions": ["US", "GB", "DE", "IT", "IN"], "score_threshold": 0.7, "entities": ["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER"] }, diff --git a/docs/concepts/mask-mode.mdx b/docs/concepts/mask-mode.mdx index 7524160..b1b4e1f 100644 --- a/docs/concepts/mask-mode.mdx +++ b/docs/concepts/mask-mode.mdx @@ -74,13 +74,6 @@ X-PasteGuard-Mode: mask X-PasteGuard-Provider: openai X-PasteGuard-PII-Detected: true X-PasteGuard-PII-Masked: true -X-PasteGuard-Language: en -``` - -If the detected language wasn't configured and fell back to `fallback_language`: - -``` -X-PasteGuard-Language-Fallback: true ``` ## Streaming Support diff --git a/docs/concepts/pii-detection.mdx b/docs/concepts/pii-detection.mdx index dc31ca5..acabefb 100644 --- a/docs/concepts/pii-detection.mdx +++ b/docs/concepts/pii-detection.mdx @@ -3,7 +3,7 @@ title: PII Detection description: Personal data detection via a deterministic checksum layer plus multilingual GLiNER NER --- -PasteGuard detects PII with a small open-source service that pairs a deterministic regex/checksum layer with a multilingual neural model. Detection is **language-agnostic**: the neural model finds names and places whatever the request language — an Italian name and city in a German letter are caught even with a `de` hint — and structured identifiers (IBAN, credit card, email, IP) are matched by checksum, not by language. +PasteGuard detects PII with a small open-source service that pairs a deterministic regex/checksum layer with a multilingual neural model. Detection is **language-agnostic**: the neural model finds names and places in mixed-language text — for example, an Italian name and city in a German letter — and structured identifiers (IBAN, credit card, email, IP) are matched by checksum, not by language. ## How it works @@ -25,11 +25,13 @@ The detector speaks the `/analyze` HTTP contract, is configured through the `det | `IP_ADDRESS` | 192.168.1.1 | | `VAT_CODE` | EU VAT number, e.g. `DE136695976`, `IT00743110157`, `FR40303265045` | -Names and locations are detected by the multilingual model (verified across EN/DE/IT/FR/ES/NL/PT and more). IBAN, credit card, phone, email, and IP work internationally. `VAT_CODE` covers any EU member state, validated per-country via the VAT checksum (`python-stdnum`) and requiring the country prefix. The structured identifiers (IBAN, credit card, IP, VAT) are checksum- or format-validated, so they are caught with high precision and without false positives on lookalike numbers. +Names and locations are detected by the multilingual model (verified across EN/DE/IT/FR/ES/NL/PT and more). IBAN, credit card, email, and IP work internationally. Phone numbers with a `+` country prefix are global; national-format phone numbers are validated against configured `phone_regions`. `VAT_CODE` covers any EU member state, validated per-country via the VAT checksum (`python-stdnum`) and requiring the country prefix. The structured identifiers (IBAN, credit card, IP, VAT) are checksum- or format-validated, so they are caught with high precision and without false positives on lookalike numbers. -## Languages +## Languages and phone regions -Detection is multilingual and **language-agnostic** — one pass over the text finds entities regardless of the language it is written in, so mixed-language documents (for example an Italian name and city in a German letter) are handled correctly. The `language` field in a request is used only as a hint, e.g. to pick a phone-number region; it does not gate detection. +Detection is multilingual and **language-agnostic** — one pass over the text finds entities regardless of the language it is written in, so mixed-language documents (for example an Italian name and city in a German letter) are handled correctly. PasteGuard does not auto-detect language. + +National-format phone numbers are the exception because the same digits can be valid or invalid depending on the country numbering plan. Configure `phone_regions` for the regions your traffic commonly contains. ## Confidence Scoring @@ -49,11 +51,4 @@ When PII is detected: ``` X-PasteGuard-PII-Detected: true X-PasteGuard-PII-Masked: true # mask mode only -X-PasteGuard-Language: en -``` - -If the fallback language was used: - -``` -X-PasteGuard-Language-Fallback: true ``` diff --git a/docs/concepts/route-mode.mdx b/docs/concepts/route-mode.mdx index 3887ecb..cdac76d 100644 --- a/docs/concepts/route-mode.mdx +++ b/docs/concepts/route-mode.mdx @@ -98,7 +98,6 @@ When a request is routed to local: X-PasteGuard-Mode: route X-PasteGuard-Provider: local X-PasteGuard-PII-Detected: true -X-PasteGuard-Language: en ``` When routed to OpenAI or Anthropic: @@ -107,11 +106,4 @@ When routed to OpenAI or Anthropic: X-PasteGuard-Mode: route X-PasteGuard-Provider: openai X-PasteGuard-PII-Detected: false -X-PasteGuard-Language: en -``` - -If the detected language wasn't configured and fell back to `fallback_language`: - -``` -X-PasteGuard-Language-Fallback: true ``` diff --git a/docs/configuration/pii-detection.mdx b/docs/configuration/pii-detection.mdx index 11e9dbc..b53ded6 100644 --- a/docs/configuration/pii-detection.mdx +++ b/docs/configuration/pii-detection.mdx @@ -6,11 +6,12 @@ description: Configure PII detection settings ```yaml pii_detection: detector_url: http://localhost:5002 - languages: - - en - - de - - it - fallback_language: en + phone_regions: + - US + - GB + - DE + - IT + - IN score_threshold: 0.7 entities: - PERSON @@ -28,27 +29,27 @@ pii_detection: | Option | Default | Description | |--------|---------|-------------| | `detector_url` | `http://localhost:5002` | Detector `/analyze` URL | -| `languages` | `["en"]` | Languages the proxy auto-detects among (used as a hint, e.g. phone region) | -| `fallback_language` | `en` | Language hint used when none is detected | +| `phone_regions` | US, GB, DE, AT, CH, IT, FR, BE, LU, ES, NL, PT, BR, PL, RO, MD, IN | Regions used to validate national-format phone numbers without a country prefix | | `score_threshold` | `0.7` | Minimum confidence floor for the neural labels PERSON and LOCATION (0.0-1.0). Checksum-validated identifiers always score `1.0` and are unaffected | | `entities` | See below | Entity types to return | -## Languages +## Phone Regions Detection is **multilingual and language-agnostic** — the model finds names and places regardless of the language the text is written in (an Italian name and city in a German letter are caught), and structured identifiers are matched by checksum independent of language. There are no per-language images and no spaCy models to load. -The `languages` list and `fallback_language` only control the language *hint* the proxy sends with each request (which the detector uses for things like phone-number region); they do not gate detection. +Phone numbers are different: international numbers with a `+` country prefix are detected globally, but national-format numbers need one or more country numbering plans. Configure those plans with `phone_regions` instead of relying on text language. ```yaml pii_detection: - languages: - - en - - de - - it - fallback_language: de + phone_regions: + - US + - GB + - DE + - IT + - IN ``` -If only one language is configured, language detection is skipped for better performance. +Use a focused list for the traffic you expect. Adding many regions improves recall for mixed-language and international text, but can increase false positives on IDs, ticket numbers, and other digit sequences. ## Entities diff --git a/docs/installation.mdx b/docs/installation.mdx index faa5211..20783ba 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -67,11 +67,11 @@ curl -O https://raw.githubusercontent.com/sgasser/pasteguard/main/docker-compose docker compose up -d ``` -## Languages +## Languages and phone numbers -Detection is multilingual and needs no per-language configuration. The `languages` -and `fallback_language` settings only set a hint sent with each request (e.g. for -phone-number region) — see [PII Detection Config](/configuration/pii-detection). +Detection is multilingual and needs no per-language configuration. Configure +`phone_regions` only if you need national-format phone numbers without a `+` +country prefix — see [PII Detection Config](/configuration/pii-detection). ## Environment Variables diff --git a/package.json b/package.json index e0e7b14..5e71562 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ }, "dependencies": { "@hono/zod-validator": "^0.7.6", - "eld": "^2.0.3", "hono": "^4.12.23", "hono-tailwind": "^2.2.0", "postcss": "^8.5.15", diff --git a/src/config.test.ts b/src/config.test.ts index 0d5c102..315ef1a 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -124,6 +124,45 @@ pii_detection: } }); + test("accepts phone regions as comma-separated config", () => { + const path = writeConfig(` +mode: mask +providers: + openai: {} + anthropic: {} +pii_detection: + detector_url: http://localhost:5002 + phone_regions: us,gb,in +`); + + try { + const config = loadConfig(path); + + expect(config.pii_detection.phone_regions).toEqual(["US", "GB", "IN"]); + } finally { + cleanupConfig(path); + } + }); + + test("rejects invalid phone region codes", () => { + const path = writeConfig(` +mode: mask +providers: + openai: {} + anthropic: {} +pii_detection: + detector_url: http://localhost:5002 + phone_regions: + - USA +`); + + try { + expect(() => loadConfig(path)).toThrow("Invalid configuration"); + } finally { + cleanupConfig(path); + } + }); + test("rejects invalid masking allowlist regex patterns", () => { const path = writeConfig(` mode: mask diff --git a/src/config.ts b/src/config.ts index 2c9a9a6..2cc9416 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,6 @@ import { existsSync, readFileSync, statSync } from "node:fs"; import { parse as parseYaml } from "yaml"; import { z } from "zod"; -import { SUPPORTED_LANGUAGES } from "./constants/languages"; // Schema definitions @@ -99,24 +98,46 @@ const MaskingSchema = z.object({ denylist: z.array(DenylistPatternSchema).default([]), }); -const LanguageEnum = z.enum(SUPPORTED_LANGUAGES); +const PhoneRegionSchema = z + .string() + .trim() + .transform((value) => value.toUpperCase()) + .pipe(z.string().regex(/^[A-Z]{2}$/, "Expected ISO 3166-1 alpha-2 region code")); -// Accept either an array or a comma-separated string for languages (the latter -// supports ${ENV_VAR} substitution in the YAML config, e.g. languages: ${LANGS:-en}). -const LanguagesSchema = z - .union([z.array(LanguageEnum), z.string()]) +const PhoneRegionsSchema = z + .union([z.array(PhoneRegionSchema), z.string()]) .transform((val) => { if (Array.isArray(val)) return val; - return val.split(",").map((s) => s.trim()) as (typeof SUPPORTED_LANGUAGES)[number][]; + return val + .split(",") + .map((s) => s.trim()) + .filter(Boolean); }) - .pipe(z.array(LanguageEnum)) - .default(["en"]); + .pipe(z.array(PhoneRegionSchema).min(1)) + .default([ + "US", + "GB", + "DE", + "AT", + "CH", + "IT", + "FR", + "BE", + "LU", + "ES", + "NL", + "PT", + "BR", + "PL", + "RO", + "MD", + "IN", + ]); const PIIDetectionSchema = z.object({ enabled: z.boolean().default(true), detector_url: z.string().url(), - languages: LanguagesSchema, - fallback_language: LanguageEnum.default("en"), + phone_regions: PhoneRegionsSchema, score_threshold: z.coerce.number().min(0).max(1).default(0.7), entities: z .array(z.string()) diff --git a/src/constants/languages.ts b/src/constants/languages.ts deleted file mode 100644 index 7e30f42..0000000 --- a/src/constants/languages.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Languages the proxy can detect and send as a hint to the detector. - * Detection itself is language-agnostic; this list only gates the hint. - */ -export const SUPPORTED_LANGUAGES = [ - "ca", // Catalan - "zh", // Chinese - "hr", // Croatian - "da", // Danish - "nl", // Dutch - "en", // English - "fi", // Finnish - "fr", // French - "de", // German - "el", // Greek - "it", // Italian - "ja", // Japanese - "ko", // Korean - "lt", // Lithuanian - "mk", // Macedonian - "nb", // Norwegian - "pl", // Polish - "pt", // Portuguese - "ro", // Romanian - "ru", // Russian - "sl", // Slovenian - "es", // Spanish - "sv", // Swedish - "uk", // Ukrainian -] as const; - -export type SupportedLanguage = (typeof SUPPORTED_LANGUAGES)[number]; diff --git a/src/index.ts b/src/index.ts index 6ef1d2b..476d969 100644 --- a/src/index.ts +++ b/src/index.ts @@ -125,8 +125,6 @@ async function validateStartup() { } console.log("[STARTUP] ✓ Detector connected"); - // Language config is validated at load time (config.ts LanguageEnum); the - // detector is language-agnostic, so there is no runtime language gate. } function printStartupBanner(config: ReturnType, host: string, port: number) { @@ -168,8 +166,7 @@ Mode: ${config.mode.toUpperCase()} ${modeInfo} PII Detection: - Languages: ${config.pii_detection.languages.join(", ")} - Fallback: ${config.pii_detection.fallback_language} + Phone regions: ${config.pii_detection.phone_regions.join(", ")} Threshold: ${config.pii_detection.score_threshold} Entities: ${config.pii_detection.entities.join(", ")} diff --git a/src/pii/detect.test.ts b/src/pii/detect.test.ts index cc65f59..9ff5aeb 100644 --- a/src/pii/detect.test.ts +++ b/src/pii/detect.test.ts @@ -17,6 +17,8 @@ function mockDetector( Array<{ entity_type: string; start: number; end: number; score: number }> >, ) { + const analyzeRequests: unknown[] = []; + globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => { const urlStr = url.toString(); @@ -26,6 +28,7 @@ function mockDetector( if (urlStr.includes("/analyze") && init?.body) { const body = JSON.parse(init.body as string); + analyzeRequests.push(body); const text = body.text as string; for (const [key, entities] of Object.entries(responses)) { @@ -45,6 +48,8 @@ function mockDetector( return originalFetch(url, init); }) as unknown as typeof fetch; + + return analyzeRequests; } function createRequest(messages: OpenAIMessage[]): OpenAIRequest { @@ -273,7 +278,7 @@ describe("PIIDetector", () => { }); const detector = new PIIDetector(); - const entities = await detector.detectPII("test@example.com", "en"); + const entities = await detector.detectPII("test@example.com"); expect(entities).toHaveLength(1); expect(entities[0].entity_type).toBe("EMAIL_ADDRESS"); @@ -283,10 +288,30 @@ describe("PIIDetector", () => { mockDetector({}); const detector = new PIIDetector(); - const entities = await detector.detectPII("Hello world", "en"); + const entities = await detector.detectPII("Hello world"); expect(entities).toHaveLength(0); }); + + test("sends configured phone regions to the detector", async () => { + const config = getConfig(); + const previousPhoneRegions = config.pii_detection.phone_regions; + config.pii_detection.phone_regions = ["US", "IN", "IT"]; + const analyzeRequests = mockDetector({}); + + try { + const detector = new PIIDetector(); + await detector.detectPII("Call 080 1234 5678"); + + expect(analyzeRequests).toHaveLength(1); + expect(analyzeRequests[0]).toMatchObject({ + text: "Call 080 1234 5678", + phone_regions: ["US", "IN", "IT"], + }); + } finally { + config.pii_detection.phone_regions = previousPhoneRegions; + } + }); }); describe("healthCheck", () => { diff --git a/src/pii/detect.ts b/src/pii/detect.ts index 0183961..f79e177 100644 --- a/src/pii/detect.ts +++ b/src/pii/detect.ts @@ -2,7 +2,6 @@ import { type AllowlistPattern, type DenylistPattern, getConfig } from "../confi import { HEALTH_CHECK_TIMEOUT_MS } from "../constants/timeouts"; import { overlaps, resolveConflicts } from "../masking/conflict-resolver"; import type { RequestExtractor } from "../masking/types"; -import { getLanguageDetector, type SupportedLanguage } from "../services/language-detector"; export interface PIIEntity { entity_type: string; @@ -131,7 +130,7 @@ export function filterAllowlistedEntities( interface AnalyzeRequest { text: string; - language: string; + phone_regions?: string[]; entities?: string[]; score_threshold?: number; } @@ -141,29 +140,28 @@ export interface PIIDetectionResult { spanEntities: PIIEntity[][]; allEntities: PIIEntity[]; scanTimeMs: number; - language: SupportedLanguage; - languageFallback: boolean; - detectedLanguage?: string; } export class PIIDetector { private detectorUrl: string; private scoreThreshold: number; private entityTypes: string[]; + private phoneRegions: string[]; constructor() { const config = getConfig(); this.detectorUrl = config.pii_detection.detector_url; this.scoreThreshold = config.pii_detection.score_threshold; this.entityTypes = config.pii_detection.entities; + this.phoneRegions = config.pii_detection.phone_regions; } - async detectPII(text: string, language: SupportedLanguage): Promise { + async detectPII(text: string): Promise { const analyzeEndpoint = `${this.detectorUrl}/analyze`; const request: AnalyzeRequest = { text, - language, + phone_regions: this.phoneRegions, entities: this.entityTypes, score_threshold: this.scoreThreshold, }; @@ -210,28 +208,18 @@ export class PIIDetector { const startTime = Date.now(); const config = getConfig(); - // Pure pass-through: detection off and no denylist, so skip extraction and language detection. if (!config.pii_detection.enabled && config.masking.denylist.length === 0) { return { hasPII: false, spanEntities: [], allEntities: [], scanTimeMs: 0, - language: config.pii_detection.fallback_language, - languageFallback: true, }; } // Extract all text spans from request const spans = extractor.extractTexts(request); - // Detect language from message content (skip system spans with messageIndex -1) - const messageSpans = spans.filter((span) => span.messageIndex >= 0); - const langText = messageSpans.map((s) => s.text).join("\n"); - const langResult = langText - ? getLanguageDetector().detect(langText) - : { language: config.pii_detection.fallback_language, usedFallback: true }; - // Detect PII for each span independently const scanRoles = config.pii_detection.scan_roles ? new Set(config.pii_detection.scan_roles) @@ -249,7 +237,7 @@ export class PIIDetector { } const detectedEntities = config.pii_detection.enabled - ? await this.detectPII(span.text, langResult.language) + ? await this.detectPII(span.text) : []; const filteredEntities = filterAllowlistedEntities(span.text, detectedEntities, allowlist); return mergeDenylistEntities(filteredEntities, denylistedEntities); @@ -263,9 +251,6 @@ export class PIIDetector { spanEntities, allEntities, scanTimeMs: Date.now() - startTime, - language: langResult.language, - languageFallback: langResult.usedFallback, - detectedLanguage: langResult.detectedLanguage, }; } diff --git a/src/routes/api.test.ts b/src/routes/api.test.ts index 207556c..f9e3199 100644 --- a/src/routes/api.test.ts +++ b/src/routes/api.test.ts @@ -8,9 +8,7 @@ import { } from "../pii/detect"; // Mock the PII detector to avoid needing the detector running -const mockDetectPII = mock<(text: string, language: string) => Promise>(() => - Promise.resolve([]), -); +const mockDetectPII = mock<(text: string) => Promise>(() => Promise.resolve([])); mock.module("../pii/detect", () => ({ getPIIDetector: () => ({ detectPII: mockDetectPII, @@ -112,12 +110,10 @@ describe("POST /api/mask", () => { masked: string; context: Record; entities: unknown[]; - language: string; }; expect(body.masked).toBe("Hello world"); expect(body.context).toEqual({}); expect(body.entities).toEqual([]); - expect(body.language).toBeDefined(); }); test("masks PII entities", async () => { @@ -417,22 +413,6 @@ describe("POST /api/mask", () => { expect(body.error.details[0].message).toBe("Detector connection failed"); }); - test("includes languageFallback in response", async () => { - mockDetectPII.mockResolvedValueOnce([]); - - const res = await app.request("/api/mask", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ text: "Hello world" }), - }); - - expect(res.status).toBe(200); - const body = (await res.json()) as { - languageFallback: boolean; - }; - expect(typeof body.languageFallback).toBe("boolean"); - }); - test("respects multiple entity types in startFrom", async () => { mockDetectPII.mockResolvedValueOnce([ { entity_type: "PERSON", start: 0, end: 4, score: 0.9 }, diff --git a/src/routes/api.ts b/src/routes/api.ts index 5bff1fa..f13bd43 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -18,7 +18,6 @@ import { import { mask as maskPII } from "../pii/mask"; import { detectSecrets } from "../secrets/detect"; import { maskSecrets } from "../secrets/mask"; -import { getLanguageDetector, type SupportedLanguage } from "../services/language-detector"; import { logRequest, normalizeRequestSource } from "../services/logger"; import { createLogData } from "./utils"; @@ -27,7 +26,6 @@ export const apiRoutes = new Hono(); // Request schema const MaskRequestSchema = z.object({ text: z.string().trim().min(1, "text is required"), - language: z.string().optional(), startFrom: z.record(z.string(), z.number()).optional(), detect: z.array(z.enum(["pii", "secrets"])).optional(), }); @@ -45,8 +43,6 @@ interface MaskResponse { context: Record; counters: Record; entities: MaskEntity[]; - language: string; - languageFallback: boolean; } /** @@ -118,20 +114,6 @@ apiRoutes.post("/mask", async (c) => { } } - // Detect language (use provided or auto-detect) - let language: SupportedLanguage; - let languageFallback = false; - if ( - request.language && - config.pii_detection.languages.includes(request.language as SupportedLanguage) - ) { - language = request.language as SupportedLanguage; - } else { - const langResult = getLanguageDetector().detect(request.text); - language = langResult.language; - languageFallback = langResult.usedFallback; - } - let maskedText = request.text; const allEntities: MaskEntity[] = []; const piiEntityTypes: string[] = []; @@ -177,8 +159,6 @@ apiRoutes.post("/mask", async (c) => { pii: { hasPII: piiEntityTypes.length > 0, entityTypes: piiEntityTypes, - language, - languageFallback, scanTimeMs, }, statusCode: 503, @@ -205,9 +185,7 @@ apiRoutes.post("/mask", async (c) => { try { const piiStartTime = Date.now(); const detector = getPIIDetector(); - const piiEntities = config.pii_detection.enabled - ? await detector.detectPII(maskedText, language) - : []; + const piiEntities = config.pii_detection.enabled ? await detector.detectPII(maskedText) : []; scanTimeMs = Date.now() - piiStartTime; const filteredEntities = filterAllowlistedEntities( @@ -240,7 +218,7 @@ apiRoutes.post("/mask", async (c) => { source, model: "mask", startTime, - pii: { hasPII: false, entityTypes: [], language, languageFallback, scanTimeMs: 0 }, + pii: { hasPII: false, entityTypes: [], scanTimeMs: 0 }, statusCode: 503, errorMessage: error instanceof Error ? error.message : "PII detection failed", }), @@ -270,8 +248,6 @@ apiRoutes.post("/mask", async (c) => { pii: { hasPII: piiEntityTypes.length > 0, entityTypes: piiEntityTypes, - language, - languageFallback, scanTimeMs, }, secrets: @@ -288,8 +264,6 @@ apiRoutes.post("/mask", async (c) => { context: context.mapping, counters: { ...context.counters }, entities: allEntities, - language, - languageFallback, }; return c.json(response); diff --git a/src/routes/codex.test.ts b/src/routes/codex.test.ts index db8e370..070973a 100644 --- a/src/routes/codex.test.ts +++ b/src/routes/codex.test.ts @@ -9,8 +9,6 @@ const mockAnalyzeRequest = mock<() => Promise>(() => spanEntities: [], allEntities: [], scanTimeMs: 0, - language: "en", - languageFallback: false, }), ); const mockLogRequest = mock(() => {}); @@ -55,8 +53,6 @@ afterEach(() => { spanEntities: [], allEntities: [], scanTimeMs: 0, - language: "en", - languageFallback: false, }); mockLogRequest.mockClear(); }); @@ -105,8 +101,6 @@ describe("Codex proxy", () => { spanEntities: [[{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }]], allEntities: [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }], scanTimeMs: 3, - language: "en", - languageFallback: false, }); const calls: CapturedRequest[] = []; @@ -159,8 +153,6 @@ describe("Codex proxy", () => { spanEntities: [[{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }]], allEntities: [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }], scanTimeMs: 3, - language: "en", - languageFallback: false, }); let fetchCalled = false; @@ -194,8 +186,6 @@ describe("Codex proxy", () => { spanEntities: [[{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }]], allEntities: [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }], scanTimeMs: 3, - language: "en", - languageFallback: false, }); globalThis.fetch = (async (_input: string | URL | Request, _init?: RequestInit) => diff --git a/src/routes/info.ts b/src/routes/info.ts index fca7a13..fe1a344 100644 --- a/src/routes/info.ts +++ b/src/routes/info.ts @@ -29,8 +29,7 @@ infoRoutes.get("/info", (c) => { mode: config.mode, providers, pii_detection: { - languages: config.pii_detection.languages, - fallback_language: config.pii_detection.fallback_language, + phone_regions: config.pii_detection.phone_regions, score_threshold: config.pii_detection.score_threshold, entities: config.pii_detection.entities, }, diff --git a/src/routes/utils.ts b/src/routes/utils.ts index b5cfd66..5aed03a 100644 --- a/src/routes/utils.ts +++ b/src/routes/utils.ts @@ -80,8 +80,6 @@ export const errorFormats = { export interface PIIHeaderData { hasPII: boolean; - language: string; - languageFallback: boolean; } export interface SecretsHeaderData { @@ -103,11 +101,7 @@ export function setResponseHeaders( c.header("X-PasteGuard-Mode", mode); c.header("X-PasteGuard-Provider", provider); c.header("X-PasteGuard-PII-Detected", pii.hasPII.toString()); - c.header("X-PasteGuard-Language", pii.language); - if (pii.languageFallback) { - c.header("X-PasteGuard-Language-Fallback", "true"); - } if (mode === "mask" && pii.hasPII) { c.header("X-PasteGuard-PII-Masked", "true"); } @@ -138,9 +132,6 @@ export function setBlockedHeaders(c: Context, secretTypes: string[]): void { export interface PIILogData { hasPII: boolean; entityTypes: string[]; - language: string; - languageFallback: boolean; - detectedLanguage?: string; scanTimeMs: number; } @@ -160,9 +151,6 @@ export function toPIILogData(piiResult: PIIDetectResult): PIILogData { return { hasPII: piiResult.hasPII, entityTypes: [...new Set(piiResult.detection.allEntities.map((e) => e.entity_type))], - language: piiResult.detection.language, - languageFallback: piiResult.detection.languageFallback, - detectedLanguage: piiResult.detection.detectedLanguage, scanTimeMs: piiResult.detection.scanTimeMs, }; } @@ -173,8 +161,6 @@ export function toPIILogData(piiResult: PIIDetectResult): PIILogData { export function toPIIHeaderData(piiResult: PIIDetectResult): PIIHeaderData { return { hasPII: piiResult.hasPII, - language: piiResult.detection.language, - languageFallback: piiResult.detection.languageFallback, }; } @@ -236,9 +222,6 @@ export function createLogData(options: CreateLogDataOptions): RequestLogData { entities: pii?.entityTypes ?? [], latencyMs: Date.now() - startTime, scanTimeMs: pii?.scanTimeMs ?? 0, - language: pii?.language ?? config.pii_detection.fallback_language, - languageFallback: pii?.languageFallback ?? false, - detectedLanguage: pii?.detectedLanguage, maskedContent, secretsDetected: secrets?.detected, secretsMasked: secrets?.masked, diff --git a/src/services/language-detector.ts b/src/services/language-detector.ts deleted file mode 100644 index 0baf7c5..0000000 --- a/src/services/language-detector.ts +++ /dev/null @@ -1,64 +0,0 @@ -import eld from "eld/small"; -import { getConfig } from "../config"; -import type { SupportedLanguage } from "../constants/languages"; - -export type { SupportedLanguage } from "../constants/languages"; - -export interface LanguageDetectionResult { - language: SupportedLanguage; - usedFallback: boolean; - detectedLanguage?: string; - confidence?: number; -} - -// Map detected ISO codes onto the supported-language list where they differ. -const ISO_TO_SUPPORTED_OVERRIDES: Record = { - no: "nb", // Norwegian (generic) → Norwegian Bokmål -}; - -export class LanguageDetector { - private configuredLanguages: SupportedLanguage[]; - private fallbackLanguage: SupportedLanguage; - - constructor() { - const config = getConfig(); - this.configuredLanguages = config.pii_detection.languages; - this.fallbackLanguage = config.pii_detection.fallback_language; - } - - detect(text: string): LanguageDetectionResult { - const result = eld.detect(text); - const detectedIso = result.language; - const scores = result.getScores(); - const confidence = scores[detectedIso] ?? 0; - - // Use override if exists, otherwise use the detected code as-is (most are 1:1) - const mappedLang = (ISO_TO_SUPPORTED_OVERRIDES[detectedIso] || - detectedIso) as SupportedLanguage; - - if (mappedLang && this.configuredLanguages.includes(mappedLang)) { - return { - language: mappedLang, - usedFallback: false, - detectedLanguage: detectedIso, - confidence, - }; - } - - return { - language: this.fallbackLanguage, - usedFallback: true, - detectedLanguage: detectedIso, - confidence, - }; - } -} - -let detectorInstance: LanguageDetector | null = null; - -export function getLanguageDetector(): LanguageDetector { - if (!detectorInstance) { - detectorInstance = new LanguageDetector(); - } - return detectorInstance; -} diff --git a/src/services/logger.ts b/src/services/logger.ts index 7727f25..edcb42b 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -20,9 +20,6 @@ export interface RequestLog { prompt_tokens: number | null; completion_tokens: number | null; user_agent: string | null; - language: string; - language_fallback: boolean; - detected_language: string | null; masked_content: string | null; secrets_detected: number | null; secrets_types: string | null; @@ -99,9 +96,6 @@ export class Logger { prompt_tokens INTEGER, completion_tokens INTEGER, user_agent TEXT, - language TEXT NOT NULL DEFAULT 'en', - language_fallback INTEGER NOT NULL DEFAULT 0, - detected_language TEXT, masked_content TEXT, secrets_detected INTEGER, secrets_types TEXT, @@ -141,9 +135,9 @@ export class Logger { log(entry: Omit): void { const stmt = this.db.prepare(` INSERT INTO request_logs - (timestamp, mode, provider, source, model, pii_detected, entities, latency_ms, scan_time_ms, prompt_tokens, completion_tokens, user_agent, language, language_fallback, detected_language, masked_content, secrets_detected, secrets_types, status_code, error_message) + (timestamp, mode, provider, source, model, pii_detected, entities, latency_ms, scan_time_ms, prompt_tokens, completion_tokens, user_agent, masked_content, secrets_detected, secrets_types, status_code, error_message) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); stmt.run( @@ -159,9 +153,6 @@ export class Logger { entry.prompt_tokens, entry.completion_tokens, entry.user_agent, - entry.language, - entry.language_fallback ? 1 : 0, - entry.detected_language, entry.masked_content, entry.secrets_detected ?? null, entry.secrets_types ?? null, @@ -175,7 +166,26 @@ export class Logger { */ getLogs(limit: number = 100, offset: number = 0): RequestLog[] { const stmt = this.db.prepare(` - SELECT * FROM request_logs + SELECT + id, + timestamp, + mode, + provider, + source, + model, + pii_detected, + entities, + latency_ms, + scan_time_ms, + prompt_tokens, + completion_tokens, + user_agent, + masked_content, + secrets_detected, + secrets_types, + status_code, + error_message + FROM request_logs ORDER BY timestamp DESC LIMIT ? OFFSET ? `); @@ -334,9 +344,6 @@ export interface RequestLogData { scanTimeMs: number; promptTokens?: number; completionTokens?: number; - language: string; - languageFallback: boolean; - detectedLanguage?: string; maskedContent?: string; secretsDetected?: boolean; secretsMasked?: boolean; @@ -374,9 +381,6 @@ export function logRequest(data: RequestLogData, userAgent: string | null): void prompt_tokens: data.promptTokens ?? null, completion_tokens: data.completionTokens ?? null, user_agent: userAgent, - language: data.language, - language_fallback: data.languageFallback, - detected_language: data.detectedLanguage ?? null, masked_content: shouldLogContent ? (data.maskedContent ?? null) : null, secrets_detected: data.secretsDetected !== undefined ? (data.secretsDetected ? 1 : 0) : null, secrets_types: shouldLogSecretTypes ? data.secretsTypes!.join(",") : null, diff --git a/src/test-utils/detection-results.ts b/src/test-utils/detection-results.ts index 5088c54..3e4bcc8 100644 --- a/src/test-utils/detection-results.ts +++ b/src/test-utils/detection-results.ts @@ -2,7 +2,6 @@ * Test utilities for creating detection results */ -import type { SupportedLanguage } from "../constants/languages"; import type { PIIDetectionResult, PIIEntity } from "../pii/detect"; import type { MessageSecretsResult, SecretLocation } from "../secrets/detect"; @@ -12,9 +11,6 @@ import type { MessageSecretsResult, SecretLocation } from "../secrets/detect"; export function createPIIResultFromSpans( spanEntities: PIIEntity[][], options: { - language?: SupportedLanguage; - languageFallback?: boolean; - detectedLanguage?: string; scanTimeMs?: number; } = {}, ): PIIDetectionResult { @@ -24,9 +20,6 @@ export function createPIIResultFromSpans( spanEntities, allEntities, scanTimeMs: options.scanTimeMs ?? 0, - language: options.language ?? "en", - languageFallback: options.languageFallback ?? false, - detectedLanguage: options.detectedLanguage, }; } diff --git a/src/views/dashboard/page.tsx b/src/views/dashboard/page.tsx index 0ccb2a0..90eb5b3 100644 --- a/src/views/dashboard/page.tsx +++ b/src/views/dashboard/page.tsx @@ -387,9 +387,6 @@ const LogsSection: FC = () => ( Model - - Language - PII Entities @@ -403,7 +400,7 @@ const LogsSection: FC = () => ( - +
@@ -577,7 +574,7 @@ async function fetchLogs() { const tbody = document.getElementById('logs-body'); if (data.logs.length === 0) { - tbody.innerHTML = '
📋
No requests yet
'; + tbody.innerHTML = '
📋
No requests yet
'; return; } @@ -587,16 +584,7 @@ async function fetchLogs() { const secretsTypes = log.secrets_types ? log.secrets_types.split(',').filter(s => s.trim()) : []; const secretsDetected = log.secrets_detected === 1; const isError = log.status_code && log.status_code >= 400; - const lang = log.language || 'en'; - const detectedLang = log.detected_language; const source = log.source; - - const formatLang = (code) => code ? code.toUpperCase() : lang.toUpperCase(); - - // Show original→fallback when fallback was used (e.g. FR→EN) - const langDisplay = log.language_fallback && detectedLang - ? '' + formatLang(detectedLang) + '→' + lang.toUpperCase() + '' - : lang.toUpperCase(); const logId = log.id || index; const isExpanded = expandedRowId === logId; @@ -615,7 +603,6 @@ async function fetchLogs() { '' + sourceBadge + '' + '' + statusBadge + '' + '' + log.model + '' + - '' + langDisplay + '' + '' + (entities.length > 0 ? '
' + entities.map(e => '' + e.trim() + '').join('') + '
' @@ -635,7 +622,7 @@ async function fetchLogs() { const detailRow = '' + - '' + + '' + '
' + detailContent + '
' + '' + '';