## How detection works
-Detection runs as a separate service that PasteGuard calls over HTTP, so you can run it wherever you like. It mixes two things: exact checks with checksums (IBANs, credit cards, emails, phones, IPs) and a small AI model ([GLiNER](https://github.com/urchade/GLiNER)) for names and places. It works the same in any language.
+Detection runs as a separate service that PasteGuard calls over HTTP, so you can run it wherever you like. It mixes two things: exact checks with checksums (IBANs, credit cards, emails, phones, IPs) and a small AI model ([GLiNER](https://github.com/urchade/GLiNER)) for names and places. It works the same in any language. National-format phone numbers are validated against configured phone regions; international `+` numbers work globally.
Code, Docker image, and tests are in [`detector/`](detector/).
"name": "pasteguard",
"dependencies": {
"@hono/zod-validator": "^0.7.6",
- "eld": "^2.0.3",
"hono": "^4.12.23",
"hono-tailwind": "^2.2.0",
"postcss": "^8.5.15",
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
- "eld": ["eld@2.0.3", "", {}, "sha512-KiwFrycPPJ2XgKb7MGsJ3m3grUFXq6+55R8ZuS9fj+Qv0WRFgeEBhmorHO/YwHV9VqhdJURKFGJ+CY24sEikOw=="],
-
"enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="],
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
# for local dev (bun run dev against the docker-compose detector service).
detector_url: ${DETECTOR_URL:-http://localhost:5002}
- # Detection is multilingual and language-agnostic — one model finds entities
- # regardless of the language. This list is only a hint (e.g. phone-number
- # region) and the language reported in headers; it does not gate detection.
- languages:
- - en
- # - de
- # - fr
- # - es
- # - it
-
- # Fallback language if detected language is not in the list above
- fallback_language: en
+ # Regions for national-format phone numbers; +international numbers work globally.
+ phone_regions:
+ - US
+ - GB
+ - DE
+ - AT
+ - CH
+ - IT
+ - FR
+ - BE
+ - LU
+ - ES
+ - NL
+ - PT
+ - BR
+ - PL
+ - RO
+ - MD
+ - IN
score_threshold: 0.7 # Minimum confidence score (0.0 - 1.0)
-"""The /analyze service PasteGuard speaks (src/pii/detect.ts):
- POST /analyze {text, language, entities?, score_threshold?}
- -> [{entity_type, start, end, score}] (offsets into text)
- GET /health -> 200
-
-The service is language-agnostic: every language returns 200 with a (possibly
-empty) array, so the PasteGuard language probe always treats configured
-languages as supported.
-"""
-
from __future__ import annotations
from bisect import bisect_left
class AnalyzeRequest(BaseModel):
text: str
- language: str = ""
+ phone_regions: list[str] | None = None
entities: list[str] | None = None
score_threshold: float = 0.0
@app.post("/analyze", response_model=list[Entity])
def analyze(req: AnalyzeRequest) -> list[Entity]:
- deterministic = detect_deterministic(req.text, req.language)
- # NER already applied per-label floors, so merge must not re-drop by the
- # global threshold; deterministic spans are 1.0, so a 0.0 floor is safe.
+ deterministic = detect_deterministic(req.text, req.phone_regions)
fuzzy = detect_gliner(req.text, req.score_threshold)
spans = merge(deterministic, fuzzy, req.entities, 0.0)
to_u16 = _utf16_mapper(req.text)
overlaps,
)
-# Default phonenumbers region per request language, so national-format numbers
-# (not only +international) are matched.
-LANG_TO_REGION = {
- "de": "DE",
- "it": "IT",
- "fr": "FR",
- "es": "ES",
- "en": "US",
- "nl": "NL",
- "pt": "PT",
- "pl": "PL",
- "ro": "RO",
-}
-EXTRA_LANGUAGE_REGIONS = {
- # English benchmark and real traffic may contain either US or UK national
- # formats. The primary region stays US; GB is an additional pass.
- "en": ["GB"],
- # Conservative CLDR-backed language regions where national phone formats
- # are common in same-language text.
- "de": ["AT", "CH"],
- "fr": ["BE", "CH", "LU"],
- "nl": ["BE"],
- "pt": ["BR"],
- "ro": ["MD"],
-}
+DEFAULT_PHONE_REGIONS = [
+ "US",
+ "GB",
+ "DE",
+ "AT",
+ "CH",
+ "IT",
+ "FR",
+ "BE",
+ "LU",
+ "ES",
+ "NL",
+ "PT",
+ "BR",
+ "PL",
+ "RO",
+ "MD",
+ "IN",
+]
# `\w` is Unicode-aware so accented names (müller@, andré.) match in full;
# structure rejects leading/trailing/consecutive dots.
return out
-def _phone(text: str, language: str) -> list[Span]:
- language_key = (language or "").lower()
- region = LANG_TO_REGION.get(language_key)
- regions = [region] if region else []
- regions.extend(EXTRA_LANGUAGE_REGIONS.get(language_key, []))
+def _phone_regions(phone_regions: list[str] | None) -> list[str]:
+ if phone_regions:
+ normalized = []
+ seen = set()
+ for region in phone_regions:
+ region = (region or "").upper()
+ if not re.fullmatch(r"[A-Z]{2}", region) or region in seen:
+ continue
+ normalized.append(region)
+ seen.add(region)
+ return normalized
+
+ return DEFAULT_PHONE_REGIONS
+
+
+def _phone(text: str, phone_regions: list[str] | None = None) -> list[Span]:
+ regions: list[str | None] = []
+ regions.extend(_phone_regions(phone_regions))
+ if not regions:
+ regions.append(None)
out: list[Span] = []
- # VALID leniency: only well-formed, assignable numbers. POSSIBLE would flag
- # long invoice/ID digit runs as phones.
- for candidate_region in regions or [None]:
+ for candidate_region in regions:
for match in phonenumbers.PhoneNumberMatcher(
text, candidate_region, leniency=phonenumbers.Leniency.VALID
):
span = Span(PHONE_NUMBER, match.start, match.end, 1.0)
if not any(s.start == span.start and s.end == span.end for s in out):
out.append(span)
- return out
+ out.sort(key=lambda s: (s.start, -(s.end - s.start)))
+ return [s for i, s in enumerate(out) if not any(overlaps(s, prev) for prev in out[:i])]
-# Priority order: most specific first. A later detector's span is dropped if it
-# overlaps an already-accepted one.
-def detect_deterministic(text: str, language: str = "") -> list[Span]:
+def detect_deterministic(text: str, phone_regions: list[str] | None = None) -> list[Span]:
if not text:
return []
ordered: list[Span] = []
ordered += _email(text)
- # IPv6 before IPv4 so an IPv4-mapped/embedded IPv6 (e.g. ::ffff:192.168.0.1)
- # is claimed in full, not truncated to its IPv4 tail by overlap resolution.
ordered += _ipv6(text)
ordered += _ipv4(text)
ordered += _iban(text)
ordered += _vat(text)
ordered += _credit_card(text)
- ordered += _phone(text, language)
+ ordered += _phone(text, phone_regions)
accepted: list[Span] = []
for span in ordered:
from fastapi.testclient import TestClient
import detector.app as appmod
-from detector.entities import LOCATION, PERSON, Span
+from detector.entities import LOCATION, PERSON, PHONE_NUMBER, Span
@pytest.fixture
def test_response_shape_and_offsets(client):
text = "IBAN IT60X0542811101000000123456"
- r = client.post("/analyze", json={"text": text, "language": "it", "score_threshold": 0.7})
+ r = client.post("/analyze", json={"text": text, "score_threshold": 0.7})
assert r.status_code == 200
body = r.json()
assert body and all({"entity_type", "start", "end", "score"} == set(e) for e in body)
def test_routing_iban_in_german_text(client):
- # An Italian IBAN inside German text, language=de — the deterministic layer
- # is language-agnostic, so it must still be found.
text = "Der Mandant Mario Rossi (IT60X0542811101000000123456) zahlt."
- r = client.post("/analyze", json={"text": text, "language": "de", "score_threshold": 0.7})
+ r = client.post("/analyze", json={"text": text, "score_threshold": 0.7})
types = {e["entity_type"] for e in r.json()}
assert "IBAN_CODE" in types
assert "PERSON" in types # from the (stubbed) multilingual NER
"/analyze",
json={
"text": text,
- "language": "it",
"entities": ["IBAN_CODE"],
"score_threshold": 0.7,
},
assert types == {"IBAN_CODE"}
+def test_phone_regions_control_national_formats(client):
+ text = "English ticket text with Indian callback 98765 43210."
+ r = client.post(
+ "/analyze",
+ json={
+ "text": text,
+ "phone_regions": ["IN"],
+ "entities": ["PHONE_NUMBER"],
+ "score_threshold": 0.7,
+ },
+ )
+ types = {e["entity_type"] for e in r.json()}
+ assert PHONE_NUMBER in types
+
+
def test_score_threshold_drops_fuzzy_keeps_deterministic(client):
text = "Mario Rossi, IBAN IT60X0542811101000000123456"
- r = client.post("/analyze", json={"text": text, "language": "it", "score_threshold": 0.99})
+ r = client.post("/analyze", json={"text": text, "score_threshold": 0.99})
types = {e["entity_type"] for e in r.json()}
assert "IBAN_CODE" in types # deterministic, score 1.0
assert "PERSON" not in types # fuzzy 0.95 < 0.99
-def test_language_probe_never_errors(client):
- # The PasteGuard probe: any language must return 200 with an array, never
- # an error, so every configured language is treated as supported.
- r = client.post("/analyze", json={"text": "test", "language": "xx", "entities": ["PERSON"]})
+def test_minimal_request_never_errors(client):
+ r = client.post("/analyze", json={"text": "test", "entities": ["PERSON"]})
assert r.status_code == 200
assert isinstance(r.json(), list)
def test_empty_text(client):
- r = client.post("/analyze", json={"text": "", "language": "de"})
+ r = client.post("/analyze", json={"text": ""})
assert r.status_code == 200
assert r.json() == []
# An emoji (astral, 2 UTF-16 code units) before the email must shift the
# returned offsets so PasteGuard's JS text.slice lands on the email.
text = "Hi 😀 mail@x.com"
- r = client.post("/analyze", json={"text": text, "language": "en", "score_threshold": 0.7})
+ r = client.post("/analyze", json={"text": text, "score_threshold": 0.7})
email = next(e for e in r.json() if e["entity_type"] == "EMAIL_ADDRESS")
# JS code units: "Hi " (3) + emoji (2) + " " (1) = 6
assert email["start"] == 6
)
-def types_texts(text, language=""):
- return [(s.entity_type, text[s.start : s.end]) for s in detect_deterministic(text, language)]
+def types_texts(text, phone_regions=None):
+ return [
+ (s.entity_type, text[s.start : s.end]) for s in detect_deterministic(text, phone_regions)
+ ]
# --- IBAN ---
def test_iban_spaced_keeps_spacing_in_span():
text = "Bonifico IBAN IT60 X054 2811 1010 0000 0123 456 entro lunedì"
- assert (IBAN_CODE, "IT60 X054 2811 1010 0000 0123 456") in types_texts(text, "it")
+ assert (IBAN_CODE, "IT60 X054 2811 1010 0000 0123 456") in types_texts(text)
def test_iban_german():
assert (IBAN_CODE, "DE89 3704 0044 0532 0130 00") in types_texts(
- "auf IBAN DE89 3704 0044 0532 0130 00", "de"
+ "auf IBAN DE89 3704 0044 0532 0130 00"
)
def test_iban_invalid_checksum_rejected():
- assert types_texts("IBAN errato: IT60X0542811101000000123457", "it") == []
+ assert types_texts("IBAN errato: IT60X0542811101000000123457") == []
def test_iban_lowercase():
assert (IBAN_CODE, "de89 3704 0044 0532 0130 00") in types_texts(
- "bitte auf iban de89 3704 0044 0532 0130 00", "de"
+ "bitte auf iban de89 3704 0044 0532 0130 00"
)
def test_iban_lowercase_does_not_bleed_into_following_word():
- spans = detect_deterministic("iban de89 3704 0044 0532 0130 00 grazie", "it")
+ spans = detect_deterministic("iban de89 3704 0044 0532 0130 00 grazie")
iban = next(s for s in spans if s.entity_type == IBAN_CODE)
text = "iban de89 3704 0044 0532 0130 00 grazie"
assert "grazie" not in text[iban.start : iban.end]
def test_iban_does_not_bleed_into_following_word():
# The lowercase word after the IBAN must not be swallowed.
- spans = detect_deterministic("IBAN IT60 X054 2811 1010 0000 0123 456 entro", "it")
+ spans = detect_deterministic("IBAN IT60 X054 2811 1010 0000 0123 456 entro")
iban = next(s for s in spans if s.entity_type == IBAN_CODE)
assert "entro" not in "IBAN IT60 X054 2811 1010 0000 0123 456 entro"[iban.start : iban.end]
# --- VAT (EU, stdnum-validated) ---
def test_vat_valid_multiple_countries():
- for v, lang in [
- ("DE136695976", "de"),
- ("IT00743110157", "it"),
- ("FR40303265045", "fr"),
- ("ESA13585625", "es"),
- ("ATU13585627", "de"),
- ("BE0428759497", "nl"),
- ("PL5260001246", "pl"),
+ for v in [
+ "DE136695976",
+ "IT00743110157",
+ "FR40303265045",
+ "ESA13585625",
+ "ATU13585627",
+ "BE0428759497",
+ "PL5260001246",
]:
- assert (VAT_CODE, v) in types_texts(f"VAT {v} on the invoice", lang)
+ assert (VAT_CODE, v) in types_texts(f"VAT {v} on the invoice")
def test_vat_spaced_after_prefix():
- assert (VAT_CODE, "DE 136695976") in types_texts("USt-IdNr DE 136695976", "de")
+ assert (VAT_CODE, "DE 136695976") in types_texts("USt-IdNr DE 136695976")
def test_vat_invalid_checksum_rejected():
- assert all(t != VAT_CODE for t, _ in types_texts("VAT DE136695977 is wrong", "de"))
+ assert all(t != VAT_CODE for t, _ in types_texts("VAT DE136695977 is wrong"))
def test_vat_does_not_claim_iban():
# An IBAN must stay IBAN_CODE and never be mistagged as VAT.
text = "auf IBAN DE89 3704 0044 0532 0130 00"
- types = types_texts(text, "de")
+ types = types_texts(text)
assert (IBAN_CODE, "DE89 3704 0044 0532 0130 00") in types
assert all(t != VAT_CODE for t, _ in types)
def test_vat_label_prefix_not_absorbed():
# A 2-letter label before the VAT (e.g. "ID") must not be taken as the country prefix.
for text in ["Tax ID DE136695976 here", "Steuer-ID DE136695976 anbei"]:
- assert (VAT_CODE, "DE136695976") in types_texts(text, "en")
+ assert (VAT_CODE, "DE136695976") in types_texts(text)
def test_vat_lowercase():
- assert (VAT_CODE, "de136695976") in types_texts("the vat is de136695976.", "en")
- assert (VAT_CODE, "it00743110157") in types_texts("partita iva it00743110157.", "it")
+ assert (VAT_CODE, "de136695976") in types_texts("the vat is de136695976.")
+ assert (VAT_CODE, "it00743110157") in types_texts("partita iva it00743110157.")
def test_vat_word_prefix_does_not_swallow_following_vat():
# A lowercase word that is also a country code ("es", "it") must not hide the real VAT.
for text in ["es DE136695976", "it DE136695976"]:
- assert (VAT_CODE, "DE136695976") in types_texts(text, "en")
+ assert (VAT_CODE, "DE136695976") in types_texts(text)
# --- Email / IP ---
# --- Phone (VALID leniency: no FP on long ID digit runs) ---
def test_phone_german_national():
- assert (PHONE_NUMBER, "0171-1234567") in types_texts("Telefon 0171-1234567", "de")
+ assert (PHONE_NUMBER, "0171-1234567") in types_texts("Telefon 0171-1234567", ["DE"])
def test_phone_international():
- assert (PHONE_NUMBER, "+49 171 1234567") in types_texts("Tel: +49 171 1234567", "de")
+ assert (PHONE_NUMBER, "+49 171 1234567") in types_texts("Tel: +49 171 1234567")
+
+
+def test_phone_regions_control_national_formats():
+ assert (PHONE_NUMBER, "98765 43210") in types_texts(
+ "Please call the customer on 98765 43210.", ["IN"]
+ )
+ assert (PHONE_NUMBER, "06 6982") in types_texts(
+ "El contacto italiano atiende en 06 6982.", ["IT"]
+ )
+
+
+def test_phone_default_regions_keep_longest_overlap():
+ types = types_texts("Please call the customer on 98765 43210.")
+ assert (PHONE_NUMBER, "98765 43210") in types
+ assert (PHONE_NUMBER, "43210") not in types
def test_phone_no_false_positive_on_invoice_number():
- assert all(t != PHONE_NUMBER for t, _ in types_texts("Rechnung 2893081508152 vom", "de"))
+ assert all(t != PHONE_NUMBER for t, _ in types_texts("Rechnung 2893081508152 vom"))
def test_phone_english_uk_national():
assert (PHONE_NUMBER, "0121 234 5678") in types_texts(
- "The Birmingham callback number is 0121 234 5678.", "en"
+ "The Birmingham callback number is 0121 234 5678.", ["GB"]
)
def test_phone_german_extra_regions():
assert (PHONE_NUMBER, "01 234567890") in types_texts(
- "Die Wiener Kontaktnummer ist 01 234567890.", "de"
+ "Die Wiener Kontaktnummer ist 01 234567890.", ["AT"]
)
assert (PHONE_NUMBER, "0848 800 800") in types_texts(
- "Die Schweizer Kontaktnummer ist 0848 800 800.", "de"
+ "Die Schweizer Kontaktnummer ist 0848 800 800.", ["CH"]
)
def test_phone_french_extra_regions():
assert (PHONE_NUMBER, "012 34 56 78") in types_texts(
- "Le numéro belge du contact est 012 34 56 78.", "fr"
+ "Le numéro belge du contact est 012 34 56 78.", ["BE"]
)
assert (PHONE_NUMBER, "0848 800 800") in types_texts(
- "Le numéro suisse du contact est 0848 800 800.", "fr"
+ "Le numéro suisse du contact est 0848 800 800.", ["CH"]
)
assert (PHONE_NUMBER, "27 12 34 56") in types_texts(
- "Le numéro luxembourgeois du contact est 27 12 34 56.", "fr"
+ "Le numéro luxembourgeois du contact est 27 12 34 56.", ["LU"]
)
def test_phone_dutch_belgium_region():
assert (PHONE_NUMBER, "012 34 56 78") in types_texts(
- "Het Belgische telefoonnummer is 012 34 56 78.", "nl"
+ "Het Belgische telefoonnummer is 012 34 56 78.", ["BE"]
)
def test_phone_portuguese_brazil_region():
assert (PHONE_NUMBER, "(11) 2345-6789") in types_texts(
- "O número brasileiro do contato é (11) 2345-6789.", "pt"
+ "O número brasileiro do contato é (11) 2345-6789.", ["BR"]
)
def test_phone_polish_and_romanian_primary_regions():
assert (PHONE_NUMBER, "12 345 67 89") in types_texts(
- "Numer krajowy kontaktu to 12 345 67 89.", "pl"
+ "Numer krajowy kontaktu to 12 345 67 89.", ["PL"]
)
assert (PHONE_NUMBER, "021 123 4567") in types_texts(
- "Numărul național al contactului este 021 123 4567.", "ro"
+ "Numărul național al contactului este 021 123 4567.", ["RO"]
)
def test_phone_romanian_moldova_region():
assert (PHONE_NUMBER, "022 212 345") in types_texts(
- "Numărul de contact din Moldova este 022 212 345.", "ro"
+ "Numărul de contact din Moldova este 022 212 345.", ["MD"]
)
# --- overlap / priority ---
def test_no_overlapping_spans():
text = "IBAN IT60 X054 2811 1010 0000 0123 456, mail luca@example.it"
- spans = detect_deterministic(text, "it")
+ spans = detect_deterministic(text)
spans.sort(key=lambda s: s.start)
for a, b in pairwise(spans):
assert a.end <= b.start
def test_empty_text():
- assert detect_deterministic("", "de") == []
+ assert detect_deterministic("") == []
| `X-PasteGuard-Provider` | Provider used (`anthropic` or `local`) |
| `X-PasteGuard-PII-Detected` | `true` if PII was found |
| `X-PasteGuard-PII-Masked` | `true` if PII was masked (mask mode only) |
-| `X-PasteGuard-Language` | Detected language code |
-| `X-PasteGuard-Language-Fallback` | `true` if configured language was not available |
| `X-PasteGuard-Secrets-Detected` | `true` if secrets were found |
| `X-PasteGuard-Secrets-Types` | Comma-separated list of detected secret types |
| `X-PasteGuard-Secrets-Masked` | `true` if secrets were masked |
"prompt_tokens": 150,
"completion_tokens": 200,
"user_agent": "OpenAI-Python/1.0.0",
- "language": "en",
- "language_fallback": false,
- "detected_language": "en",
"masked_content": "Hello [[EMAIL_ADDRESS_1]]",
"secrets_detected": 0,
"secrets_types": null
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `text` | string | Yes | Text to scan and mask |
-| `language` | string | No | Language code for PII detection (auto-detected if not provided) |
| `startFrom` | object | No | Counter values to continue numbering from previous calls |
| `detect` | array | No | What to detect: `["pii"]`, `["secrets"]`, or `["pii", "secrets"]` (default: both) |
"entities": [
{ "type": "EMAIL_ADDRESS", "placeholder": "[[EMAIL_ADDRESS_1]]" },
{ "type": "PHONE_NUMBER", "placeholder": "[[PHONE_NUMBER_1]]" }
- ],
- "language": "en",
- "languageFallback": false
+ ]
}
```
| `context` | Mapping of placeholder to original value (for client-side unmasking) |
| `counters` | Final counter values per entity type |
| `entities` | List of detected entities with their placeholders |
-| `language` | Language used for PII detection |
-| `languageFallback` | Whether the configured fallback language was used (auto-detection failed) |
## Detection Options
| `X-PasteGuard-Provider` | Provider used (`openai` or `local`) |
| `X-PasteGuard-PII-Detected` | `true` if PII was found |
| `X-PasteGuard-PII-Masked` | `true` if PII was masked (mask mode only) |
-| `X-PasteGuard-Language` | Detected language code |
-| `X-PasteGuard-Language-Fallback` | `true` if configured language was not available |
| `X-PasteGuard-Secrets-Detected` | `true` if secrets were found |
| `X-PasteGuard-Secrets-Types` | Comma-separated list of detected secret types |
| `X-PasteGuard-Secrets-Masked` | `true` if secrets were masked |
}
},
"pii_detection": {
- "languages": ["en"],
- "fallback_language": "en",
+ "phone_regions": ["US", "GB", "DE", "IT", "IN"],
"score_threshold": 0.7,
"entities": ["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER"]
},
X-PasteGuard-Provider: openai
X-PasteGuard-PII-Detected: true
X-PasteGuard-PII-Masked: true
-X-PasteGuard-Language: en
-```
-
-If the detected language wasn't configured and fell back to `fallback_language`:
-
-```
-X-PasteGuard-Language-Fallback: true
```
## Streaming Support
description: Personal data detection via a deterministic checksum layer plus multilingual GLiNER NER
---
-PasteGuard detects PII with a small open-source service that pairs a deterministic regex/checksum layer with a multilingual neural model. Detection is **language-agnostic**: the neural model finds names and places whatever the request language — an Italian name and city in a German letter are caught even with a `de` hint — and structured identifiers (IBAN, credit card, email, IP) are matched by checksum, not by language.
+PasteGuard detects PII with a small open-source service that pairs a deterministic regex/checksum layer with a multilingual neural model. Detection is **language-agnostic**: the neural model finds names and places in mixed-language text — for example, an Italian name and city in a German letter — and structured identifiers (IBAN, credit card, email, IP) are matched by checksum, not by language.
## How it works
| `IP_ADDRESS` | 192.168.1.1 |
| `VAT_CODE` | EU VAT number, e.g. `DE136695976`, `IT00743110157`, `FR40303265045` |
-Names and locations are detected by the multilingual model (verified across EN/DE/IT/FR/ES/NL/PT and more). IBAN, credit card, phone, email, and IP work internationally. `VAT_CODE` covers any EU member state, validated per-country via the VAT checksum (`python-stdnum`) and requiring the country prefix. The structured identifiers (IBAN, credit card, IP, VAT) are checksum- or format-validated, so they are caught with high precision and without false positives on lookalike numbers.
+Names and locations are detected by the multilingual model (verified across EN/DE/IT/FR/ES/NL/PT and more). IBAN, credit card, email, and IP work internationally. Phone numbers with a `+` country prefix are global; national-format phone numbers are validated against configured `phone_regions`. `VAT_CODE` covers any EU member state, validated per-country via the VAT checksum (`python-stdnum`) and requiring the country prefix. The structured identifiers (IBAN, credit card, IP, VAT) are checksum- or format-validated, so they are caught with high precision and without false positives on lookalike numbers.
-## Languages
+## Languages and phone regions
-Detection is multilingual and **language-agnostic** — one pass over the text finds entities regardless of the language it is written in, so mixed-language documents (for example an Italian name and city in a German letter) are handled correctly. The `language` field in a request is used only as a hint, e.g. to pick a phone-number region; it does not gate detection.
+Detection is multilingual and **language-agnostic** — one pass over the text finds entities regardless of the language it is written in, so mixed-language documents (for example an Italian name and city in a German letter) are handled correctly. PasteGuard does not auto-detect language.
+
+National-format phone numbers are the exception because the same digits can be valid or invalid depending on the country numbering plan. Configure `phone_regions` for the regions your traffic commonly contains.
## Confidence Scoring
```
X-PasteGuard-PII-Detected: true
X-PasteGuard-PII-Masked: true # mask mode only
-X-PasteGuard-Language: en
-```
-
-If the fallback language was used:
-
-```
-X-PasteGuard-Language-Fallback: true
```
X-PasteGuard-Mode: route
X-PasteGuard-Provider: local
X-PasteGuard-PII-Detected: true
-X-PasteGuard-Language: en
```
When routed to OpenAI or Anthropic:
X-PasteGuard-Mode: route
X-PasteGuard-Provider: openai
X-PasteGuard-PII-Detected: false
-X-PasteGuard-Language: en
-```
-
-If the detected language wasn't configured and fell back to `fallback_language`:
-
-```
-X-PasteGuard-Language-Fallback: true
```
```yaml
pii_detection:
detector_url: http://localhost:5002
- languages:
- - en
- - de
- - it
- fallback_language: en
+ phone_regions:
+ - US
+ - GB
+ - DE
+ - IT
+ - IN
score_threshold: 0.7
entities:
- PERSON
| Option | Default | Description |
|--------|---------|-------------|
| `detector_url` | `http://localhost:5002` | Detector `/analyze` URL |
-| `languages` | `["en"]` | Languages the proxy auto-detects among (used as a hint, e.g. phone region) |
-| `fallback_language` | `en` | Language hint used when none is detected |
+| `phone_regions` | US, GB, DE, AT, CH, IT, FR, BE, LU, ES, NL, PT, BR, PL, RO, MD, IN | Regions used to validate national-format phone numbers without a country prefix |
| `score_threshold` | `0.7` | Minimum confidence floor for the neural labels PERSON and LOCATION (0.0-1.0). Checksum-validated identifiers always score `1.0` and are unaffected |
| `entities` | See below | Entity types to return |
-## Languages
+## Phone Regions
Detection is **multilingual and language-agnostic** — the model finds names and places regardless of the language the text is written in (an Italian name and city in a German letter are caught), and structured identifiers are matched by checksum independent of language. There are no per-language images and no spaCy models to load.
-The `languages` list and `fallback_language` only control the language *hint* the proxy sends with each request (which the detector uses for things like phone-number region); they do not gate detection.
+Phone numbers are different: international numbers with a `+` country prefix are detected globally, but national-format numbers need one or more country numbering plans. Configure those plans with `phone_regions` instead of relying on text language.
```yaml
pii_detection:
- languages:
- - en
- - de
- - it
- fallback_language: de
+ phone_regions:
+ - US
+ - GB
+ - DE
+ - IT
+ - IN
```
-If only one language is configured, language detection is skipped for better performance.
+Use a focused list for the traffic you expect. Adding many regions improves recall for mixed-language and international text, but can increase false positives on IDs, ticket numbers, and other digit sequences.
## Entities
docker compose up -d
```
-## Languages
+## Languages and phone numbers
-Detection is multilingual and needs no per-language configuration. The `languages`
-and `fallback_language` settings only set a hint sent with each request (e.g. for
-phone-number region) — see [PII Detection Config](/configuration/pii-detection).
+Detection is multilingual and needs no per-language configuration. Configure
+`phone_regions` only if you need national-format phone numbers without a `+`
+country prefix — see [PII Detection Config](/configuration/pii-detection).
## Environment Variables
},
"dependencies": {
"@hono/zod-validator": "^0.7.6",
- "eld": "^2.0.3",
"hono": "^4.12.23",
"hono-tailwind": "^2.2.0",
"postcss": "^8.5.15",
}
});
+ test("accepts phone regions as comma-separated config", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+ phone_regions: us,gb,in
+`);
+
+ try {
+ const config = loadConfig(path);
+
+ expect(config.pii_detection.phone_regions).toEqual(["US", "GB", "IN"]);
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
+ test("rejects invalid phone region codes", () => {
+ const path = writeConfig(`
+mode: mask
+providers:
+ openai: {}
+ anthropic: {}
+pii_detection:
+ detector_url: http://localhost:5002
+ phone_regions:
+ - USA
+`);
+
+ try {
+ expect(() => loadConfig(path)).toThrow("Invalid configuration");
+ } finally {
+ cleanupConfig(path);
+ }
+ });
+
test("rejects invalid masking allowlist regex patterns", () => {
const path = writeConfig(`
mode: mask
import { existsSync, readFileSync, statSync } from "node:fs";
import { parse as parseYaml } from "yaml";
import { z } from "zod";
-import { SUPPORTED_LANGUAGES } from "./constants/languages";
// Schema definitions
denylist: z.array(DenylistPatternSchema).default([]),
});
-const LanguageEnum = z.enum(SUPPORTED_LANGUAGES);
+const PhoneRegionSchema = z
+ .string()
+ .trim()
+ .transform((value) => value.toUpperCase())
+ .pipe(z.string().regex(/^[A-Z]{2}$/, "Expected ISO 3166-1 alpha-2 region code"));
-// Accept either an array or a comma-separated string for languages (the latter
-// supports ${ENV_VAR} substitution in the YAML config, e.g. languages: ${LANGS:-en}).
-const LanguagesSchema = z
- .union([z.array(LanguageEnum), z.string()])
+const PhoneRegionsSchema = z
+ .union([z.array(PhoneRegionSchema), z.string()])
.transform((val) => {
if (Array.isArray(val)) return val;
- return val.split(",").map((s) => s.trim()) as (typeof SUPPORTED_LANGUAGES)[number][];
+ return val
+ .split(",")
+ .map((s) => s.trim())
+ .filter(Boolean);
})
- .pipe(z.array(LanguageEnum))
- .default(["en"]);
+ .pipe(z.array(PhoneRegionSchema).min(1))
+ .default([
+ "US",
+ "GB",
+ "DE",
+ "AT",
+ "CH",
+ "IT",
+ "FR",
+ "BE",
+ "LU",
+ "ES",
+ "NL",
+ "PT",
+ "BR",
+ "PL",
+ "RO",
+ "MD",
+ "IN",
+ ]);
const PIIDetectionSchema = z.object({
enabled: z.boolean().default(true),
detector_url: z.string().url(),
- languages: LanguagesSchema,
- fallback_language: LanguageEnum.default("en"),
+ phone_regions: PhoneRegionsSchema,
score_threshold: z.coerce.number().min(0).max(1).default(0.7),
entities: z
.array(z.string())
+++ /dev/null
-/**
- * Languages the proxy can detect and send as a hint to the detector.
- * Detection itself is language-agnostic; this list only gates the hint.
- */
-export const SUPPORTED_LANGUAGES = [
- "ca", // Catalan
- "zh", // Chinese
- "hr", // Croatian
- "da", // Danish
- "nl", // Dutch
- "en", // English
- "fi", // Finnish
- "fr", // French
- "de", // German
- "el", // Greek
- "it", // Italian
- "ja", // Japanese
- "ko", // Korean
- "lt", // Lithuanian
- "mk", // Macedonian
- "nb", // Norwegian
- "pl", // Polish
- "pt", // Portuguese
- "ro", // Romanian
- "ru", // Russian
- "sl", // Slovenian
- "es", // Spanish
- "sv", // Swedish
- "uk", // Ukrainian
-] as const;
-
-export type SupportedLanguage = (typeof SUPPORTED_LANGUAGES)[number];
}
console.log("[STARTUP] ✓ Detector connected");
- // Language config is validated at load time (config.ts LanguageEnum); the
- // detector is language-agnostic, so there is no runtime language gate.
}
function printStartupBanner(config: ReturnType<typeof getConfig>, host: string, port: number) {
${modeInfo}
PII Detection:
- Languages: ${config.pii_detection.languages.join(", ")}
- Fallback: ${config.pii_detection.fallback_language}
+ Phone regions: ${config.pii_detection.phone_regions.join(", ")}
Threshold: ${config.pii_detection.score_threshold}
Entities: ${config.pii_detection.entities.join(", ")}
Array<{ entity_type: string; start: number; end: number; score: number }>
>,
) {
+ const analyzeRequests: unknown[] = [];
+
globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
const urlStr = url.toString();
if (urlStr.includes("/analyze") && init?.body) {
const body = JSON.parse(init.body as string);
+ analyzeRequests.push(body);
const text = body.text as string;
for (const [key, entities] of Object.entries(responses)) {
return originalFetch(url, init);
}) as unknown as typeof fetch;
+
+ return analyzeRequests;
}
function createRequest(messages: OpenAIMessage[]): OpenAIRequest {
});
const detector = new PIIDetector();
- const entities = await detector.detectPII("test@example.com", "en");
+ const entities = await detector.detectPII("test@example.com");
expect(entities).toHaveLength(1);
expect(entities[0].entity_type).toBe("EMAIL_ADDRESS");
mockDetector({});
const detector = new PIIDetector();
- const entities = await detector.detectPII("Hello world", "en");
+ const entities = await detector.detectPII("Hello world");
expect(entities).toHaveLength(0);
});
+
+ test("sends configured phone regions to the detector", async () => {
+ const config = getConfig();
+ const previousPhoneRegions = config.pii_detection.phone_regions;
+ config.pii_detection.phone_regions = ["US", "IN", "IT"];
+ const analyzeRequests = mockDetector({});
+
+ try {
+ const detector = new PIIDetector();
+ await detector.detectPII("Call 080 1234 5678");
+
+ expect(analyzeRequests).toHaveLength(1);
+ expect(analyzeRequests[0]).toMatchObject({
+ text: "Call 080 1234 5678",
+ phone_regions: ["US", "IN", "IT"],
+ });
+ } finally {
+ config.pii_detection.phone_regions = previousPhoneRegions;
+ }
+ });
});
describe("healthCheck", () => {
import { HEALTH_CHECK_TIMEOUT_MS } from "../constants/timeouts";
import { overlaps, resolveConflicts } from "../masking/conflict-resolver";
import type { RequestExtractor } from "../masking/types";
-import { getLanguageDetector, type SupportedLanguage } from "../services/language-detector";
export interface PIIEntity {
entity_type: string;
interface AnalyzeRequest {
text: string;
- language: string;
+ phone_regions?: string[];
entities?: string[];
score_threshold?: number;
}
spanEntities: PIIEntity[][];
allEntities: PIIEntity[];
scanTimeMs: number;
- language: SupportedLanguage;
- languageFallback: boolean;
- detectedLanguage?: string;
}
export class PIIDetector {
private detectorUrl: string;
private scoreThreshold: number;
private entityTypes: string[];
+ private phoneRegions: string[];
constructor() {
const config = getConfig();
this.detectorUrl = config.pii_detection.detector_url;
this.scoreThreshold = config.pii_detection.score_threshold;
this.entityTypes = config.pii_detection.entities;
+ this.phoneRegions = config.pii_detection.phone_regions;
}
- async detectPII(text: string
, language: SupportedLanguage): Promise<PIIEntity[]> {
+ async detectPII(text: string): Promise<PIIEntity[]> {
const analyzeEndpoint = `${this.detectorUrl}/analyze`;
const request: AnalyzeRequest = {
text,
- language,
+ phone_regions: this.phoneRegions,
entities: this.entityTypes,
score_threshold: this.scoreThreshold,
};
const startTime = Date.now();
const config = getConfig();
- // Pure pass-through: detection off and no denylist, so skip extraction and language detection.
if (!config.pii_detection.enabled && config.masking.denylist.length === 0) {
return {
hasPII: false,
spanEntities: [],
allEntities: [],
scanTimeMs: 0,
- language: config.pii_detection.fallback_language,
- languageFallback: true,
};
}
// Extract all text spans from request
const spans = extractor.extractTexts(request);
- // Detect language from message content (skip system spans with messageIndex -1)
- const messageSpans = spans.filter((span) => span.messageIndex >= 0);
- const langText = messageSpans.map((s) => s.text).join("\n");
- const langResult = langText
- ? getLanguageDetector().detect(langText)
- : { language: config.pii_detection.fallback_language, usedFallback: true };
-
// Detect PII for each span independently
const scanRoles = config.pii_detection.scan_roles
? new Set(config.pii_detection.scan_roles)
}
const detectedEntities = config.pii_detection.enabled
- ? await this.detectPII(span.text, langResult.language)
+ ? await this.detectPII(span.text)
: [];
const filteredEntities = filterAllowlistedEntities(span.text, detectedEntities, allowlist);
return mergeDenylistEntities(filteredEntities, denylistedEntities);
spanEntities,
allEntities,
scanTimeMs: Date.now() - startTime,
- language: langResult.language,
- languageFallback: langResult.usedFallback,
- detectedLanguage: langResult.detectedLanguage,
};
}
} from "../pii/detect";
// Mock the PII detector to avoid needing the detector running
-const mockDetectPII = mock<(text: string, language: string) => Promise<PIIEntity[]>>(() =>
- Promise.resolve([]),
-);
+const mockDetectPII = mock<(text: string) => Promise<PIIEntity[]>>(() => Promise.resolve([]));
mock.module("../pii/detect", () => ({
getPIIDetector: () => ({
detectPII: mockDetectPII,
masked: string;
context: Record<string, string>;
entities: unknown[];
- language: string;
};
expect(body.masked).toBe("Hello world");
expect(body.context).toEqual({});
expect(body.entities).toEqual([]);
- expect(body.language).toBeDefined();
});
test("masks PII entities", async () => {
expect(body.error.details[0].message).toBe("Detector connection failed");
});
- test("includes languageFallback in response", async () => {
- mockDetectPII.mockResolvedValueOnce([]);
-
- const res = await app.request("/api/mask", {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ text: "Hello world" }),
- });
-
- expect(res.status).toBe(200);
- const body = (await res.json()) as {
- languageFallback: boolean;
- };
- expect(typeof body.languageFallback).toBe("boolean");
- });
-
test("respects multiple entity types in startFrom", async () => {
mockDetectPII.mockResolvedValueOnce([
{ entity_type: "PERSON", start: 0, end: 4, score: 0.9 },
import { mask as maskPII } from "../pii/mask";
import { detectSecrets } from "../secrets/detect";
import { maskSecrets } from "../secrets/mask";
-import { getLanguageDetector, type SupportedLanguage } from "../services/language-detector";
import { logRequest, normalizeRequestSource } from "../services/logger";
import { createLogData } from "./utils";
// Request schema
const MaskRequestSchema = z.object({
text: z.string().trim().min(1, "text is required"),
- language: z.string().optional(),
startFrom: z.record(z.string(), z.number()).optional(),
detect: z.array(z.enum(["pii", "secrets"])).optional(),
});
context: Record<string, string>;
counters: Record<string, number>;
entities: MaskEntity[];
- language: string;
- languageFallback: boolean;
}
/**
}
}
- // Detect language (use provided or auto-detect)
- let language: SupportedLanguage;
- let languageFallback = false;
- if (
- request.language &&
- config.pii_detection.languages.includes(request.language as SupportedLanguage)
- ) {
- language = request.language as SupportedLanguage;
- } else {
- const langResult = getLanguageDetector().detect(request.text);
- language = langResult.language;
- languageFallback = langResult.usedFallback;
- }
-
let maskedText = request.text;
const allEntities: MaskEntity[] = [];
const piiEntityTypes: string[] = [];
pii: {
hasPII: piiEntityTypes.length > 0,
entityTypes: piiEntityTypes,
- language,
- languageFallback,
scanTimeMs,
},
statusCode: 503,
try {
const piiStartTime = Date.now();
const detector = getPIIDetector();
- const piiEntities = config.pii_detection.enabled
- ? await detector.detectPII(maskedText, language)
- : [];
+ const piiEntities = config.pii_detection.enabled ? await detector.detectPII(maskedText) : [];
scanTimeMs = Date.now() - piiStartTime;
const filteredEntities = filterAllowlistedEntities(
source,
model: "mask",
startTime,
- pii: { hasPII: false, entityTypes: [], language, languageFallback, scanTimeMs: 0 },
+ pii: { hasPII: false, entityTypes: [], scanTimeMs: 0 },
statusCode: 503,
errorMessage: error instanceof Error ? error.message : "PII detection failed",
}),
pii: {
hasPII: piiEntityTypes.length > 0,
entityTypes: piiEntityTypes,
- language,
- languageFallback,
scanTimeMs,
},
secrets:
context: context.mapping,
counters: { ...context.counters },
entities: allEntities,
- language,
- languageFallback,
};
return c.json(response);
spanEntities: [],
allEntities: [],
scanTimeMs: 0,
- language: "en",
- languageFallback: false,
}),
);
const mockLogRequest = mock(() => {});
spanEntities: [],
allEntities: [],
scanTimeMs: 0,
- language: "en",
- languageFallback: false,
});
mockLogRequest.mockClear();
});
spanEntities: [[{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }]],
allEntities: [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }],
scanTimeMs: 3,
- language: "en",
- languageFallback: false,
});
const calls: CapturedRequest[] = [];
spanEntities: [[{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }]],
allEntities: [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }],
scanTimeMs: 3,
- language: "en",
- languageFallback: false,
});
let fetchCalled = false;
spanEntities: [[{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }]],
allEntities: [{ entity_type: "EMAIL_ADDRESS", start: 6, end: 22, score: 0.99 }],
scanTimeMs: 3,
- language: "en",
- languageFallback: false,
});
globalThis.fetch = (async (_input: string | URL | Request, _init?: RequestInit) =>
mode: config.mode,
providers,
pii_detection: {
- languages: config.pii_detection.languages,
- fallback_language: config.pii_detection.fallback_language,
+ phone_regions: config.pii_detection.phone_regions,
score_threshold: config.pii_detection.score_threshold,
entities: config.pii_detection.entities,
},
export interface PIIHeaderData {
hasPII: boolean;
- language: string;
- languageFallback: boolean;
}
export interface SecretsHeaderData {
c.header("X-PasteGuard-Mode", mode);
c.header("X-PasteGuard-Provider", provider);
c.header("X-PasteGuard-PII-Detected", pii.hasPII.toString());
- c.header("X-PasteGuard-Language", pii.language);
- if (pii.languageFallback) {
- c.header("X-PasteGuard-Language-Fallback", "true");
- }
if (mode === "mask" && pii.hasPII) {
c.header("X-PasteGuard-PII-Masked", "true");
}
export interface PIILogData {
hasPII: boolean;
entityTypes: string[];
- language: string;
- languageFallback: boolean;
- detectedLanguage?: string;
scanTimeMs: number;
}
return {
hasPII: piiResult.hasPII,
entityTypes: [...new Set(piiResult.detection.allEntities.map((e) => e.entity_type))],
- language: piiResult.detection.language,
- languageFallback: piiResult.detection.languageFallback,
- detectedLanguage: piiResult.detection.detectedLanguage,
scanTimeMs: piiResult.detection.scanTimeMs,
};
}
export function toPIIHeaderData(piiResult: PIIDetectResult): PIIHeaderData {
return {
hasPII: piiResult.hasPII,
- language: piiResult.detection.language,
- languageFallback: piiResult.detection.languageFallback,
};
}
entities: pii?.entityTypes ?? [],
latencyMs: Date.now() - startTime,
scanTimeMs: pii?.scanTimeMs ?? 0,
- language: pii?.language ?? config.pii_detection.fallback_language,
- languageFallback: pii?.languageFallback ?? false,
- detectedLanguage: pii?.detectedLanguage,
maskedContent,
secretsDetected: secrets?.detected,
secretsMasked: secrets?.masked,
+++ /dev/null
-import eld from "eld/small";
-import { getConfig } from "../config";
-import type { SupportedLanguage } from "../constants/languages";
-
-export type { SupportedLanguage } from "../constants/languages";
-
-export interface LanguageDetectionResult {
- language: SupportedLanguage;
- usedFallback: boolean;
- detectedLanguage?: string;
- confidence?: number;
-}
-
-// Map detected ISO codes onto the supported-language list where they differ.
-const ISO_TO_SUPPORTED_OVERRIDES: Record<string, SupportedLanguage> = {
- no: "nb", // Norwegian (generic) → Norwegian Bokmål
-};
-
-export class LanguageDetector {
- private configuredLanguages: SupportedLanguage[];
- private fallbackLanguage: SupportedLanguage;
-
- constructor() {
- const config = getConfig();
- this.configuredLanguages = config.pii_detection.languages;
- this.fallbackLanguage = config.pii_detection.fallback_language;
- }
-
- detect(text: string): LanguageDetectionResult {
- const result = eld.detect(text);
- const detectedIso = result.language;
- const scores = result.getScores();
- const confidence = scores[detectedIso] ?? 0;
-
- // Use override if exists, otherwise use the detected code as-is (most are 1:1)
- const mappedLang = (ISO_TO_SUPPORTED_OVERRIDES[detectedIso] ||
- detectedIso) as SupportedLanguage;
-
- if (mappedLang && this.configuredLanguages.includes(mappedLang)) {
- return {
- language: mappedLang,
- usedFallback: false,
- detectedLanguage: detectedIso,
- confidence,
- };
- }
-
- return {
- language: this.fallbackLanguage,
- usedFallback: true,
- detectedLanguage: detectedIso,
- confidence,
- };
- }
-}
-
-let detectorInstance: LanguageDetector | null = null;
-
-export function getLanguageDetector(): LanguageDetector {
- if (!detectorInstance) {
- detectorInstance = new LanguageDetector();
- }
- return detectorInstance;
-}
prompt_tokens: number | null;
completion_tokens: number | null;
user_agent: string | null;
- language: string;
- language_fallback: boolean;
- detected_language: string | null;
masked_content: string | null;
secrets_detected: number | null;
secrets_types: string | null;
prompt_tokens INTEGER,
completion_tokens INTEGER,
user_agent TEXT,
- language TEXT NOT NULL DEFAULT 'en',
- language_fallback INTEGER NOT NULL DEFAULT 0,
- detected_language TEXT,
masked_content TEXT,
secrets_detected INTEGER,
secrets_types TEXT,
log(entry: Omit<RequestLog, "id">): void {
const stmt = this.db.prepare(`
INSERT INTO request_logs
- (timestamp, mode, provider, source, model, pii_detected, entities, latency_ms, scan_time_ms, prompt_tokens, completion_tokens, user_agent, language, language_fallback, detected_language, masked_content, secrets_detected, secrets_types, status_code, error_message)
+ (timestamp, mode, provider, source, model, pii_detected, entities, latency_ms, scan_time_ms, prompt_tokens, completion_tokens, user_agent, masked_content, secrets_detected, secrets_types, status_code, error_message)
VALUES
- (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
stmt.run(
entry.prompt_tokens,
entry.completion_tokens,
entry.user_agent,
- entry.language,
- entry.language_fallback ? 1 : 0,
- entry.detected_language,
entry.masked_content,
entry.secrets_detected ?? null,
entry.secrets_types ?? null,
*/
getLogs(limit: number = 100, offset: number = 0): RequestLog[] {
const stmt = this.db.prepare(`
- SELECT * FROM request_logs
+ SELECT
+ id,
+ timestamp,
+ mode,
+ provider,
+ source,
+ model,
+ pii_detected,
+ entities,
+ latency_ms,
+ scan_time_ms,
+ prompt_tokens,
+ completion_tokens,
+ user_agent,
+ masked_content,
+ secrets_detected,
+ secrets_types,
+ status_code,
+ error_message
+ FROM request_logs
ORDER BY timestamp DESC
LIMIT ? OFFSET ?
`);
scanTimeMs: number;
promptTokens?: number;
completionTokens?: number;
- language: string;
- languageFallback: boolean;
- detectedLanguage?: string;
maskedContent?: string;
secretsDetected?: boolean;
secretsMasked?: boolean;
prompt_tokens: data.promptTokens ?? null,
completion_tokens: data.completionTokens ?? null,
user_agent: userAgent,
- language: data.language,
- language_fallback: data.languageFallback,
- detected_language: data.detectedLanguage ?? null,
masked_content: shouldLogContent ? (data.maskedContent ?? null) : null,
secrets_detected: data.secretsDetected !== undefined ? (data.secretsDetected ? 1 : 0) : null,
secrets_types: shouldLogSecretTypes ? data.secretsTypes!.join(",") : null,
* Test utilities for creating detection results
*/
-import type { SupportedLanguage } from "../constants/languages";
import type { PIIDetectionResult, PIIEntity } from "../pii/detect";
import type { MessageSecretsResult, SecretLocation } from "../secrets/detect";
export function createPIIResultFromSpans(
spanEntities: PIIEntity[][],
options: {
- language?: SupportedLanguage;
- languageFallback?: boolean;
- detectedLanguage?: string;
scanTimeMs?: number;
} = {},
): PIIDetectionResult {
spanEntities,
allEntities,
scanTimeMs: options.scanTimeMs ?? 0,
- language: options.language ?? "en",
- languageFallback: options.languageFallback ?? false,
- detectedLanguage: options.detectedLanguage,
};
}
<th class="bg-elevated font-mono text-[0.65rem] font-medium uppercase tracking-widest text-text-muted px-4 py-3.5 text-left border-b border-border sticky top-0">
Model
</th>
- <th class="bg-elevated font-mono text-[0.65rem] font-medium uppercase tracking-widest text-text-muted px-4 py-3.5 text-left border-b border-border sticky top-0">
- Language
- </th>
<th class="bg-elevated font-mono text-[0.65rem] font-medium uppercase tracking-widest text-text-muted px-4 py-3.5 text-left border-b border-border sticky top-0">
PII Entities
</th>
</thead>
<tbody id="logs-body">
<tr>
- <td colSpan={8}>
+ <td colSpan={7}>
<div class="flex flex-col justify-center items-center p-10 gap-3">
<div class="loader-bars">
<div class="loader-bar" />
const tbody = document.getElementById('logs-body');
if (data.logs.length === 0) {
- tbody.innerHTML = '<tr><td colspan="
8"><div class="text-center py-10 text-text-muted"><div class="text-2xl mb-3 opacity-40">📋</div><div class="text-sm">No requests yet</div></div></td></tr>';
+ tbody.innerHTML = '<tr><td colspan="
7"><div class="text-center py-10 text-text-muted"><div class="text-2xl mb-3 opacity-40">📋</div><div class="text-sm">No requests yet</div></div></td></tr>';
return;
}
const secretsTypes = log.secrets_types ? log.secrets_types.split(',').filter(s => s.trim()) : [];
const secretsDetected = log.secrets_detected === 1;
const isError = log.status_code && log.status_code >= 400;
- const lang = log.language || 'en';
- const detectedLang = log.detected_language;
const source = log.source;
-
- const formatLang = (code) => code ? code.toUpperCase() : lang.toUpperCase();
-
- // Show original→fallback when fallback was used (e.g. FR→EN)
- const langDisplay = log.language_fallback && detectedLang
- ? '<span class="text-accent" title="Language not supported, fallback used">' + formatLang(detectedLang) + '</span><span class="text-text-muted text-[0.5rem] mx-0.5">→</span><span>' + lang.toUpperCase() + '</span>'
- : lang.toUpperCase();
const logId = log.id || index;
const isExpanded = expandedRowId === logId;
'<td class="text-sm px-4 py-3 border-b border-border-subtle align-middle">' + sourceBadge + '</td>' +
'<td class="text-sm px-4 py-3 border-b border-border-subtle align-middle">' + statusBadge + '</td>' +
'<td class="font-mono text-[0.7rem] text-text-secondary px-4 py-3 border-b border-border-subtle align-middle">' + log.model + '</td>' +
- '<td class="font-mono text-[0.65rem] font-medium px-4 py-3 border-b border-border-subtle align-middle">' + langDisplay + '</td>' +
'<td class="text-sm px-4 py-3 border-b border-border-subtle align-middle">' +
(entities.length > 0
? '<div class="flex flex-wrap gap-1">' + entities.map(e => '<span class="font-mono text-[0.55rem] px-1.5 py-0.5 bg-elevated border border-border rounded-sm text-text-secondary">' + e.trim() + '</span>').join('') + '</div>'
const detailRow =
'<tr id="detail-' + logId + '" class="' + (isExpanded ? 'detail-row-visible' : 'hidden') + '">' +
- '<td colspan="8" class="p-0 bg-detail border-b border-border-subtle">' +
+ '<td colspan="7" class="p-0 bg-detail border-b border-border-subtle">' +
'<div class="p-4 px-5 animate-slide-down">' + detailContent + '</div>' +
'</td>' +
'</tr>';
git.99rst.org / sgasser-llm-shield.git / commitdiff