From: Stefan Gasser Date: Thu, 30 Jul 2026 05:51:29 +0000 (+0200) Subject: Fix person detection beside customer roles (#160) X-Git-Tag: v0.9.1~2 X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=7a9885dde26cbd56918729e31dc3f63f56a973e0;p=sgasser-llm-shield.git Fix person detection beside customer roles (#160) --- diff --git a/benchmarks/pii-accuracy/test-data/core.yaml b/benchmarks/pii-accuracy/test-data/core.yaml index d847c2d..ba003d6 100644 --- a/benchmarks/pii-accuracy/test-data/core.yaml +++ b/benchmarks/pii-accuracy/test-data/core.yaml @@ -285,6 +285,50 @@ cases: text: John Smith match: contains + - id: core_person_customer_copula_en + suite: core + category: core + split: test + language: en + text: "The customer is Marco Rossi." + expected: + - entity: PERSON + text: Marco Rossi + match: contains + + - id: core_person_customer_direct_object_en + suite: core + category: core + split: test + language: en + text: "Please reply to the customer Marco Rossi." + expected: + - entity: PERSON + text: Marco Rossi + match: contains + + - id: core_person_kunde_de + suite: core + category: core + split: test + language: de + text: "Unser Kunde Thomas Müller hat angerufen." + expected: + - entity: PERSON + text: Thomas Müller + match: contains + + - id: core_person_customer_turkish_name_en + suite: core + category: core + split: test + language: en + text: "The customer is Mehmet Yılmaz." + expected: + - entity: PERSON + text: Mehmet Yılmaz + match: contains + - id: core_person_non_western_arabic suite: core category: core diff --git a/benchmarks/pii-accuracy/test-data/precision.yaml b/benchmarks/pii-accuracy/test-data/precision.yaml index 1d4dbc5..4c5b4f7 100644 --- a/benchmarks/pii-accuracy/test-data/precision.yaml +++ b/benchmarks/pii-accuracy/test-data/precision.yaml @@ -199,7 +199,7 @@ cases: language: ro entities: [PERSON, LOCATION] gate: false - note: "GLiNER tags the Romanian role noun 'Clientul' as PERSON (~0.85); the suppressor labels don't demote it here. Report-only." + note: "GLiNER can tag the Romanian role noun 'Clientul' as PERSON (~0.85); this report-only case guards PERSON-floor calibration." text: "Clientul cere factura până la sfârșitul săptămânii." expected: [] @@ -241,10 +241,8 @@ cases: text: "Sarcina a fost mutată din eu-west-1 în us-east-1 în timpul testului." expected: [] - # --- Known over-masking after removing the stoplist (report-only) --- - # These document false positives GLiNER produces with high confidence and that - # no competing suppressor label separates. Tracked here so the benchmark stops - # hiding them; not gating. + # --- Known semantic-model over-masking (report-only) --- + # These document model false positives near the calibrated per-label floors. - id: precision_de_pronoun_not_person suite: precision category: precision @@ -252,7 +250,7 @@ cases: language: de entities: [PERSON, LOCATION] gate: false - note: "GLiNER tags the German pronoun 'Ich' as PERSON (~0.97), above the floor and not separable by a competing label. Report-only." + note: "GLiNER can tag the German pronoun 'Ich' as PERSON (~0.97); this report-only case guards PERSON-floor calibration." text: "Ich habe die Rechnung bereits bezahlt." expected: [] @@ -274,7 +272,7 @@ cases: language: de entities: [PERSON, LOCATION] gate: false - note: "GLiNER tags the German term 'Mandant' (client) as PERSON (~0.9) even against suppressor labels; it is not suppressed. Report-only." + note: "GLiNER can tag the German term 'Mandant' (client) as PERSON (~0.9); this report-only case guards PERSON-floor calibration." text: "Der Mandant hat die Unterlagen eingereicht." expected: [] diff --git a/detector/detector/gliner_layer.py b/detector/detector/gliner_layer.py index 7c134d2..e832500 100644 --- a/detector/detector/gliner_layer.py +++ b/detector/detector/gliner_layer.py @@ -1,6 +1,5 @@ """Fuzzy layer: multilingual GLiNER NER for person, location, and address -(addresses are emitted as LOCATION). Generic role nouns are demoted via -competing suppressor labels rather than a per-language denylist. +(addresses are emitted as LOCATION). Each label has its own confidence floor (PER_LABEL_FLOOR). The request `score_threshold` only raises the tunable labels (person, location, address). @@ -65,10 +64,9 @@ def _max_tokens(default: int = 384) -> int: # Per-label confidence floors (calibrated against the accuracy benchmark; -# overridable via env, e.g. GLINER_FLOOR_LOCATION=0.6). Role nouns are demoted -# by the suppressor labels (below), not by this floor. +# overridable via env, e.g. GLINER_FLOOR_LOCATION=0.6). PER_LABEL_FLOOR = { - "person": _floor("person", 0.95), + "person": _floor("person", 0.99), "location": _floor("location", 0.80), # A dedicated "address" label recovers full street addresses that a bare # "location" reading misses; emitted as LOCATION (see _LABEL_TO_TYPE). @@ -85,11 +83,6 @@ _LABEL_TO_TYPE = { # LOCATION so the response entity set stays the Presidio drop-in set. "address": LOCATION, } -# Suppressor labels: predicted but never emitted. They give GLiNER a competing -# reading for generic role nouns (client/customer/...) in any language, so it -# tags those instead of "person" — no per-language denylist needed. -_SUPPRESS_LABELS = ["customer", "role"] -_PREDICT_LABELS = _LABELS + _SUPPRESS_LABELS # Capture candidates below every floor so per-label filtering has them. _PREDICT_FLOOR = min(PER_LABEL_FLOOR.values()) - 0.1 @@ -154,7 +147,7 @@ def load_model(model_name: str = DEFAULT_MODEL) -> None: @lru_cache(maxsize=MODEL_INFERENCE_CACHE_SIZE) def _predict_window(text: str): - return _model.predict_entities(text, _PREDICT_LABELS, threshold=max(0.0, _PREDICT_FLOOR)) + return _model.predict_entities(text, _LABELS, threshold=max(0.0, _PREDICT_FLOOR)) def detect_gliner(text: str, score_threshold: float = 0.0) -> list[Span]: @@ -174,17 +167,8 @@ def detect_gliner(text: str, score_threshold: float = 0.0) -> list[Span]: if score > best.get(key, -1.0): best[key] = score - # Highest suppressor score per span: if a "customer"/"role" reading of the - # exact same span outscores its entity reading, it is a role noun, not PII. - suppressor: dict[tuple[int, int], float] = {} - for (start, end, label), score in best.items(): - if label in _SUPPRESS_LABELS and score > suppressor.get((start, end), -1.0): - suppressor[start, end] = score - out: list[Span] = [] for (start, end, label), score in best.items(): - if label in _SUPPRESS_LABELS: - continue # label is always one of _LABELS (== _LABEL_TO_TYPE keys), so a direct # lookup is safe. etype = _LABEL_TO_TYPE[label] @@ -193,9 +177,6 @@ def detect_gliner(text: str, score_threshold: float = 0.0) -> list[Span]: floor = max(floor, score_threshold) if score < floor: continue - # A stronger role-noun reading of the same span wins (drops the entity). - if score < suppressor.get((start, end), -1.0): - continue # Drop out-of-bounds spans from tokenization bugs (would mask wrong text). if not 0 <= start < end <= n: continue diff --git a/detector/tests/test_gliner_layer.py b/detector/tests/test_gliner_layer.py index 62d2c11..3da8a5a 100644 --- a/detector/tests/test_gliner_layer.py +++ b/detector/tests/test_gliner_layer.py @@ -1,7 +1,6 @@ """Unit tests for the GLiNER layer's precision calibration (no model load). -The role-noun suppressor labels and per-label floors are the precision layer; the -full model integration is covered by benchmarks/pii-accuracy. +Full model integration is covered by benchmarks/pii-accuracy. """ from unittest.mock import Mock @@ -11,7 +10,6 @@ import pytest import detector.gliner_layer as layer from detector.gliner_layer import ( _MAX_TOKENS, - _SUPPRESS_LABELS, _TOKEN_RE, PER_LABEL_FLOOR, Span, @@ -34,12 +32,6 @@ def test_token_re_matches_gliner_splitter(): assert _TOKEN_RE.pattern == WhitespaceTokenSplitter().whitespace_pattern.pattern -def test_role_nouns_handled_by_suppressor_labels(): - # Generic role nouns are disambiguated language-agnostically by competing - # suppressor labels rather than any hard-coded denylist. - assert "customer" in _SUPPRESS_LABELS - - def test_windows_single_for_short_text(): text = "Mario Rossi lives in Rome." assert list(_windows(text)) == [(0, text)] @@ -60,6 +52,7 @@ def test_windows_overlapping_and_cover_long_text(): def test_per_label_floors_present_and_ordered(): assert set(PER_LABEL_FLOOR) == {"person", "location", "address"} assert all(0.0 <= v <= 1.0 for v in PER_LABEL_FLOOR.values()) + assert PER_LABEL_FLOOR["person"] == 0.99 # Person carries a stricter floor than location: higher volume and no # structural validator, so a higher floor curbs false positives. assert PER_LABEL_FLOOR["location"] <= PER_LABEL_FLOOR["person"] @@ -103,21 +96,21 @@ def test_floor_env_defaults(monkeypatch): monkeypatch.delenv("GLINER_FLOOR_PERSON", raising=False) monkeypatch.delenv("DETECTOR_FLOOR_PERSON", raising=False) - assert _floor("person", 0.95) == 0.95 + assert _floor("person", 0.99) == 0.99 def test_gliner_floor_env_wins_over_existing_legacy_name(monkeypatch): monkeypatch.setenv("GLINER_FLOOR_PERSON", "0.91") monkeypatch.setenv("DETECTOR_FLOOR_PERSON", "0.50") - assert _floor("person", 0.95) == 0.91 + assert _floor("person", 0.99) == 0.91 def test_existing_floor_env_remains_supported(monkeypatch): monkeypatch.delenv("GLINER_FLOOR_PERSON", raising=False) monkeypatch.setenv("DETECTOR_FLOOR_PERSON", "0.90") - assert _floor("person", 0.95) == 0.90 + assert _floor("person", 0.99) == 0.90 def test_invalid_existing_floor_env_names_the_source(monkeypatch): @@ -128,7 +121,7 @@ def test_invalid_existing_floor_env_names_the_source(monkeypatch): ValueError, match="DETECTOR_FLOOR_PERSON must be a number between 0 and 1", ): - _floor("person", 0.95) + _floor("person", 0.99) @pytest.mark.parametrize("value", ["bad", "-0.1", "1.1", "nan", "inf"]) @@ -136,7 +129,37 @@ def test_invalid_gliner_floor_fails_clearly(monkeypatch, value): monkeypatch.setenv("GLINER_FLOOR_PERSON", value) with pytest.raises(ValueError, match="GLINER_FLOOR_PERSON must be a number between 0 and 1"): - _floor("person", 0.95) + _floor("person", 0.99) + + +def test_gliner_calls_model_with_only_emitted_semantic_labels(monkeypatch): + text = "No personal data is present." + mock_gliner = Mock() + mock_gliner.predict_entities.return_value = [] + monkeypatch.setattr(layer, "_model", mock_gliner) + layer._predict_window.cache_clear() + + assert detect_gliner(text, 0.0) == [] + + labels = mock_gliner.predict_entities.call_args.args[1] + assert labels == ["person", "location", "address"] + + +def test_detect_does_not_suppress_emitted_semantic_labels(monkeypatch): + monkeypatch.setattr(layer, "_model", Mock()) + monkeypatch.setattr( + layer, + "_predict_window", + lambda _text: [ + {"start": 0, "end": 5, "label": "person", "score": 0.995}, + {"start": 0, "end": 5, "label": "location", "score": 1.0}, + ], + ) + + assert detect_gliner("Alice", 0.0) == [ + Span(entity_type="PERSON", start=0, end=5, score=0.995), + Span(entity_type="LOCATION", start=0, end=5, score=1.0), + ] def test_max_tokens_env_defaults(monkeypatch): diff --git a/docs/configuration/gliner.mdx b/docs/configuration/gliner.mdx index f61ba9d..a0bec35 100644 --- a/docs/configuration/gliner.mdx +++ b/docs/configuration/gliner.mdx @@ -71,7 +71,7 @@ docker run --rm -p 3000:3000 \ | Variable | Default | Description | |----------|---------|-------------| -| `GLINER_FLOOR_PERSON` | `0.95` | Minimum confidence for person detections | +| `GLINER_FLOOR_PERSON` | `0.99` | Minimum confidence for person detections | | `GLINER_FLOOR_LOCATION` | `0.80` | Minimum confidence for location detections | | `GLINER_FLOOR_ADDRESS` | `0.80` | Minimum confidence for street-address detections | | `GLINER_MAX_TOKENS` | `384` | Maximum model tokens per input window; must be at least `64` |