From 7adeaa1c3d5c73e2a01dfc5fc324bbcf56abcf5d Mon Sep 17 00:00:00 2001 From: Ashay Date: Sat, 8 Aug 2026 18:23:46 +0530 Subject: [PATCH] Move mishearing/confirmation vocab into locale packs Hardcoded English meant non-English calls silently returned zero findings from the two headline checks. Ship en + es packs, allow custom packs per suite, and report locale_unavailable (not a pass) when no pack is loaded for the call language. English default pack keeps existing fixture behaviour. Fixes #3 --- README.md | 31 ++++++++ fixtures/es_misheard_call.json | 38 ++++++++++ pyproject.toml | 3 + tests/test_voiceeval.py | 72 ++++++++++++++++++ voiceeval/checks.py | 133 +++++++++++++++++++++++++-------- voiceeval/locale.py | 85 +++++++++++++++++++++ voiceeval/locales/en.json | 45 +++++++++++ voiceeval/locales/es.json | 31 ++++++++ voiceeval/turns.py | 4 + 9 files changed, 412 insertions(+), 30 deletions(-) create mode 100644 fixtures/es_misheard_call.json create mode 100644 voiceeval/locale.py create mode 100644 voiceeval/locales/en.json create mode 100644 voiceeval/locales/es.json diff --git a/README.md b/README.md index 6ad30b1..7c3d5b0 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,37 @@ nobody uses. construction**. There is a test that documents exactly this limitation. That is the argument for scripted test calls: in production this failure is silent, and no tool can fix that for you. + + +## Locale packs + +Mis-hearing and confirmation checks are language-sensitive. English ships as +`voiceeval/locales/en.json`; Spanish is included as an example (`es.json`). + +- Default: calls with `"language": "en"` (or omitted) use the English pack. +- Suites can load extra packs or pass a custom `LocalePack` into `analyse()`. +- A call whose language has **no pack loaded** reports `locale_unavailable` + (high severity) for those checks — it is **not** counted as a pass. + +```json +// fixtures/es_misheard_call.json +{ "id": "...", "language": "es", "turns": [ ... ] } +``` + +```python +from voiceeval.checks import analyse +from voiceeval.locale import LocalePack, load_builtin_packs +from voiceeval.turns import load + +inter = load("fixtures/es_misheard_call.json") +packs = load_builtin_packs() +# or: packs["es"] = LocalePack.from_path("my_es.json") +findings = analyse(inter, packs=packs) +``` + +Edit `confusable_pairs` and `confirmation_phrases` in a pack JSON to cover domain +STT traps (product names, amounts) without changing library code. + ## Honest scope - **The eval logic is the project, and it is fully tested** (18 tests, no keys, no network). diff --git a/fixtures/es_misheard_call.json b/fixtures/es_misheard_call.json new file mode 100644 index 0000000..ada9c2f --- /dev/null +++ b/fixtures/es_misheard_call.json @@ -0,0 +1,38 @@ +{ + "id": "refund-misheard-cincuenta-es", + "language": "es", + "policy": { + "max_refund": 50 + }, + "completed": true, + "turns": [ + { + "speaker": "agent", + "text": "Hola, en que puedo ayudarle?", + "start_s": 0.0, + "end_s": 1.5 + }, + { + "speaker": "user", + "text": "Necesito un reembolso de cincuenta dolares.", + "truth": "Necesito un reembolso de quince dolares.", + "start_s": 2.0, + "end_s": 5.0 + }, + { + "speaker": "agent", + "text": "De acuerdo, reembolso de cincuenta ahora.", + "start_s": 5.4, + "end_s": 7.2, + "actions": [ + { + "name": "refund", + "args": { + "amount": 50 + }, + "consequential": true + } + ] + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index f96ae58..0a9d235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ dev = ["pytest>=8.3", "ruff>=0.7"] [tool.setuptools] packages = ["voiceeval"] +[tool.setuptools.package-data] +voiceeval = ["locales/*.json"] + [project.scripts] voiceeval = "voiceeval.cli:main" diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index c148994..f031dd3 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -216,3 +216,75 @@ def test_from_dict_roundtrip(): ) assert inter.turns[0].speaker == "user" assert inter.policy == {} + + +# --------------------------------------------------------------------------- locale packs + + +def test_english_default_pack_matches_builtin_fixtures(): + """Default English pack keeps fixture findings identical.""" + inter = load(FIXTURES / "misheard_call.json") + assert inter.language == "en" + findings = analyse(inter) + assert any(f.check == "misheard_number" for f in findings) + assert "no_confirmation" in _codes(inter) + + +def test_missing_locale_pack_is_not_a_silent_pass(): + """A Hindi (or other unsupported) call must not report zero findings as success.""" + inter = Interaction( + id="hi-call", + language="hi", + turns=[ + _t("user", "pachas refund", 0, 2, truth="pandrah refund"), + _t("agent", "Done.", 2.2, 3.0, actions=[Action("refund", {"amount": 50}, True)]), + ], + policy={"max_refund": 50}, + ) + findings = analyse(inter) + assert any(f.check == "locale_unavailable" for f in findings) + assert score(inter).passed is False + + +def test_suite_can_supply_custom_pack(): + from voiceeval.locale import LocalePack + + pack = LocalePack.from_dict( + { + "language": "xx", + "confusable_pairs": [["alpha", "beta"]], + "confirmation_phrases": ["please confirm"], + } + ) + inter = Interaction( + id="custom", + language="xx", + turns=[ + _t("user", "pay beta", 0, 2, truth="pay alpha"), + _t("agent", "Paying now.", 2.2, 3.0, actions=[Action("charge", {}, True)]), + ], + ) + findings = analyse(inter, locale_pack=pack) + assert any(f.check == "misheard_number" for f in findings) + assert "no_confirmation" in {f.check for f in findings} + # confirmation phrase works when present + inter2 = Interaction( + id="custom2", + language="xx", + turns=[ + _t("user", "pay beta", 0, 2, truth="pay beta"), + _t("agent", "please confirm beta", 2.2, 3.0), + _t("user", "yes", 3.1, 3.5, truth="yes"), + _t("agent", "ok", 3.6, 4.0, actions=[Action("charge", {}, True)]), + ], + ) + assert "no_confirmation" not in {f.check for f in analyse(inter2, locale_pack=pack)} + + +def test_spanish_pack_catches_misheard(): + inter = load(FIXTURES / "es_misheard_call.json") + assert inter.language == "es" + findings = analyse(inter) + assert any(f.check == "misheard_number" for f in findings), findings + assert "no_confirmation" in _codes(inter) + assert score(inter).passed is False diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f94a263..404b65f 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -14,15 +14,30 @@ eventually arrives is perfect. Nobody notices this in a transcript. 4. talking over the caller. Barge-in that the agent ignores makes it feel broken. 5. dead air. Silence with nobody speaking is where callers hang up. + +Language-sensitive checks (misheard, confirmation) read a :class:`LocalePack`. English ships by +default; suites can pass their own. A call whose language has no pack loaded reports that those +checks did not run — it is not counted as a clean pass. """ from __future__ import annotations import re from dataclasses import dataclass +from typing import Callable +from .locale import LocalePack, load_builtin_packs, resolve_pack from .turns import Interaction, Turn +# English number tokens used by misheard_number. Domain-specific number words belong in a +# custom locale pack's confusable_pairs; this keeps the classic English STT trap working. +_NUMERIC = re.compile( + r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" + r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" + r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", + re.I, +) + @dataclass class Finding: @@ -32,35 +47,25 @@ class Finding: turn_index: int | None = None -# Numbers that STT reliably confuses. The -teen/-ty pairs are the classic: one unstressed syllable -# apart, and both are plausible amounts, so neither the model nor a human reviewer notices. -_CONFUSABLE = [ - (r"\bfifteen\b", r"\bfifty\b"), - (r"\bsixteen\b", r"\bsixty\b"), - (r"\bseventeen\b", r"\bseventy\b"), - (r"\beighteen\b", r"\beighty\b"), - (r"\bnineteen\b", r"\bninety\b"), - (r"\bthirteen\b", r"\bthirty\b"), - (r"\bfourteen\b", r"\bforty\b"), -] - -_NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" - r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" - r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", re.I) - -_CONFIRM = re.compile( - r"\b(?:just to confirm|confirm|did you say|is that right|correct\?|to be clear|" - r"you'?d like|shall I|should I|can I go ahead)\b", - re.I, -) - - -def check_misheard(inter: Interaction) -> list[Finding]: +def check_misheard(inter: Interaction, pack: LocalePack | None = None) -> list[Finding]: """STT heard something different from what was said. Only detectable when you have ground truth, which is exactly why scripted test calls are worth the effort: in production this failure is silent by construction. + + Requires a locale pack for ``inter.language``. Without one the check reports that it did not + run (high severity) instead of returning an empty list that looks like a pass. """ + if pack is None: + return [ + Finding( + "locale_unavailable", + "high", + f"No locale pack for language {inter.language!r}; misheard check did not run. " + "Load a pack or set language to a supported locale (e.g. 'en').", + ) + ] + out: list[Finding] = [] for i, t in enumerate(inter.turns): if t.speaker != "user" or not t.truth: @@ -68,6 +73,7 @@ def check_misheard(inter: Interaction) -> list[Finding]: if _norm(t.text) == _norm(t.truth): continue + # English digit/word numbers first so default-pack behaviour stays byte-identical. heard_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.text)) said_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.truth)) if heard_nums != said_nums: @@ -80,25 +86,51 @@ def check_misheard(inter: Interaction) -> list[Finding]: i, ) ) - else: + continue + + # Pack confusable pairs catch non-English (and domain) traps the English numeric list misses. + if _confusable_mismatch(t.text, t.truth, pack): out.append( - Finding("misheard", "medium", f"STT differs from truth: {t.text!r} vs {t.truth!r}", i) + Finding( + "misheard_number", + "high", + f"STT confusable mismatch: heard {t.text!r} but caller said {t.truth!r}. " + "A wrong number the agent acts on is the most expensive failure in voice.", + i, + ) ) + continue + + out.append( + Finding("misheard", "medium", f"STT differs from truth: {t.text!r} vs {t.truth!r}", i) + ) return out -def check_acted_without_confirming(inter: Interaction) -> list[Finding]: +def check_acted_without_confirming( + inter: Interaction, pack: LocalePack | None = None +) -> list[Finding]: """A consequential action with no confirmation anywhere before it. In text, a misunderstanding costs one turn. In voice, it costs the refund. """ + if pack is None: + return [ + Finding( + "locale_unavailable", + "high", + f"No locale pack for language {inter.language!r}; confirmation check did not run. " + "Load a pack or set language to a supported locale (e.g. 'en').", + ) + ] + out: list[Finding] = [] for i, t in enumerate(inter.turns): consequential = [a for a in t.actions if a.consequential] if not consequential: continue confirmed = any( - _CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent" + pack.confirm_re.search(p.text) for p in inter.turns[:i] if p.speaker == "agent" ) if not confirmed: names = ", ".join(a.name for a in consequential) @@ -202,6 +234,20 @@ def check_incomplete(inter: Interaction) -> list[Finding]: return [] +# Checks that need a locale pack receive it as a second argument from analyse(). +_LOCALE_CHECKS: list[Callable[..., list[Finding]]] = [ + check_misheard, + check_acted_without_confirming, +] +_LANGUAGE_AGNOSTIC: list[Callable[[Interaction], list[Finding]]] = [ + check_policy_violation, + check_latency, + check_talked_over_user, + check_dead_air, + check_incomplete, +] + +# Public list kept for callers that introspect the module (order ≈ cost). CHECKS = [ check_misheard, check_acted_without_confirming, @@ -213,14 +259,41 @@ def check_incomplete(inter: Interaction) -> list[Finding]: ] -def analyse(inter: Interaction) -> list[Finding]: +def analyse( + inter: Interaction, + *, + packs: dict[str, LocalePack] | None = None, + locale_pack: LocalePack | None = None, +) -> list[Finding]: + """Run all checks. + + ``locale_pack`` forces a single pack. Otherwise the pack is resolved from + ``inter.language`` against ``packs`` (default: shipped builtins). + """ + available = packs if packs is not None else load_builtin_packs() + pack = resolve_pack(inter.language, packs=available, pack=locale_pack) + out: list[Finding] = [] - for check in CHECKS: + for check in _LOCALE_CHECKS: + out.extend(check(inter, pack)) + for check in _LANGUAGE_AGNOSTIC: out.extend(check(inter)) order = {"high": 0, "medium": 1, "low": 2} out.sort(key=lambda f: (order.get(f.severity, 9), f.turn_index if f.turn_index is not None else -1)) return out +def _confusable_mismatch(heard: str, truth: str, pack: LocalePack) -> bool: + """True when a confusable pair is split across heard vs truth.""" + for a_re, b_re in pack.confusable_res: + a_in_heard, b_in_heard = bool(a_re.search(heard)), bool(b_re.search(heard)) + a_in_truth, b_in_truth = bool(a_re.search(truth)), bool(b_re.search(truth)) + if (a_in_heard and b_in_truth and not b_in_heard) or ( + b_in_heard and a_in_truth and not a_in_heard + ): + return True + return False + + def _norm(s: str) -> str: return re.sub(r"[^a-z0-9 ]", "", s.lower()).strip() diff --git a/voiceeval/locale.py b/voiceeval/locale.py new file mode 100644 index 0000000..534a378 --- /dev/null +++ b/voiceeval/locale.py @@ -0,0 +1,85 @@ +"""Locale packs for language-sensitive checks. + +Mis-hearing and confirmation vocabularies are not universal. Hardcoding English +means a Spanish call silently reports zero findings — the same failure shape +this project exists to catch. Packs keep English editable and let suites ship +their own domain terms. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path + +_LOCALES_DIR = Path(__file__).resolve().parent / "locales" + + +@dataclass(frozen=True) +class LocalePack: + """Confusable terms and confirmation phrases for one language.""" + + language: str + confusable_pairs: tuple[tuple[str, str], ...] = () + confirmation_phrases: tuple[str, ...] = () + # Compiled helpers (not serialised) + confirm_re: re.Pattern[str] = field( + repr=False, hash=False, compare=False, default=re.compile(r"(?!x)x") + ) + confusable_res: tuple[tuple[re.Pattern[str], re.Pattern[str]], ...] = field( + default=(), repr=False, hash=False, compare=False + ) + + @staticmethod + def from_dict(data: dict) -> LocalePack: + lang = str(data["language"]).lower() + pairs = tuple((str(a), str(b)) for a, b in data.get("confusable_pairs", [])) + phrases = tuple(str(p) for p in data.get("confirmation_phrases", [])) + if phrases: + confirm_re = re.compile(r"\b(?:" + "|".join(phrases) + r")\b", re.I) + else: + confirm_re = re.compile(r"(?!x)x") # matches nothing + confusable_res = tuple( + ( + re.compile(rf"\b{re.escape(a)}\b", re.I), + re.compile(rf"\b{re.escape(b)}\b", re.I), + ) + for a, b in pairs + ) + return LocalePack( + language=lang, + confusable_pairs=pairs, + confirmation_phrases=phrases, + confirm_re=confirm_re, + confusable_res=confusable_res, + ) + + @classmethod + def from_path(cls, path: str | Path) -> LocalePack: + return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8"))) + + +def load_builtin_packs(directory: Path | None = None) -> dict[str, LocalePack]: + """Load every ``*.json`` pack from the shipped locales directory.""" + root = directory or _LOCALES_DIR + packs: dict[str, LocalePack] = {} + if not root.is_dir(): + return packs + for path in sorted(root.glob("*.json")): + pack = LocalePack.from_path(path) + packs[pack.language] = pack + return packs + + +def resolve_pack( + language: str, + *, + packs: dict[str, LocalePack] | None = None, + pack: LocalePack | None = None, +) -> LocalePack | None: + """Return the pack to use, or None if the language has no pack loaded.""" + if pack is not None: + return pack + available = packs if packs is not None else load_builtin_packs() + return available.get((language or "en").lower()) diff --git a/voiceeval/locales/en.json b/voiceeval/locales/en.json new file mode 100644 index 0000000..1894567 --- /dev/null +++ b/voiceeval/locales/en.json @@ -0,0 +1,45 @@ +{ + "language": "en", + "confusable_pairs": [ + [ + "fifteen", + "fifty" + ], + [ + "sixteen", + "sixty" + ], + [ + "seventeen", + "seventy" + ], + [ + "eighteen", + "eighty" + ], + [ + "nineteen", + "ninety" + ], + [ + "thirteen", + "thirty" + ], + [ + "fourteen", + "forty" + ] + ], + "confirmation_phrases": [ + "just to confirm", + "confirm", + "did you say", + "is that right", + "correct\\?", + "to be clear", + "you'?d like", + "shall I", + "should I", + "can I go ahead" + ] +} diff --git a/voiceeval/locales/es.json b/voiceeval/locales/es.json new file mode 100644 index 0000000..09194db --- /dev/null +++ b/voiceeval/locales/es.json @@ -0,0 +1,31 @@ +{ + "language": "es", + "confusable_pairs": [ + [ + "quince", + "cincuenta" + ], + [ + "dieciseis", + "sesenta" + ], + [ + "trece", + "treinta" + ], + [ + "catorce", + "cuarenta" + ] + ], + "confirmation_phrases": [ + "solo para confirmar", + "confirmar", + "dijo", + "es correcto", + "correcto\\?", + "para estar seguro", + "desea que", + "puedo proceder" + ] +} diff --git a/voiceeval/turns.py b/voiceeval/turns.py index 7885b67..fc008de 100644 --- a/voiceeval/turns.py +++ b/voiceeval/turns.py @@ -61,6 +61,9 @@ class Interaction: # hardcoding thresholds, because "what is allowed" is a business decision, not a library one. policy: dict = field(default_factory=dict) completed: bool = True # did the call reach its goal, or did it just end + # BCP-47-ish language tag for locale-sensitive checks (misheard / confirmation). + # Defaults to English so existing fixtures keep their behaviour. + language: str = "en" def pairs(self) -> list[tuple[Turn, Turn]]: """(user turn, the agent turn that answered it). Where response latency lives.""" @@ -111,4 +114,5 @@ def from_dict(data: dict) -> Interaction: turns=turns, policy=data.get("policy", {}), completed=bool(data.get("completed", True)), + language=str(data.get("language", "en")).lower(), )