From 60595acc61ef99510bf500446b412bcea679e2a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 28 Feb 2026 05:13:58 +0000 Subject: [PATCH] =?UTF-8?q?refactor:=20tidy=20codebase=20=E2=80=94=20fix?= =?UTF-8?q?=20bugs,=20remove=20duplication?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix broken regex in detectors/base.py: r"\\D+" → r"\D+" (digits_only was a no-op) - Remove duplicate dataclass/typing imports and duplicate Detector Protocol definition in base.py - Remove duplicate `phonenumbers` and `stdnum` imports in regexes.py - Remove duplicate `shannon_entropy` function in entropy.py (use utils.py import) - Remove unused `Detector` import and `math` import in entropy.py - Fix syntax error in decors.py: '. . .' → '...' - Replace wildcard import in hide.py with explicit named import - Fix transforms/tokenise.py which was an identical copy of mask.py; implement SHA-256 tokenization - Remove duplicate Python SDK section in README.md - Replace placeholder description in pyproject.toml https://claude.ai/code/session_01SmfiwftPFKGgQr1W4uYYjs --- README.md | 12 ------ pyproject.toml | 2 +- src/redactable/decors.py | 2 +- src/redactable/detectors/base.py | 61 +++++++++++++++------------ src/redactable/detectors/entropy.py | 26 +++--------- src/redactable/detectors/regexes.py | 18 ++------ src/redactable/hide.py | 8 +--- src/redactable/transforms/tokenise.py | 15 ++++--- 8 files changed, 57 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 418727e..8f1795f 100644 --- a/README.md +++ b/README.md @@ -57,18 +57,6 @@ redactable --policy gdpr.yaml input.log output.redacted.log ```` -### Python SDK - -````bash -from redactable import apply - -data = "Customer email: test@example.com" -result = apply(data, policy="gdpr.yaml") -print(result) -# → "Customer email: ****@example.com" -```` - - ### Python SDK ````bash diff --git a/pyproject.toml b/pyproject.toml index 0bcdcf1..75ab6cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "redactable" -description = "One-line value prop." +description = "Policy-driven framework for data redaction, masking, and privacy-preserving transformations." readme = "README.md" requires-python = ">=3.13" license = { text = "MIT" } diff --git a/src/redactable/decors.py b/src/redactable/decors.py index 62cdff9..3b93567 100644 --- a/src/redactable/decors.py +++ b/src/redactable/decors.py @@ -1 +1 @@ -def redactable_io(policy): . . . +def redactable_io(policy): ... diff --git a/src/redactable/detectors/base.py b/src/redactable/detectors/base.py index 146845c..b94099c 100644 --- a/src/redactable/detectors/base.py +++ b/src/redactable/detectors/base.py @@ -3,23 +3,21 @@ Contents: - Finding: dataclass representing a detected entity. -- Detector: Protocol interface that all detectors must implement. +- Match: dataclass for the module-level detection system. +- Detector: Protocol interface that Finding-based detectors must implement. - Shared helper functions: digits_only, luhn_ok, guess_card_brand. """ -from dataclasses import dataclass -from typing import Iterable, Optional, Protocol, Tuple, Dict, Any +from __future__ import annotations + import re +from dataclasses import dataclass +from typing import Any, Iterable, Optional, Protocol # -------------------------------------------------------------------- -# Shared type aliases -Span = Tuple[int, int] -Extras = Dict[str, Any] +# Match-based detection types (used by individual detector modules) -from dataclasses import dataclass -from typing import Iterable, Protocol, TypedDict, Optional, Dict, Any, List - @dataclass(slots=True) class Match: label: str # e.g. "EMAIL", "CREDIT_CARD" @@ -27,31 +25,36 @@ class Match: end: int value: str # matched text (pre-transform) confidence: float = 1.0 # 0..1 - meta: dict[str, Any] = None + meta: dict[str, Any] | None = None -class Detector(Protocol): - name: str - labels: tuple[str, ...] - def detect(self, text: str, *, context: Optional[dict[str, Any]] = None) -> Iterable[Match]: ... # Simple registry -_REGISTRY: Dict[str, Detector] = {} -_LABEL_TO_DETECTORS: Dict[str, List[str]] = {} +_REGISTRY: dict[str, Any] = {} +_LABEL_TO_DETECTORS: dict[str, list[str]] = {} + -def register(detector: Detector) -> None: +def register(detector: Any) -> None: _REGISTRY[detector.name] = detector for label in detector.labels: _LABEL_TO_DETECTORS.setdefault(label, []).append(detector.name) -def get(name: str) -> Detector: + +def get(name: str) -> Any: return _REGISTRY[name] -def detectors_for(label: str) -> list[Detector]: - return [ _REGISTRY[n] for n in _LABEL_TO_DETECTORS.get(label, []) ] -def all_detectors() -> list[Detector]: +def detectors_for(label: str) -> list[Any]: + return [_REGISTRY[n] for n in _LABEL_TO_DETECTORS.get(label, [])] + + +def all_detectors() -> list[Any]: return list(_REGISTRY.values()) + +# -------------------------------------------------------------------- +# Finding-based detection types (used by DetectorRegistry) + + @dataclass(slots=True) class Finding: """ @@ -65,12 +68,13 @@ class Finding: normalized: Canonicalized form (e.g. digits-only phone number). extras: Additional metadata (brand, region, reasons, etc.). """ + kind: str value: str - span: Span + span: tuple[int, int] confidence: float normalized: Optional[str] = None - extras: Extras | None = None + extras: dict[str, Any] | None = None def __post_init__(self) -> None: if not (0.0 <= self.confidence <= 1.0): @@ -84,22 +88,26 @@ def __str__(self) -> str: class Detector(Protocol): """ - Protocol that all detectors must follow. + Protocol that Finding-based detectors must follow. Each detector must expose a `name` and a `detect` method. """ + name: str def detect(self, text: str) -> Iterable[Finding]: ... + # -------------------------------------------------------------------- # Shared helpers -_DIGITS = re.compile(r"\\D+") +_DIGITS = re.compile(r"\D+") + def digits_only(s: str) -> str: """Strip all non-digit characters from a string.""" return _DIGITS.sub("", s) + def luhn_ok(num: str) -> bool: """ Check if a string of digits passes the Luhn algorithm. @@ -120,6 +128,7 @@ def luhn_ok(num: str) -> bool: alt = not alt return total % 10 == 0 + def guess_card_brand(pan: str) -> str | None: """ Make a naive guess of card brand from PAN digits. @@ -154,5 +163,3 @@ def guess_card_brand(pan: str) -> str | None: return "unionpay" return None - - diff --git a/src/redactable/detectors/entropy.py b/src/redactable/detectors/entropy.py index 781fda4..312d521 100644 --- a/src/redactable/detectors/entropy.py +++ b/src/redactable/detectors/entropy.py @@ -12,25 +12,10 @@ """ import re -from .base import Match, register, Finding, Detector -from .utils import shannon_entropy, looks_like_secret from typing import Iterable -import math - -# -------------------------------------------------------------------- -# Helpers - -def shannon_entropy(s: str) -> float: - """ - Calculate Shannon entropy of a string. - Returns a value >= 0, higher means more random. - """ - if not s: - return 0.0 - freq = {ch: s.count(ch) for ch in set(s)} - n = len(s) - return -sum((c/n) * math.log2(c/n) for c in freq.values()) +from .base import Finding, Match, register +from .utils import looks_like_secret, shannon_entropy # -------------------------------------------------------------------- # Regex pattern: matches candidate secrets @@ -56,6 +41,7 @@ class HighEntropyTokenDetector: entropy_threshold: minimum Shannon entropy (default ~3.5). min_len: minimum string length to consider. """ + name = "high_entropy_token" def __init__(self, entropy_threshold: float = 3.5, min_len: int = 24) -> None: @@ -86,6 +72,7 @@ def detect(self, text: str) -> Iterable[Finding]: # Tokens separated by non-word; allow -,_,= typical in JWT/base64url _TOKEN = re.compile(r'([A-Za-z0-9_\-=+/]{20,})') + class EntropyDetector: name = "entropy" labels = ("SECRET",) @@ -101,6 +88,7 @@ def detect(self, text: str, *, context=None): continue H = shannon_entropy(token) if H >= threshold: - yield Match("SECRET", m.start(1), m.end(1), token, min(0.99, 0.7 + (H-threshold)/4), {"entropy": H}) + yield Match("SECRET", m.start(1), m.end(1), token, min(0.99, 0.7 + (H - threshold) / 4), {"entropy": H}) + -register(EntropyDetector()) \ No newline at end of file +register(EntropyDetector()) diff --git a/src/redactable/detectors/regexes.py b/src/redactable/detectors/regexes.py index a996393..6988b89 100644 --- a/src/redactable/detectors/regexes.py +++ b/src/redactable/detectors/regexes.py @@ -17,9 +17,9 @@ """ import re -from typing import Iterable, Dict, Any +from typing import Any, Iterable -from .base import Finding, Detector, digits_only, luhn_ok, guess_card_brand +from .base import Finding, digits_only, luhn_ok, guess_card_brand # -------------------------------------------------------------------- # Optional external dependencies (gracefully degrade if missing) @@ -86,11 +86,6 @@ def detect(self, text: str) -> Iterable[Finding]: extras={"luhn_valid": ok, "brand": brand}, ) -try: - import phonenumbers # type: ignore -except Exception: # pragma: no cover - phonenumbers = None - # -------------------------------------------------------------------- # Simple phone regex fallback RE_PHONE = re.compile( @@ -157,7 +152,7 @@ def detect(self, text: str) -> Iterable[Finding]: start, end = m.span() conf = 0.6 norm = raw - extras: Dict[str, Any] = {} + extras: dict[str, Any] = {} # If email-validator is available, upgrade confidence if validate_email is not None: try: @@ -176,13 +171,6 @@ def detect(self, text: str) -> Iterable[Finding]: extras=extras, ) -try: - from stdnum import iban as std_iban # type: ignore - from stdnum.gb import nhs as std_nhs # type: ignore - from stdnum.us import ssn as std_us_ssn # type: ignore -except Exception: # pragma: no cover - std_iban = std_nhs = std_us_ssn = None - # -------------------------------------------------------------------- # Regex patterns RE_NHS = re.compile(r"\b(\d{3})[\s-]?(\d{3})[\s-]?(\d{4})\b") diff --git a/src/redactable/hide.py b/src/redactable/hide.py index d8ff7a8..b88833f 100644 --- a/src/redactable/hide.py +++ b/src/redactable/hide.py @@ -1,7 +1,3 @@ -""" -Use this as a main entrypoint? -""" +from .decors import redactable_io -from .decors import * - -__all__ = [] \ No newline at end of file +__all__ = ["redactable_io"] diff --git a/src/redactable/transforms/tokenise.py b/src/redactable/transforms/tokenise.py index bb5c5bb..89772cd 100644 --- a/src/redactable/transforms/tokenise.py +++ b/src/redactable/transforms/tokenise.py @@ -1,14 +1,17 @@ +import hashlib from typing import Iterable + from redactable.detectors import Finding -def _mask(value: str, keep_head: int = 0, keep_tail: int = 4, glyph: str = "•") -> str: - if len(value) <= keep_head + keep_tail: - return glyph * len(value) - return value[:keep_head] + glyph * (len(value) - keep_head - keep_tail) + value[-keep_tail:] -def mask_in_place(text: str, findings: Iterable[Finding], keep_head: int = 0, keep_tail: int = 4, glyph: str = "•") -> str: +def _sha256(value: str, salt: str = "") -> str: + return hashlib.sha256((salt + value).encode("utf-8")).hexdigest() + + +def tokenise_in_place(text: str, findings: Iterable[Finding], salt: str = "") -> str: out = text for f in sorted(findings, key=lambda x: x.span[0], reverse=True): s, e = f.span - out = out[:s] + _mask(out[s:e], keep_head, keep_tail, glyph) + out[e:] + token = _sha256(f.normalized or f.value, salt) + out = out[:s] + token + out[e:] return out