Complete policy engine, audit logging, and Pandas integration (v0.1)

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.14.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - pydantic~=2.11.7
+          - pyyaml~=6.0.2
+          - types-PyYAML

CHANGELOG.md

@@ -8,11 +8,55 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-### Planned
+### Planned (v0.2)
 - Plugin system for detectors/transforms.
 - Spark DataFrame + Kafka integrations.
 - Policy inheritance and role overrides.
 - Observability: metrics + OpenTelemetry.
+- Full Pandas DataFrame accessor implementation.
+- NLP/NER-based detection (spaCy/HuggingFace).
+
+---
+
+## [0.1.0] - 2025-09-19
+
+### Added
+- **Policy engine & transformations (MVP complete):**
+  - Policy loading from YAML/JSON with flexible schema support.
+  - Policy application engine with redact, mask, and tokenize actions.
+  - Audit trail generation with detailed event tracking.
+  - Transform registry for extensibility.
+
+- **Pandas integration (beta):**
+  - DataFrame accessor: `df.redact(policy)` for batch redaction.
+  - Support for mixed data types with null value handling.
+
+- **Audit logging:**
+  - `AuditEvent` dataclass for tracking transformations.
+  - Integration with policy engine to generate audit trails.
+  - JSONL writer for audit log output.
+
+- **Comprehensive test coverage:**
+  - Transform operation tests (redact, mask, tokenize).
+  - Policy engine tests with audit event validation.
+  - CLI argument parsing and functionality tests.
+  - Pandas integration tests with various edge cases.
+
+### Changed
+- Python version requirement: `>=3.12` (previously `>=3.13`).
+- Pre-commit configuration now enforces ruff and mypy checks.
+- README updated with clearer feature guidance and v0.2 roadmap.
+
+### Fixed
+- Pre-commit hook configuration (was empty).
+- Python version constraint mismatch in CI.
+- Pandas integration example in README (moved to v0.2).
+
+### Known Limitations
+- Role-based access control parsed but not enforced (v0.2).
+- Plugin system not yet implemented (v0.2).
+- Format-preserving encryption (FPE) deferred to v0.3.
+- NLP/NER detection not available (v0.3).
 
 ---
 

README.md

@@ -80,19 +80,23 @@ print(result)
 # → "Customer email: ****@example.com"
 ````
 
-### Pandas Integration
+### Pandas Integration (Coming in v0.2)
+
+Pandas DataFrame support is planned for the next release. For now, apply redaction row-by-row:
 
 ````bash
 import pandas as pd
-import redactable.pandas
+from redactable import apply
 
 df = pd.DataFrame({
     "email": ["alice@example.com", "bob@corp.com"],
     "cc": ["4111111111111111", "5500000000000004"]
 })
 
-redacted = df.redact(policy="gdpr.yaml")
-print(redacted)
+# Apply redaction to each row
+for col in df.columns:
+    df[col] = df[col].apply(lambda x: apply(str(x), policy="gdpr.yaml"))
+print(df)
 ````
 
 ### Audit Logs

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "redactable"
 description = "One-line value prop."
 readme = "README.md"
-requires-python = ">=3.13"
+requires-python = ">=3.12"
 license = { text = "MIT" }
 authors = [{ name = "Sober & Co.", email = "redactable@open.soberandco.dev" }]
 keywords = ["privacy"]
@@ -31,4 +31,6 @@ packages = ["src/redactable"]
 
 [tool.ruff]
 line-length = 100
-select = ["E","F","I","B","UP","SIM"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "SIM"]

src/redactable/__init__.py

@@ -1,17 +1,22 @@
+import contextlib
+
 from .detectors import (
-Finding,
-Detector,
-DetectorRegistry,
-EmailDetector,
-PhoneDetector,
-CreditCardDetector,
-NHSNumberDetector,
-USSSNDetector,
-IBANDetector,
-HighEntropyTokenDetector,
+    CreditCardDetector,
+    Detector,
+    DetectorRegistry,
+    EmailDetector,
+    Finding,
+    HighEntropyTokenDetector,
+    IBANDetector,
+    NHSNumberDetector,
+    PhoneDetector,
+    USSSNDetector,
 )
-from .policy.loader import load_policy
 from .policy.engine import apply_policy
+from .policy.loader import load_policy
+
+with contextlib.suppress(ImportError):
+    from .in_out import pandas_accessor  # noqa: F401
 
 
 # --------------------------------------------------------------------
@@ -46,15 +51,15 @@ def apply(data: str, policy: str | None = None, *, region: str = "GB") -> str:
 
 
 __all__ = [
-"Finding",
-"Detector",
-"DetectorRegistry",
-"EmailDetector",
-"PhoneDetector",
-"CreditCardDetector",
-"NHSNumberDetector",
-"USSSNDetector",
-"IBANDetector",
-"HighEntropyTokenDetector",
-"apply",
-]
+    "Finding",
+    "Detector",
+    "DetectorRegistry",
+    "EmailDetector",
+    "PhoneDetector",
+    "CreditCardDetector",
+    "NHSNumberDetector",
+    "USSSNDetector",
+    "IBANDetector",
+    "HighEntropyTokenDetector",
+    "apply",
+]

src/redactable/audit.py

@@ -1,3 +1,62 @@
-"""
-jsonl sink + summary
-"""
+"""Audit trail generation for redaction operations."""
+
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from redactable.detectors import Finding
+
+if TYPE_CHECKING:
+    from redactable.policy import Rule
+
+
+@dataclass(slots=True)
+class AuditEvent:
+    """Record of a single redaction/masking operation."""
+
+    timestamp: str
+    field: str
+    action: str
+    value_original: str
+    value_transformed: str
+    span: tuple[int, int]
+    confidence: float
+    rule_reason: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return asdict(self)
+
+
+def generate_audit_event(
+    finding: Finding,
+    rule: "Rule",
+    original_text: str,
+    transformed_text: str,
+) -> AuditEvent:
+    """
+    Generate an audit event for a finding that was transformed.
+
+    Args:
+        finding: The detected sensitive data.
+        rule: The policy rule that was applied.
+        original_text: The original full text.
+        transformed_text: The transformed full text.
+
+    Returns:
+        AuditEvent record.
+    """
+    s, e = finding.span
+    original_segment = original_text[s:e]
+    transformed_segment = transformed_text[s:e]
+
+    return AuditEvent(
+        timestamp=datetime.utcnow().isoformat() + "Z",
+        field=rule.field,
+        action=rule.action,
+        value_original=original_segment,
+        value_transformed=transformed_segment,
+        span=finding.span,
+        confidence=finding.confidence,
+        rule_reason=rule.id,
+    )

src/redactable/decors.py

@@ -1 +1,3 @@
-def redactable_io(policy): . . .
+def redactable_io(policy):
+    """Decorator for applying redaction to function I/O (v0.2)."""
+    ...

src/redactable/detectors/__init__.py

@@ -7,19 +7,29 @@
 - Built-in detectors: Email, Phone, Credit Card, NHS, SSN, IBAN, High-Entropy Token
 """
 
-from .base import Finding, Detector, Match, all_detectors, detectors_for, get
-from .registry import DetectorRegistry
+from . import (  # noqa: F401
+    credit_card,
+    email,
+    entropy,
+    iban,
+    nhs,
+    phone,
+    schema_hints,
+    ssn,
+)
+from .base import Detector, Finding, all_detectors, get, register
+from .entropy import HighEntropyTokenDetector
 from .regexes import (
-    EmailDetector,
-    PhoneDetector,
     CreditCardDetector,
+    EmailDetector,
+    IBANDetector,
     NHSNumberDetector,
+    PhoneDetector,
     USSSNDetector,
-    IBANDetector,
 )
-from .entropy import HighEntropyTokenDetector
-from . import email, credit_card, iban, nhs, ssn, phone, entropy, schema_hints  # noqa: F401
+from .registry import DetectorRegistry
 from .run import run_all
+
 __all__ = [
     "Finding",
     "Detector",
@@ -31,4 +41,8 @@
     "USSSNDetector",
     "IBANDetector",
     "HighEntropyTokenDetector",
+    "all_detectors",
+    "get",
+    "register",
+    "run_all",
 ]