From 5870d300ce96c7c1bb8ae5a90d9032ebaf61e4d6 Mon Sep 17 00:00:00 2001 From: Ashay Date: Sat, 8 Aug 2026 18:23:31 +0530 Subject: [PATCH] Refuse exact token claims for Claude sessions under tiktoken cl100k tiktoken defaults to OpenAI's cl100k_base encoding. That is exact for OpenAI/Codex transcripts and only approximate for Claude Code. Surface token_accuracy and accuracy_note in analysis/JSON, label the terminal tokenizer when approximate, warn on the CLI, and document per-format accuracy in the README. Tokenizer name always includes the encoding id. Fixes #3 --- README.md | 21 +++++-- src/ctxlens/cli.py | 16 ++++-- src/ctxlens/engine.py | 10 +++- src/ctxlens/reporters/json_report.py | 2 + src/ctxlens/reporters/terminal.py | 14 ++++- src/ctxlens/tokenizers/__init__.py | 14 ++++- src/ctxlens/tokenizers/registry.py | 60 +++++++++++++++++++- src/ctxlens/tokenizers/tiktoken_tokenizer.py | 12 +++- tests/test_cli.py | 25 ++++++++ tests/test_reporters.py | 8 +++ tests/test_tokenizers.py | 35 ++++++++++++ 11 files changed, 198 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 3b7eedd..1a387db 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ turn, tool schemas re-sent on every step. Token dashboards tell you the bill. **ctxlens tells you where the bytes went and what to delete.** It works offline with a deterministic heuristic tokenizer (no network, no heavy -deps), and upgrades to exact counts automatically when `tiktoken` is installed. +deps), and upgrades to OpenAI-exact counts via `tiktoken` when installed. Counts +for Claude Code sessions remain **approximate** under tiktoken — `cl100k_base` is +OpenAI's encoding, not Anthropic's. ## Quickstart @@ -24,7 +26,7 @@ deps), and upgrades to exact counts automatically when `tiktoken` is installed. ```bash pip install ctxlens-cli # core -pip install "ctxlens-cli[tiktoken]" # optional exact token counts +pip install "ctxlens-cli[tiktoken]" # optional OpenAI-exact token counts ctxlens analyze session.jsonl ctxlens report session.jsonl --html -o report.html @@ -110,9 +112,20 @@ output with `--json`, or diff a baseline against a candidate in CI with ## Tokenizers +Counts are only as accurate as the encoding vs the model that produced the +session. The report labels each run `exact` or `approximate` (`token_accuracy` +in JSON) and never claims exactness for Claude sessions under OpenAI encodings. + +| Tokenizer | Name shown | OpenAI chat / Codex | Claude Code | +| --- | --- | --- | --- | +| `heuristic` | `heuristic` | approximate | approximate | +| `tiktoken` (default `cl100k_base`) | `tiktoken:cl100k_base` | **exact** (OpenAI BPE) | **approximate** (wrong encoding) | + - `heuristic` (default fallback): deterministic, dependency-free, great for - relative profiling and CI. -- `tiktoken`: exact BPE counts when installed. `--tokenizer auto` prefers it. + relative profiling and CI. Always an estimate. +- `tiktoken`: OpenAI `cl100k_base` BPE when installed. Exact for OpenAI/Codex + transcripts; approximate for Claude Code. `--tokenizer auto` prefers it, and + the CLI prints a warning when the count is approximate for the session format. ## Contributing diff --git a/src/ctxlens/cli.py b/src/ctxlens/cli.py index 8b1dcc7..1f3d8d7 100644 --- a/src/ctxlens/cli.py +++ b/src/ctxlens/cli.py @@ -144,15 +144,21 @@ def _load(path, fmt, tokenizer, top, tool_result_cap, tool_def_budget): try: if str(path) == "-": raw = sys.stdin.read() - return analyze_text(raw, source="", **common) - p = Path(path) - if not p.is_file(): - raise ParseError(f"no such file: {path}") - return analyze_file(p, **common) + analysis = analyze_text(raw, source="", **common) + else: + p = Path(path) + if not p.is_file(): + raise ParseError(f"no such file: {path}") + analysis = analyze_file(p, **common) except (ParseError, ImportError, ValueError, OSError) as exc: err_console.print(f"[red]error:[/red] {exc}") raise typer.Exit(EXIT_ERROR) from exc + # Refuse to imply exactness for formats without a matching tokenizer. + if analysis.token_accuracy == "approximate" and analysis.accuracy_note: + err_console.print(f"[yellow]warning:[/yellow] {analysis.accuracy_note}") + return analysis + def _maybe_fail(ratio: float, threshold: float | None): if threshold is not None and ratio > threshold: diff --git a/src/ctxlens/engine.py b/src/ctxlens/engine.py index 7164fcf..13add86 100644 --- a/src/ctxlens/engine.py +++ b/src/ctxlens/engine.py @@ -14,7 +14,7 @@ from ctxlens.analysis.waste import WasteReport, build_waste_report from ctxlens.models import Session from ctxlens.parsers import parse_file, parse_text -from ctxlens.tokenizers import get_tokenizer +from ctxlens.tokenizers import accuracy_note, count_accuracy, get_tokenizer @dataclass @@ -24,6 +24,10 @@ class Analysis: waste: WasteReport recommendations: list[Recommendation] source: str | None = None + #: "exact" only when the tokenizer encoding matches the session's model family + token_accuracy: str = "approximate" + #: human-readable caveat when counts are not exact; None when exact + accuracy_note: str | None = None @property def total_tokens(self) -> int: @@ -49,12 +53,16 @@ def analyze_session( session, tool_result_cap=tool_result_cap, tool_def_budget=tool_def_budget ) recs = recommend(profile, waste) + accuracy = count_accuracy(tok, session.source_format) + note = accuracy_note(tok, session.source_format) return Analysis( session=session, profile=profile, waste=waste, recommendations=recs, source=source, + token_accuracy=accuracy, + accuracy_note=note, ) diff --git a/src/ctxlens/reporters/json_report.py b/src/ctxlens/reporters/json_report.py index 0249d3d..e9347e5 100644 --- a/src/ctxlens/reporters/json_report.py +++ b/src/ctxlens/reporters/json_report.py @@ -15,6 +15,8 @@ def to_dict(analysis: Analysis) -> dict: "source": analysis.source, "format": analysis.session.source_format, "tokenizer": p.tokenizer_name, + "token_accuracy": analysis.token_accuracy, + "accuracy_note": analysis.accuracy_note, "total_tokens": total, "high_water_mark": p.high_water_mark, "turns": len(p.turn_stats), diff --git a/src/ctxlens/reporters/terminal.py b/src/ctxlens/reporters/terminal.py index 142bbac..551b204 100644 --- a/src/ctxlens/reporters/terminal.py +++ b/src/ctxlens/reporters/terminal.py @@ -31,16 +31,24 @@ def _header(a: Analysis) -> Panel: src = a.source or "" ratio = a.waste_ratio * 100 ratio_style = "red" if ratio >= 25 else "yellow" if ratio >= 10 else "green" - body = Text.assemble( + accuracy = getattr(a, "token_accuracy", "approximate") + tok_label = p.tokenizer_name + if accuracy == "approximate": + tok_label = f"{p.tokenizer_name} (approximate)" + parts: list = [ ("Source ", "dim"), f"{src}\n", ("Format ", "dim"), f"{a.session.source_format} ", - ("Tokenizer ", "dim"), f"{p.tokenizer_name}\n", + ("Tokenizer ", "dim"), f"{tok_label}\n", ("Tokens ", "dim"), f"{p.total_tokens:,} ", ("Turns ", "dim"), f"{len(p.turn_stats)} ", ("High-water ", "dim"), f"{p.high_water_mark:,}\n", ("Waste ", "dim"), (f"{a.waste.total_waste:,} tokens ({ratio:.1f}%)", ratio_style), - ) + ] + note = getattr(a, "accuracy_note", None) + if note: + parts.extend(["\n", ("Note ", "dim"), (note, "yellow")]) + body = Text.assemble(*parts) return Panel(body, title="ctxlens", border_style="cyan", expand=False) diff --git a/src/ctxlens/tokenizers/__init__.py b/src/ctxlens/tokenizers/__init__.py index 66d5e9d..a5f3943 100644 --- a/src/ctxlens/tokenizers/__init__.py +++ b/src/ctxlens/tokenizers/__init__.py @@ -3,16 +3,26 @@ The analysis layer never counts tokens directly; it goes through a :class:`Tokenizer`. This keeps ctxlens usable with no heavy dependencies (the :class:`HeuristicTokenizer` is deterministic and network-free) while still -allowing an exact count via tiktoken when it is installed. +allowing OpenAI-exact counts via tiktoken when it is installed. + +Tiktoken's default encoding is OpenAI ``cl100k_base``. It is **not** exact for +Claude Code sessions; see :func:`count_accuracy`. """ from ctxlens.tokenizers.base import Tokenizer from ctxlens.tokenizers.heuristic import HeuristicTokenizer -from ctxlens.tokenizers.registry import available_tokenizers, get_tokenizer +from ctxlens.tokenizers.registry import ( + accuracy_note, + available_tokenizers, + count_accuracy, + get_tokenizer, +) __all__ = [ "Tokenizer", "HeuristicTokenizer", "get_tokenizer", "available_tokenizers", + "count_accuracy", + "accuracy_note", ] diff --git a/src/ctxlens/tokenizers/registry.py b/src/ctxlens/tokenizers/registry.py index c733b11..a051bf8 100644 --- a/src/ctxlens/tokenizers/registry.py +++ b/src/ctxlens/tokenizers/registry.py @@ -1,15 +1,29 @@ """Tokenizer selection. ``get_tokenizer("auto")`` prefers tiktoken when available and silently falls -back to the heuristic tokenizer otherwise, so the default experience is exact -where possible and always works offline. +back to the heuristic tokenizer otherwise. + +**Accuracy is format-dependent.** ``tiktoken`` defaults to OpenAI's +``cl100k_base`` encoding. That is exact for OpenAI/Codex chat transcripts and +only approximate for Claude Code sessions (Anthropic models do not use +``cl100k_base``). The heuristic is always an estimate. Callers must not treat +every tiktoken count as exact — use :func:`count_accuracy` / the analysis +``token_accuracy`` field. """ from __future__ import annotations +from typing import Literal + from ctxlens.tokenizers.base import Tokenizer from ctxlens.tokenizers.heuristic import HeuristicTokenizer +# Source formats whose native tokenizers match OpenAI cl100k_base closely enough +# that a tiktoken:cl100k_base count is treated as exact. +_OPENAI_FORMATS = frozenset({"openai-chat", "codex-session"}) + +Accuracy = Literal["exact", "approximate"] + def _try_tiktoken(encoding: str = "cl100k_base") -> Tokenizer | None: try: @@ -32,7 +46,11 @@ def get_tokenizer(name: str = "auto") -> Tokenizer: ``auto`` -> tiktoken if installed, else heuristic ``heuristic`` -> always the dependency-free heuristic - ``tiktoken`` -> tiktoken, raising ImportError if unavailable + ``tiktoken`` -> tiktoken (OpenAI cl100k_base), raising ImportError if unavailable + + The returned tokenizer's ``name`` identifies the encoding (e.g. + ``tiktoken:cl100k_base``). Exactness for a given session still depends on + ``source_format`` — see :func:`count_accuracy`. """ name = (name or "auto").lower() if name == "heuristic": @@ -47,3 +65,39 @@ def get_tokenizer(name: str = "auto") -> Tokenizer: if name == "auto": return _try_tiktoken() or HeuristicTokenizer() raise ValueError(f"unknown tokenizer: {name!r}") + + +def count_accuracy(tokenizer: Tokenizer, source_format: str) -> Accuracy: + """Whether counts from ``tokenizer`` are exact for ``source_format``. + + * ``tiktoken:cl100k_base`` is exact for OpenAI chat and Codex sessions. + * Claude Code and any other format are approximate under cl100k (wrong BPE). + * The heuristic is always approximate. + """ + name = getattr(tokenizer, "name", "") or "" + if name.startswith("tiktoken:"): + # Only claim exactness when the encoding matches the model family. + # Default encoding is cl100k_base (OpenAI). No Anthropic encoding ships here. + encoding = name.split(":", 1)[1] + if encoding == "cl100k_base" and source_format in _OPENAI_FORMATS: + return "exact" + return "approximate" + return "approximate" + + +def accuracy_note(tokenizer: Tokenizer, source_format: str) -> str | None: + """Human-readable note when the count is approximate, else None.""" + if count_accuracy(tokenizer, source_format) == "exact": + return None + name = getattr(tokenizer, "name", "tokenizer") + if name.startswith("tiktoken:") and source_format.startswith("claude"): + return ( + f"{name} is OpenAI's cl100k_base encoding; counts for Claude Code " + f"sessions ({source_format}) are approximate, not exact" + ) + if name.startswith("tiktoken:"): + return ( + f"{name} counts are approximate for source format {source_format!r} " + "(no matching model encoding is available)" + ) + return f"{name} is an estimate, not an exact model tokenizer count" diff --git a/src/ctxlens/tokenizers/tiktoken_tokenizer.py b/src/ctxlens/tokenizers/tiktoken_tokenizer.py index b2e9f74..59c65b1 100644 --- a/src/ctxlens/tokenizers/tiktoken_tokenizer.py +++ b/src/ctxlens/tokenizers/tiktoken_tokenizer.py @@ -1,4 +1,10 @@ -"""Exact token counting backed by tiktoken (optional dependency).""" +"""Token counting backed by tiktoken (optional dependency). + +The default encoding is OpenAI's ``cl100k_base``. That is **exact for OpenAI / +Codex transcripts** and only an approximation for Claude Code sessions — +Anthropic models do not use this encoding. Prefer the analysis +``token_accuracy`` field over assuming every tiktoken count is exact. +""" from __future__ import annotations @@ -10,6 +16,9 @@ class TiktokenTokenizer(Tokenizer): Constructing this raises :class:`ImportError` if tiktoken is not installed, so callers can fall back to the heuristic tokenizer gracefully. + + ``name`` always includes the encoding id (e.g. ``tiktoken:cl100k_base``) so + reported counts are traceable to a specific vocabulary. """ def __init__(self, encoding: str = "cl100k_base") -> None: @@ -19,6 +28,7 @@ def __init__(self, encoding: str = "cl100k_base") -> None: raise ImportError( "tiktoken is not installed; install with `pip install ctxlens[tiktoken]`" ) from exc + self.encoding = encoding self._enc = tiktoken.get_encoding(encoding) self.name = f"tiktoken:{encoding}" diff --git a/tests/test_cli.py b/tests/test_cli.py index 7abbd17..39bc5b1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -78,3 +78,28 @@ def test_analyze_stdin(openai_array): data = json.loads(result.stdout) assert data["source"] == "" assert data["total_tokens"] > 0 + + +def test_analyze_json_marks_claude_tiktoken_approximate(claude_jsonl): + result = runner.invoke( + app, ["analyze", str(claude_jsonl), "--tokenizer", "tiktoken", "--json"] + ) + combined = (result.stdout or "") + (result.stderr or "") + if result.exit_code != 0 and "not installed" in combined.lower(): + return + assert result.exit_code == 0, combined + data = json.loads(result.stdout) + assert data["tokenizer"].startswith("tiktoken:") + assert data["token_accuracy"] == "approximate" + assert data["accuracy_note"] + assert "approximate" in data["accuracy_note"].lower() + + +def test_analyze_claude_tiktoken_warns_on_stderr(claude_jsonl): + result = runner.invoke( + app, ["analyze", str(claude_jsonl), "--tokenizer", "tiktoken", "--json"] + ) + if result.exit_code != 0: + return + err = result.stderr or "" + assert "approximate" in err.lower() or "warning" in err.lower() diff --git a/tests/test_reporters.py b/tests/test_reporters.py index 2201732..5c56881 100644 --- a/tests/test_reporters.py +++ b/tests/test_reporters.py @@ -62,3 +62,11 @@ def test_diff_dict_structure(openai_array, openai_chat): d = diff_to_dict(a, b) assert d["delta_tokens"] == b.total_tokens - a.total_tokens assert d["segments"] + + +def test_json_includes_token_accuracy(claude_jsonl): + a = analyze_file(claude_jsonl, tokenizer="heuristic") + d = to_dict(a) + assert d["token_accuracy"] == "approximate" + assert d["tokenizer"] == "heuristic" + assert d["accuracy_note"] diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py index fb3c86d..7a8fe9d 100644 --- a/tests/test_tokenizers.py +++ b/tests/test_tokenizers.py @@ -63,3 +63,38 @@ def test_tiktoken_requested_but_missing_raises_or_works(): except ImportError: return assert tok.count("hello") > 0 + + +def test_count_accuracy_claude_tiktoken_is_approximate(): + from ctxlens.tokenizers import accuracy_note, count_accuracy + + try: + tok = get_tokenizer("tiktoken") + except ImportError: + pytest.skip("tiktoken not installed") + assert tok.name == "tiktoken:cl100k_base" + assert count_accuracy(tok, "claude-code-jsonl") == "approximate" + note = accuracy_note(tok, "claude-code-jsonl") + assert note is not None + assert "approximate" in note.lower() + assert "OpenAI" in note or "cl100k" in note + + +def test_count_accuracy_openai_tiktoken_is_exact(): + from ctxlens.tokenizers import accuracy_note, count_accuracy + + try: + tok = get_tokenizer("tiktoken") + except ImportError: + pytest.skip("tiktoken not installed") + assert count_accuracy(tok, "openai-chat") == "exact" + assert count_accuracy(tok, "codex-session") == "exact" + assert accuracy_note(tok, "openai-chat") is None + + +def test_heuristic_always_approximate(): + from ctxlens.tokenizers import count_accuracy + + tok = get_tokenizer("heuristic") + assert count_accuracy(tok, "openai-chat") == "approximate" + assert count_accuracy(tok, "claude-code-jsonl") == "approximate"