Refuse exact token claims for Claude under tiktoken cl100k

README.md

@@ -16,15 +16,17 @@ turn, tool schemas re-sent on every step. Token dashboards tell you the bill.
 **ctxlens tells you where the bytes went and what to delete.**
 
 It works offline with a deterministic heuristic tokenizer (no network, no heavy
-deps), and upgrades to exact counts automatically when `tiktoken` is installed.
+deps), and upgrades to OpenAI-exact counts via `tiktoken` when installed. Counts
+for Claude Code sessions remain **approximate** under tiktoken — `cl100k_base` is
+OpenAI's encoding, not Anthropic's.
 
 ## Quickstart
 
 **PyPI:** https://pypi.org/project/ctxlens-cli/
 
 ```bash
 pip install ctxlens-cli                # core
-pip install "ctxlens-cli[tiktoken]"    # optional exact token counts
+pip install "ctxlens-cli[tiktoken]"    # optional OpenAI-exact token counts
 
 ctxlens analyze session.jsonl
 ctxlens report session.jsonl --html -o report.html
@@ -110,9 +112,20 @@ output with `--json`, or diff a baseline against a candidate in CI with
 
 ## Tokenizers
 
+Counts are only as accurate as the encoding vs the model that produced the
+session. The report labels each run `exact` or `approximate` (`token_accuracy`
+in JSON) and never claims exactness for Claude sessions under OpenAI encodings.
+
+| Tokenizer | Name shown | OpenAI chat / Codex | Claude Code |
+| --- | --- | --- | --- |
+| `heuristic` | `heuristic` | approximate | approximate |
+| `tiktoken` (default `cl100k_base`) | `tiktoken:cl100k_base` | **exact** (OpenAI BPE) | **approximate** (wrong encoding) |
+
 - `heuristic` (default fallback): deterministic, dependency-free, great for
-  relative profiling and CI.
-- `tiktoken`: exact BPE counts when installed. `--tokenizer auto` prefers it.
+  relative profiling and CI. Always an estimate.
+- `tiktoken`: OpenAI `cl100k_base` BPE when installed. Exact for OpenAI/Codex
+  transcripts; approximate for Claude Code. `--tokenizer auto` prefers it, and
+  the CLI prints a warning when the count is approximate for the session format.
 
 ## Contributing
 

src/ctxlens/cli.py

@@ -144,15 +144,21 @@ def _load(path, fmt, tokenizer, top, tool_result_cap, tool_def_budget):
     try:
         if str(path) == "-":
             raw = sys.stdin.read()
-            return analyze_text(raw, source="<stdin>", **common)
-        p = Path(path)
-        if not p.is_file():
-            raise ParseError(f"no such file: {path}")
-        return analyze_file(p, **common)
+            analysis = analyze_text(raw, source="<stdin>", **common)
+        else:
+            p = Path(path)
+            if not p.is_file():
+                raise ParseError(f"no such file: {path}")
+            analysis = analyze_file(p, **common)
     except (ParseError, ImportError, ValueError, OSError) as exc:
         err_console.print(f"[red]error:[/red] {exc}")
         raise typer.Exit(EXIT_ERROR) from exc
 
+    # Refuse to imply exactness for formats without a matching tokenizer.
+    if analysis.token_accuracy == "approximate" and analysis.accuracy_note:
+        err_console.print(f"[yellow]warning:[/yellow] {analysis.accuracy_note}")
+    return analysis
+
 
 def _maybe_fail(ratio: float, threshold: float | None):
     if threshold is not None and ratio > threshold:

src/ctxlens/engine.py

@@ -14,7 +14,7 @@
 from ctxlens.analysis.waste import WasteReport, build_waste_report
 from ctxlens.models import Session
 from ctxlens.parsers import parse_file, parse_text
-from ctxlens.tokenizers import get_tokenizer
+from ctxlens.tokenizers import accuracy_note, count_accuracy, get_tokenizer
 
 
 @dataclass
@@ -24,6 +24,10 @@ class Analysis:
     waste: WasteReport
     recommendations: list[Recommendation]
     source: str | None = None
+    #: "exact" only when the tokenizer encoding matches the session's model family
+    token_accuracy: str = "approximate"
+    #: human-readable caveat when counts are not exact; None when exact
+    accuracy_note: str | None = None
 
     @property
     def total_tokens(self) -> int:
@@ -49,12 +53,16 @@ def analyze_session(
         session, tool_result_cap=tool_result_cap, tool_def_budget=tool_def_budget
     )
     recs = recommend(profile, waste)
+    accuracy = count_accuracy(tok, session.source_format)
+    note = accuracy_note(tok, session.source_format)
     return Analysis(
         session=session,
         profile=profile,
         waste=waste,
         recommendations=recs,
         source=source,
+        token_accuracy=accuracy,
+        accuracy_note=note,
     )
 
 

src/ctxlens/reporters/json_report.py

@@ -15,6 +15,8 @@ def to_dict(analysis: Analysis) -> dict:
         "source": analysis.source,
         "format": analysis.session.source_format,
         "tokenizer": p.tokenizer_name,
+        "token_accuracy": analysis.token_accuracy,
+        "accuracy_note": analysis.accuracy_note,
         "total_tokens": total,
         "high_water_mark": p.high_water_mark,
         "turns": len(p.turn_stats),

src/ctxlens/reporters/terminal.py

@@ -31,16 +31,24 @@ def _header(a: Analysis) -> Panel:
     src = a.source or "<stdin>"
     ratio = a.waste_ratio * 100
     ratio_style = "red" if ratio >= 25 else "yellow" if ratio >= 10 else "green"
-    body = Text.assemble(
+    accuracy = getattr(a, "token_accuracy", "approximate")
+    tok_label = p.tokenizer_name
+    if accuracy == "approximate":
+        tok_label = f"{p.tokenizer_name} (approximate)"
+    parts: list = [
         ("Source   ", "dim"), f"{src}\n",
         ("Format   ", "dim"), f"{a.session.source_format}   ",
-        ("Tokenizer ", "dim"), f"{p.tokenizer_name}\n",
+        ("Tokenizer ", "dim"), f"{tok_label}\n",
         ("Tokens   ", "dim"), f"{p.total_tokens:,}   ",
         ("Turns ", "dim"), f"{len(p.turn_stats)}   ",
         ("High-water ", "dim"), f"{p.high_water_mark:,}\n",
         ("Waste    ", "dim"),
         (f"{a.waste.total_waste:,} tokens ({ratio:.1f}%)", ratio_style),
-    )
+    ]
+    note = getattr(a, "accuracy_note", None)
+    if note:
+        parts.extend(["\n", ("Note     ", "dim"), (note, "yellow")])
+    body = Text.assemble(*parts)
     return Panel(body, title="ctxlens", border_style="cyan", expand=False)
 
 

src/ctxlens/tokenizers/__init__.py

@@ -3,16 +3,26 @@
 The analysis layer never counts tokens directly; it goes through a
 :class:`Tokenizer`. This keeps ctxlens usable with no heavy dependencies (the
 :class:`HeuristicTokenizer` is deterministic and network-free) while still
-allowing an exact count via tiktoken when it is installed.
+allowing OpenAI-exact counts via tiktoken when it is installed.
+
+Tiktoken's default encoding is OpenAI ``cl100k_base``. It is **not** exact for
+Claude Code sessions; see :func:`count_accuracy`.
 """
 
 from ctxlens.tokenizers.base import Tokenizer
 from ctxlens.tokenizers.heuristic import HeuristicTokenizer
-from ctxlens.tokenizers.registry import available_tokenizers, get_tokenizer
+from ctxlens.tokenizers.registry import (
+    accuracy_note,
+    available_tokenizers,
+    count_accuracy,
+    get_tokenizer,
+)
 
 __all__ = [
     "Tokenizer",
     "HeuristicTokenizer",
     "get_tokenizer",
     "available_tokenizers",
+    "count_accuracy",
+    "accuracy_note",
 ]

src/ctxlens/tokenizers/registry.py

@@ -1,15 +1,29 @@
 """Tokenizer selection.
 
 ``get_tokenizer("auto")`` prefers tiktoken when available and silently falls
-back to the heuristic tokenizer otherwise, so the default experience is exact
-where possible and always works offline.
+back to the heuristic tokenizer otherwise.
+
+**Accuracy is format-dependent.** ``tiktoken`` defaults to OpenAI's
+``cl100k_base`` encoding. That is exact for OpenAI/Codex chat transcripts and
+only approximate for Claude Code sessions (Anthropic models do not use
+``cl100k_base``). The heuristic is always an estimate. Callers must not treat
+every tiktoken count as exact — use :func:`count_accuracy` / the analysis
+``token_accuracy`` field.
 """
 
 from __future__ import annotations
 
+from typing import Literal
+
 from ctxlens.tokenizers.base import Tokenizer
 from ctxlens.tokenizers.heuristic import HeuristicTokenizer
 
+# Source formats whose native tokenizers match OpenAI cl100k_base closely enough
+# that a tiktoken:cl100k_base count is treated as exact.
+_OPENAI_FORMATS = frozenset({"openai-chat", "codex-session"})
+
+Accuracy = Literal["exact", "approximate"]
+
 
 def _try_tiktoken(encoding: str = "cl100k_base") -> Tokenizer | None:
     try:
@@ -32,7 +46,11 @@ def get_tokenizer(name: str = "auto") -> Tokenizer:
 
     ``auto``      -> tiktoken if installed, else heuristic
     ``heuristic`` -> always the dependency-free heuristic
-    ``tiktoken``  -> tiktoken, raising ImportError if unavailable
+    ``tiktoken``  -> tiktoken (OpenAI cl100k_base), raising ImportError if unavailable
+
+    The returned tokenizer's ``name`` identifies the encoding (e.g.
+    ``tiktoken:cl100k_base``). Exactness for a given session still depends on
+    ``source_format`` — see :func:`count_accuracy`.
     """
     name = (name or "auto").lower()
     if name == "heuristic":
@@ -47,3 +65,39 @@ def get_tokenizer(name: str = "auto") -> Tokenizer:
     if name == "auto":
         return _try_tiktoken() or HeuristicTokenizer()
     raise ValueError(f"unknown tokenizer: {name!r}")
+
+
+def count_accuracy(tokenizer: Tokenizer, source_format: str) -> Accuracy:
+    """Whether counts from ``tokenizer`` are exact for ``source_format``.
+
+    * ``tiktoken:cl100k_base`` is exact for OpenAI chat and Codex sessions.
+    * Claude Code and any other format are approximate under cl100k (wrong BPE).
+    * The heuristic is always approximate.
+    """
+    name = getattr(tokenizer, "name", "") or ""
+    if name.startswith("tiktoken:"):
+        # Only claim exactness when the encoding matches the model family.
+        # Default encoding is cl100k_base (OpenAI). No Anthropic encoding ships here.
+        encoding = name.split(":", 1)[1]
+        if encoding == "cl100k_base" and source_format in _OPENAI_FORMATS:
+            return "exact"
+        return "approximate"
+    return "approximate"
+
+
+def accuracy_note(tokenizer: Tokenizer, source_format: str) -> str | None:
+    """Human-readable note when the count is approximate, else None."""
+    if count_accuracy(tokenizer, source_format) == "exact":
+        return None
+    name = getattr(tokenizer, "name", "tokenizer")
+    if name.startswith("tiktoken:") and source_format.startswith("claude"):
+        return (
+            f"{name} is OpenAI's cl100k_base encoding; counts for Claude Code "
+            f"sessions ({source_format}) are approximate, not exact"
+        )
+    if name.startswith("tiktoken:"):
+        return (
+            f"{name} counts are approximate for source format {source_format!r} "
+            "(no matching model encoding is available)"
+        )
+    return f"{name} is an estimate, not an exact model tokenizer count"

src/ctxlens/tokenizers/tiktoken_tokenizer.py

@@ -1,4 +1,10 @@
-"""Exact token counting backed by tiktoken (optional dependency)."""
+"""Token counting backed by tiktoken (optional dependency).
+
+The default encoding is OpenAI's ``cl100k_base``. That is **exact for OpenAI /
+Codex transcripts** and only an approximation for Claude Code sessions —
+Anthropic models do not use this encoding. Prefer the analysis
+``token_accuracy`` field over assuming every tiktoken count is exact.
+"""
 
 from __future__ import annotations
 
@@ -10,6 +16,9 @@ class TiktokenTokenizer(Tokenizer):
 
     Constructing this raises :class:`ImportError` if tiktoken is not installed,
     so callers can fall back to the heuristic tokenizer gracefully.
+
+    ``name`` always includes the encoding id (e.g. ``tiktoken:cl100k_base``) so
+    reported counts are traceable to a specific vocabulary.
     """
 
     def __init__(self, encoding: str = "cl100k_base") -> None:
@@ -19,6 +28,7 @@ def __init__(self, encoding: str = "cl100k_base") -> None:
             raise ImportError(
                 "tiktoken is not installed; install with `pip install ctxlens[tiktoken]`"
             ) from exc
+        self.encoding = encoding
         self._enc = tiktoken.get_encoding(encoding)
         self.name = f"tiktoken:{encoding}"
 

tests/test_cli.py

@@ -78,3 +78,28 @@ def test_analyze_stdin(openai_array):
     data = json.loads(result.stdout)
     assert data["source"] == "<stdin>"
     assert data["total_tokens"] > 0
+
+
+def test_analyze_json_marks_claude_tiktoken_approximate(claude_jsonl):
+    result = runner.invoke(
+        app, ["analyze", str(claude_jsonl), "--tokenizer", "tiktoken", "--json"]
+    )
+    combined = (result.stdout or "") + (result.stderr or "")
+    if result.exit_code != 0 and "not installed" in combined.lower():
+        return
+    assert result.exit_code == 0, combined
+    data = json.loads(result.stdout)
+    assert data["tokenizer"].startswith("tiktoken:")
+    assert data["token_accuracy"] == "approximate"
+    assert data["accuracy_note"]
+    assert "approximate" in data["accuracy_note"].lower()
+
+
+def test_analyze_claude_tiktoken_warns_on_stderr(claude_jsonl):
+    result = runner.invoke(
+        app, ["analyze", str(claude_jsonl), "--tokenizer", "tiktoken", "--json"]
+    )
+    if result.exit_code != 0:
+        return
+    err = result.stderr or ""
+    assert "approximate" in err.lower() or "warning" in err.lower()

tests/test_reporters.py

@@ -62,3 +62,11 @@ def test_diff_dict_structure(openai_array, openai_chat):
     d = diff_to_dict(a, b)
     assert d["delta_tokens"] == b.total_tokens - a.total_tokens
     assert d["segments"]
+
+
+def test_json_includes_token_accuracy(claude_jsonl):
+    a = analyze_file(claude_jsonl, tokenizer="heuristic")
+    d = to_dict(a)
+    assert d["token_accuracy"] == "approximate"
+    assert d["tokenizer"] == "heuristic"
+    assert d["accuracy_note"]

tests/test_tokenizers.py

@@ -63,3 +63,38 @@ def test_tiktoken_requested_but_missing_raises_or_works():
     except ImportError:
         return
     assert tok.count("hello") > 0
+
+
+def test_count_accuracy_claude_tiktoken_is_approximate():
+    from ctxlens.tokenizers import accuracy_note, count_accuracy
+
+    try:
+        tok = get_tokenizer("tiktoken")
+    except ImportError:
+        pytest.skip("tiktoken not installed")
+    assert tok.name == "tiktoken:cl100k_base"
+    assert count_accuracy(tok, "claude-code-jsonl") == "approximate"
+    note = accuracy_note(tok, "claude-code-jsonl")
+    assert note is not None
+    assert "approximate" in note.lower()
+    assert "OpenAI" in note or "cl100k" in note
+
+
+def test_count_accuracy_openai_tiktoken_is_exact():
+    from ctxlens.tokenizers import accuracy_note, count_accuracy
+
+    try:
+        tok = get_tokenizer("tiktoken")
+    except ImportError:
+        pytest.skip("tiktoken not installed")
+    assert count_accuracy(tok, "openai-chat") == "exact"
+    assert count_accuracy(tok, "codex-session") == "exact"
+    assert accuracy_note(tok, "openai-chat") is None
+
+
+def test_heuristic_always_approximate():
+    from ctxlens.tokenizers import count_accuracy
+
+    tok = get_tokenizer("heuristic")
+    assert count_accuracy(tok, "openai-chat") == "approximate"
+    assert count_accuracy(tok, "claude-code-jsonl") == "approximate"