From fa379ca53e5330da80c110aa44c6f618ef796b1a Mon Sep 17 00:00:00 2001 From: Laxmikant Sharma Date: Thu, 20 Aug 2026 13:10:18 -0700 Subject: [PATCH] Sort JCS object keys by UTF-16 code unit, not code point RFC 8785 section 3.2.3 orders object keys by UTF-16 code unit. json.dumps(sort_keys=True) orders them by Unicode code point. The two agree throughout the Basic Multilingual Plane and diverge above it, where a supplementary-plane key encodes as a surrogate pair beginning U+D800 and therefore sorts below keys in U+E000..U+FFFF. No published hash changes. Every key in every shipped example is ASCII, so the two orderings already agreed on all of them; spec_hash, criteria_hash and vtc_hash are byte-identical before and after. This closes a latent defect rather than correcting a live one. The module caveat previously disclosed only the number-serialization limitation and said nothing about key order, which was the more misleading omission of the two: a canonicalizer built on the shortcut passes an ASCII vector by accident and would fail on the first non-BMP key it ever saw. Two vectors now pin the behaviour, one at the root and one nested, and the caveat names what is fixed and what is still not implemented. Prompted by a canonicalization vector raised by @wowlegend in x402-foundation/x402#3065, which pins the same divergence class. --- README.md | 11 +++++--- tools/validate.py | 64 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 51926b6..8f2f753 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ fix does not depend on those design decisions. The validator also checks the rules JSON Schema cannot express (parties are distinct, one signature per named party, protected headers carry -`alg`/`kid`/`typ` with an allowed algorithm) and runs negative vectors -that must be rejected. +`alg`/`kid`/`typ` with an allowed algorithm), pins the canonical key +order described below, and runs negative vectors that must be rejected. ``` pip install jsonschema referencing @@ -103,7 +103,12 @@ producing real JWS signatures requires party keys. And `jcs()` in correct for the value types these examples use but is not a conforming general one, so a green run evidences self-consistency of these examples rather than canonicalization interoperability with another -implementation. +implementation. What it does get right, and pins with a vector, is the +key order: RFC 8785 section 3.2.3 sorts object keys by UTF-16 code unit, +which `json.dumps(sort_keys=True)` does not, the two agreeing throughout +the Basic Multilingual Plane and diverging above it. What remains +unimplemented is the ECMAScript number serialization rules over the full +float range. ## Building the draft diff --git a/tools/validate.py b/tools/validate.py index 9c832e5..d6078ff 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -16,16 +16,29 @@ 7. Rules the schemas cannot express: parties are distinct, one signature per named party, and every JOSE protected header carries alg, kid and typ with an allowed algorithm. - 8. Negative vectors: mutations that MUST be rejected actually are. + 8. Canonicalization: jcs() orders object keys the way RFC 8785 + requires, which is not the way json.dumps(sort_keys=True) does. + 9. Negative vectors: mutations that MUST be rejected actually are. Caveat on canonicalization: jcs() below is a restricted implementation of RFC 8785, correct for the value types these examples use (strings, integers, floats with exact short decimal representations, booleans, -nulls, and nested objects and arrays of those). It is not a conforming -general RFC 8785 implementation, and in particular it does not implement -the ECMAScript number serialization rules for the full float range. A -passing run therefore evidences self-consistency of these examples, not -canonicalization interoperability with another implementation. +nulls, and nested objects and arrays of those). + +Object keys are sorted by UTF-16 code unit, as RFC 8785 section 3.2.3 +requires. This is worth stating because the obvious shortcut is wrong: +json.dumps(sort_keys=True) sorts by Unicode code point, and code point +order agrees with UTF-16 order throughout the Basic Multilingual Plane +and diverges above it, where UTF-16 encodes a key as a surrogate pair +beginning U+D800 and therefore sorts it below keys in U+E000..U+FFFF. +An implementation carrying that shortcut passes an ASCII or BMP vector by +accident and fails on a supplementary-plane key. Check 8 pins the case. + +It remains a restricted implementation and not a conforming general RFC +8785 one: in particular it does not implement the ECMAScript number +serialization rules for the full float range. A passing run therefore +evidences self-consistency of these examples, not canonicalization +interoperability with another implementation. """ import json, hashlib, sys, pathlib, base64 from jsonschema import Draft202012Validator @@ -37,9 +50,26 @@ ALLOWED_ALGS = {"ES256", "ES384", "EdDSA"} +def _utf16_key_order(obj): + """Recursively reorder object keys by UTF-16 code unit (RFC 8785 3.2.3). + + Comparing UTF-16 big-endian encodings bytewise is equivalent to + comparing sequences of UTF-16 code units, which is what the RFC + specifies. json.dumps preserves dict insertion order, so building the + dict in the right order is enough; sort_keys must NOT also be set, + since that would re-sort by code point. + """ + if isinstance(obj, dict): + return {k: _utf16_key_order(obj[k]) + for k in sorted(obj, key=lambda s: s.encode("utf-16-be"))} + if isinstance(obj, list): + return [_utf16_key_order(v) for v in obj] + return obj + + def jcs(obj) -> bytes: # Restricted JCS (RFC 8785); see the caveat in the module docstring. - return json.dumps(obj, sort_keys=True, separators=(",", ":"), + return json.dumps(_utf16_key_order(obj), separators=(",", ":"), ensure_ascii=False).encode() @@ -116,6 +146,26 @@ def validate(example, schema_file, quiet=False): check("attestation matches schema", validate(att, "attestation.schema.json")) check("well-known matches schema", validate(wk, "wellknown.schema.json")) +print() +print("== canonicalization ==") + +# RFC 8785 section 3.2.3 orders object keys by UTF-16 code unit. U+E000 +# is below U+10000 by code point, and above it by UTF-16 code unit, since +# U+10000 encodes as the surrogate pair D800 DC00. A canonicalizer built +# on json.dumps(sort_keys=True) gets this backwards and no ASCII vector +# will reveal it. +_supp = {"\ue000": 1, "\U00010000": 2} +_want = ('{"' + "\U00010000" + '":2,"' + "\ue000" + '":1}').encode() +check("JCS orders keys by UTF-16 code unit, not code point", + jcs(_supp) == _want, f"got {jcs(_supp)!r}, want {_want!r}") + +# The same rule has to hold at every depth, not just at the root. +_nested = {"z": [{"\ue000": 1, "\U00010000": 2}]} +_want_nested = ('{"z":[{"' + "\U00010000" + '":2,"' + "\ue000" + '":1}]}').encode() +check("JCS key order applies inside nested objects and arrays", + jcs(_nested) == _want_nested, + f"got {jcs(_nested)!r}, want {_want_nested!r}") + print() print("== hash commitments ==") check("cfb.spec_hash == sha256(JCS(taskspec))",