diff --git a/.gitignore b/.gitignore index 731cf3d..ffc1282 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,17 @@ peers.json # Overlay identity nonce written to CWD by CLI runs (default `--nonce-file`). # A runtime artifact, like peers.json — never commit it. overlay-nonce +# Cumulative payouts issued to bee peers and metered relays, written to CWD +# by paid uploads. Runtime money state: committing it would publish who was +# paid what, and restoring a stale copy re-issues cheques the counterparty +# has already banked. +cheques.json +# Agent-local tool config. `settings.local.json` itself is usually covered by +# a global ignore, but the editor writes `*.tmp..` siblings that +# such a pattern misses — and those files quote whole shell commands, so a +# `git add -A` mid-session can publish a key that was passed on a command +# line. Ignore the whole directory. +.claude/ # `peers.seed.json` is checked in: an IP-diverse cold-start seed # harvested from a long-running daemon (~800 peers across ~800 unique # /32 IPs as of the last refresh; regenerate via diff --git a/apps/upload/src/config.ts b/apps/upload/src/config.ts index af9cc53..db71ba9 100644 --- a/apps/upload/src/config.ts +++ b/apps/upload/src/config.ts @@ -14,6 +14,12 @@ * * Chunks are sharded across the lanes (rendezvous hashing) and pushed * concurrently; a chunk unacked by one lane fails over to the next. + * + * Payment is per lane and optional (docs/pusher-incentives.md). A lane + * advertising `enforcement: "hard"` in its `/v1/status` is dropped from + * rotation at startup by `setLaneStatus`, because this dApp only stamps — + * the chequebook lives in the native client. Such a lane is listed anyway so + * a native `--chequebook` run of the same fleet picks it up. */ export const PUSHER_URLS: string[] = [ 'https://hoverfly-pusher.onrender.com', @@ -21,7 +27,11 @@ export const PUSHER_URLS: string[] = [ 'https://hoverfly-pusher-3.onrender.com', // Hugging Face Space — a different provider/IP-range from Render (probed: // HF permits outbound TCP to bee nodes), for cross-provider lane diversity. - 'https://ivam5567-hoverfly.hf.space' + 'https://ivam5567-hoverfly.hf.space', + // A self-hosted VPS lane, and the first metered one: it runs `--meter` with + // hard enforcement, so it serves paying native clients and is skipped by + // the browser. Unlike the free tiers above it doesn't cold-start. + 'https://pusher.browserbzz.link' ] /** * How long to wait for a relay's `/v1/status` before scheduling it on diff --git a/apps/upload/src/worker.ts b/apps/upload/src/worker.ts index 7d1232f..1b62bc7 100644 --- a/apps/upload/src/worker.ts +++ b/apps/upload/src/worker.ts @@ -64,8 +64,12 @@ interface UploadSession { readonly failed: number readonly hedges: number readonly done: boolean - /** Feed a lane's /v1/status JSON (pool size, batch_max, budget, overlay). */ - setLaneStatus: (lane: number, status: unknown) => void + /** + * Feed a lane's /v1/status JSON (pool size, batch_max, budget, overlay). + * False when the lane was retired instead of scheduled — it enforces + * payment and this build has no chequebook. + */ + setLaneStatus: (lane: number, status: unknown) => boolean /** Next POST to issue, or undefined if nothing is dispatchable now. */ nextRequest: (nowMs: number) => PushRequest | undefined /** One streamed NDJSON ack. Idempotent per address (hedges rely on this). */ @@ -375,10 +379,23 @@ async function pushSession (session: UploadSession, lanes: string[]): Promise { const st = await fetchLaneStatus(u) - if (st !== undefined) session.setLaneStatus(i, st) + if (st === undefined) { usable++; return } // asleep, not refusing — keep it + if (session.setLaneStatus(i, st)) usable++ + else log(`Pusher ${u} requires payment; skipping it (browser uploads are unpaid).`) })) + if (usable === 0) { + throw new Error('every relay in PUSHER_URLS requires payment — the browser cannot pay') + } const pushUrls = lanes.map(u => `${u.replace(/\/+$/, '')}/v1/push`) let lastPost = 0 diff --git a/docs/deck/build.py b/docs/deck/build.py new file mode 100644 index 0000000..1b359d7 --- /dev/null +++ b/docs/deck/build.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build the slide deck HTML from the Marp markdown. + + python3 docs/deck/build.py docs/pusher-incentives-slides.md -o /tmp/deck.html + +The markdown is the source. It stays a valid Marp deck — `marp-cli` renders it +to PDF unchanged — and this script produces the self-contained HTML version +with keyboard navigation. Inline markdown, GFM tables and column alignment are +handled by `md2html` (md4c), not by anything here. + +Four directives, all HTML comments, so every markdown renderer ignores them: + + this slide is the title card + section divider, with that kicker + the small label above the heading + the next blockquote is the warning colour + +Blockquotes become the accent callout. Tables are wrapped so wide ones scroll +inside the slide rather than pushing the page sideways. +""" +import argparse +import html +import pathlib +import re +import shutil +import subprocess +import sys + +HERE = pathlib.Path(__file__).parent + + +def render(md: str) -> str: + """Markdown fragment -> HTML, via md2html.""" + if not md.strip(): + return "" + out = subprocess.run( + ["md2html", "--github"], input=md, capture_output=True, text=True, check=True + ).stdout + # Wide content scrolls in its own box; the slide itself never does. + out = re.sub(r"(.*?
)", r'
\1
', out, flags=re.S) + return out.strip() + + +def slide(body: str, index: int) -> str: + """One markdown slide -> one
.""" + title = "" in body + part = re.search(r"", body) + eyebrow = re.search(r"", body) + body = re.sub(r"", "", body) + + # `` tags the blockquote that follows it. md2html drops + # comments, so the flag rides through as a sentinel inside the quote. + body = re.sub(r"\s*\n+>", "> @@HZ@@", body) + out = render(body) + + # Blockquote is the deck's callout. Unwrap its paragraph so the callout is + # one styled block rather than a quote wrapping a paragraph. + def callout(m: re.Match) -> str: + inner = re.sub(r"", "", m.group(1)).strip() + cls = "claim" + if inner.startswith("@@HZ@@"): + cls, inner = "claim hz", inner[len("@@HZ@@"):].strip() + return f'
{inner}
' + + out = re.sub(r"
(.*?)
", callout, out, flags=re.S) + + num = f'
{index:02d}
' + + if title: + h1 = re.search(r"

(.*?)

", out, re.S) + h2 = re.search(r"

(.*?)

", out, re.S) + meta = re.search(r"

(.*?)

", out, re.S) + return ( + '
\n' + '
\n' + f"

{h1.group(1) if h1 else ''}

\n" + f'

{h2.group(1) if h2 else ""}.

\n' + f'
{meta.group(1) if meta else ""}
\n' + "
" + ) + + if part: + h1 = re.search(r"

(.*?)

", out, re.S) + return ( + '
\n' + f'
{html.escape(part.group(1))}
\n' + f"

{h1.group(1) if h1 else ''}

\n" + f" {num}\n" + "
" + ) + + # A slide's `#` heading is an

visually —

is reserved for the deck. + out = re.sub(r"

(.*?)

", r"

\1

", out, flags=re.S) + eb = ( + f'
{eyebrow.group(1)}
\n ' + if eyebrow + else "" + ) + # Everything that is not a heading, table, code block or callout is prose. + out = re.sub( + r"((?:<(?:p|ul|ol)>.*?\s*)+)", + lambda m: f'
{m.group(1).strip()}
', + out, + flags=re.S, + ) + return f'
\n {eb}{out}\n {num}\n
' + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("source", type=pathlib.Path) + ap.add_argument("-o", "--output", type=pathlib.Path, required=True) + args = ap.parse_args() + + if not shutil.which("md2html"): + print("md2html not found (md4c). Install it, or render with marp-cli.", file=sys.stderr) + return 1 + + text = args.source.read_text() + # Drop the Marp front matter; it configures marp-cli, not this. + text = re.sub(r"\A---\n.*?\n---\n", "", text, flags=re.S) + title = "Paying for relay — an incentive layer for hoverfly pushers" + + slides = [s for s in re.split(r"\n---\n", text) if s.strip()] + sections = "\n\n ".join(slide(s, i + 1) for i, s in enumerate(slides)) + + args.output.write_text( + f"{title}\n\n" + + (HERE / "shell.css.html").read_text() + + f'\n
\n\n {sections}\n\n
\n\n' + + (HERE / "shell.bar.html").read_text() + + "\n" + + (HERE / "shell.js.html").read_text() + + "\n" + ) + print(f"{len(slides)} slides -> {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/deck/shell.bar.html b/docs/deck/shell.bar.html new file mode 100644 index 0000000..38d1235 --- /dev/null +++ b/docs/deck/shell.bar.html @@ -0,0 +1,7 @@ +
+ + +
+ +
+ diff --git a/docs/deck/shell.css.html b/docs/deck/shell.css.html new file mode 100644 index 0000000..1d46cc1 --- /dev/null +++ b/docs/deck/shell.css.html @@ -0,0 +1,279 @@ + \ No newline at end of file diff --git a/docs/deck/shell.js.html b/docs/deck/shell.js.html new file mode 100644 index 0000000..2789325 --- /dev/null +++ b/docs/deck/shell.js.html @@ -0,0 +1,68 @@ + \ No newline at end of file diff --git a/docs/pusher-design.md b/docs/pusher-design.md index 1d94739..5efe84c 100644 --- a/docs/pusher-design.md +++ b/docs/pusher-design.md @@ -646,7 +646,10 @@ worked without it: ### Deferred / watchlist - PR-sized follow-ups: `--push-quota`/`--push-challenge` hardening, P2 - workerd port (unlocks Deno Deploy), attribution-log tooling. + workerd port (unlocks Deno Deploy), attribution-log tooling. Note that + the metered-relay design (§12) subsumes `--push-quota` outright — price + is a strictly better quota — and promotes `--push-challenge` from + optional hardening to a correctness requirement. - Contiguous-arc lane assignment + deep pool specialization — only if receipt data shows forwarding depth is a real cost (§7). - WS/WT bindings of the frame format — only on demonstrated need (§4). @@ -656,7 +659,107 @@ worked without it: dial storage nodes directly and the pusher's raison d'être shrinks to constrained networks. -Deferred/watchlist: +## 12. Incentives — paying for relay + +Specified separately in **[`pusher-incentives.md`](./pusher-incentives.md)** +(status: design only, nothing implemented). + +The problem it addresses: a relay absorbs a cost it did not incur. In a +native upload the user's own machine is the peer bee debits for every +chunk; put a relay in the middle and that debt moves wholesale to the +relay, while the browser client that caused the traffic pays only postage. +§6 books this as an accepted risk and §10 identifies the dedicated egress +IP as the thing that is actually scarce. + +The design adds an optional **`metered`** relay mode (today's behaviour +becomes `open`, and the four production lanes stay there) in which a +client pays with off-chain SWAP cheques over the existing HTTPS channel. +Both counterparties are hoverfly — bee is not a party, and only the +chequebook contract and EIP-712 cheque format are borrowed, not the swap +protocol. Payment is out-of-band (`POST /v1/pay`), the account is the +batch-owner EOA already established by push auth, and the relay holds only +the beneficiary *address* — never a spendable key. + +**The trust model is one-directional, and it drives everything else.** +Relays are a curated set pinned by URL (`PUSHER_URLS`, +`apps/upload/src/config.ts:18-25`); clients are anonymous. So the design +protects the *relay* from the *client*, and a misbehaving lane is handled +socially — removed from the list — rather than cryptographically. An +earlier revision built two-sided verification for this one-sided +relationship and paid for it with a forgeable billing unit and an +unbounded residual; see incentives §2. + +Findings from that doc that constrain this one: + +- **The billing unit is bytes admitted, not receipts or acks.** The client + cannot lie about it, because the client produced the bytes and the relay + counted them — no third-party attestation to forge, no chain state to + disagree about (incentives §8). This matters here because it settles + what receipt forwarding is *for*: forwarding `PushsyncReceipt` into the + ack is still worth doing — `PushInfo` currently drops the signature, so + acks are pure relay assertion, and it is the first cryptographic signal + a relay client has ever had — but it is **telemetry that feeds lane + weighting, not evidence that feeds an invoice**. A receipt signs the + bare chunk address (`bee/pkg/pushsync/pushsync.go:277`), so it is + forgeable with any throwaway key and carries no freshness; anything that + prices work by receipt inherits both problems. +- **`--push-challenge` becomes mandatory under metering.** Swarm stamps + are public (§6), so an attacker can replay a victim's stamps at a + metered relay and have the work billed to the victim. The signed payload + must bind the relay's **origin**, not its beneficiary, and must be an + EIP-712 typed struct — the same account key already signs stamps and + cheques, so a third raw-bytes scheme over it invites confusion. And the + origin the relay compares against must come from **configuration, never + from the `Host` header** — the header is supplied by the same party + supplying the challenge, so checking one against the other is a no-op + that silently reopens the replay (incentives §11.1). +- **"A batch owner is a costly identity" is false as stated.** The relay + checks batch *liveness*, and the cheapest batch the contract accepts + costs a fraction of a cent — so any flat per-account credit line is + roughly self-financing for an attacker. The fix is to scale the credit + line to the batch's remaining on-chain value rather than gate on it + (incentives §10.3). Worth knowing here because it is the same mistake + any future per-batch quota would make. +- **`--push-quota` should be struck** if metered mode ships; price is a + strictly better quota than a volume cliff. +- **Six live bugs in open-mode code, found during those reviews — all + fixed** (incentives §16). None depend on metering; they were in + production the whole time. + - *Stamp substitution via the recent-ack cache.* Dedup was keyed on + chunk address alone, so a hit acked a frame `ok` while silently + discarding the submitted stamp. Since addresses are content-derived, + one uploader's dust batch could shadow another's year-long batch for + the 120 s TTL — the victim's chunk then garbage-collected when the + attacker's batch expires. It also fired accidentally between honest + users uploading the same file. Now keyed on `(addr, batch_id)` + (`src/pusher.rs:286`). + - *Unauthenticated RPC amplification on `/v1/push`.* `resolve_owner` + cached only successes, unbounded, with no per-request budget, so one + anonymous POST naming bogus batches became up to 512 serial + `eth_call`s — and `EthRpc::new` built a fresh HTTP client per read. + Now a bounded cache with negative caching, TTLs, and an 8-lookup + budget, over one shared client. + - *No connection limit.* The accept loop spawned per connection with + nothing bounding it, despite §3 listing a cap as table stakes. Now a + 256-permit semaphore acquired before `accept()`. + - *One transient accept error killed the relay.* `accept().await?` + propagated `EMFILE`/`ECONNABORTED` out of `run`. Now logged, backed + off, and continued. + - *No HTTP timeouts.* The comment claiming hyper's defaults sufficed was + backwards: `header_read_timeout` is inert unless a timer is installed, + so nothing was enforced. Now a timer plus a 30 s header timeout and a + 120 s body timeout. + - *Pushsync receipt addresses were never validated* — the one that + matters here, because §7's whole receipt-forwarding idea rests on it. + `receipt.address` was neither length-checked (callers + `copy_from_slice` it into `[u8; 32]`, so a missized address was a + remote panic) nor compared to the address actually pushed (so a peer + could store nothing and sign for a different address deep in its own + neighbourhood, and `is_shallow` would call it a perfect delivery). + Checked at the protocol boundary now, with regression tests. + +### Deferred / watchlist (cont.) + - Contiguous-arc lane assignment + deep pool specialization — only if receipt data shows forwarding depth is a real cost (§7). - WS/WT bindings of the frame format — only on demonstrated need (§4). diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md new file mode 100644 index 0000000..e5666ba --- /dev/null +++ b/docs/pusher-incentives-slides.md @@ -0,0 +1,174 @@ +--- +marp: true +theme: default +paginate: true +header: 'Paying for relay — an incentive layer for hoverfly pushers' +--- + +, part:, eyebrow:, hazard) are documented + in docs/deck/build.py. + + Keep slides sparse: a heading, ONE block (a short table, up to four + bullets, or two short paragraphs), and ONE callout. The detail lives in + docs/pusher-incentives.md. + + Headings name the mechanism and nothing else — "Trust model", not "Who + has to trust whom". The claim about that mechanism goes in the body, + where there is room to say it precisely. Eyebrows mark the act (the + problem / theory / practice), so they never restate the heading. --> + + + +# Paying for relay + +## An incentive layer for hoverfly pushers, reusing parts of SWAP + +*docs/pusher-incentives.md · one paid relay in production* + +--- + + + +# Unpaid relay bandwidth + +Upload directly and bee debits your own node for every chunk. Put a relay in the middle and that debt moves to it — the relay is the peer bee sees. Pseudosettle pays it in time rather than money, so what a relay actually spends is bandwidth, and nothing prices that. + +| | free | paid | +|---|---|---| +| client → relay | nothing | 4.8e8 PLUR per KiB | +| relay → bee | time, not money | unchanged | + +> A free tier hands a relay 70–100 GB of egress a month. Burn all of it under metering and it bills **$0.35–0.51**. + +--- + + + +# Trust model + +A relay is a standalone HTTP service — no registry, no list to get onto. Trust runs one way: the client checks a signed quote before sending a byte; the relay gets whoever shows up. + +> So every defence here points from the relay at the client. The other direction gets no cryptography, only bounds: the client computes its own bill, and risks at most one credit limit per lane — about **$0.0024**. + +--- + + + +# The billing unit + +``` +owed = KiB the relay accepted × price per KiB +``` + +> The client cannot lie about it. It produced the bytes; the relay counted them. + +Bytes admitted, not delivery receipts. Billing per receipt meant trusting a **third party's signature**, which took five mechanisms to make safe. Changing the unit removed all five. + +--- + + + +# Admission control + +The relay must accept or refuse **before** reading an upload — but at that moment it does not know whose account to check. + + +> Checking on every upload would mean 512 signature recoveries before it can answer at all. Cheap to attack, expensive to serve. + +So the chain lookups happen **once**, when a slip is issued, and the credit limit is sealed into it. The batch owner signs it, so stolen stamps cannot bill their owner. + +--- + + + +# The credit limit + +The cheapest live batch costs a fraction of a cent, so "owns a batch" proves nothing. The limit tracks what the batch is worth: + +``` +credit limit = min(batch's remaining value ÷ 1000, a global ceiling) +``` + +> An attacker gets back **at most a thousandth of what they funded**, and less once the ceiling binds. The ratio is what is fixed, so there is no cheap corner to aim at. + +--- + + + +# Settlement + +A cheque is a running total: "you have now paid me *this much in total*". + +- **Losing one costs nothing** — the next covers it +- **Old ones are worthless** — each must exceed the last +- **Gas is paid once per customer**, not per cheque + + +> Money set aside for an upload in progress must never be written to disk. No upload survives a restart, so nothing would ever release it. + +--- + + + +# Unit economics + +Relaying earns **$0.02 per GiB** admitted. On a host you already pay for, the only cost is cashout gas — 110k gas, a ten-billionth of a dollar. + +| | per month | +|---|---:| +| payload under a 2 TB egress cap | 503 GiB | +| billed at $0.02/GiB | **$10** | + +> So the ceiling is the allowance, not the cost — and only on bandwidth nobody bills by the gigabyte. Billed per GB, the same traffic costs **$0.36 per GiB** against $0.02 of revenue, and that relay should run free. + +--- + + + +# Deployment + +Paying is optional, and each relay sets its own mode. + +| Relay | Client can pay | Client cannot | +|---|---|---| +| free | nothing billed | nothing billed | +| paid, soft | billed, settles | billed, served anyway | +| paid, enforced | billed, settles | **dropped at startup** | + +> Four free relays run on hosts that wipe their disk on restart — and a relay that forgets what it is owed serves for free forever. The browser app skips the paid one: it can sign chunks, not cheques. + +--- + + + +# Bugs found in production + +| § | Bug | +|---|---| +| 17.1 | Carried-over debt could not be paid | +| 17.2 | Headroom measured one chunk, then sent a batch | +| 17.3 | A rule checked against the wrong number | +| 17.4 | A lane refused for bytes in flight was parked forever | +| 17.5 | One broken stream bounced every later cheque | +| 17.6 | The first upload was sized before the debt was known | + +> No test suite reached any of them, and none is reachable from one upload against a fresh relay. Between them they needed debt surviving restarts, several POSTs in flight, and a batch spent down far enough to bind. + +--- + + + +# Results + +Three runs of 2 MiB, 567/567 chunks delivered, no refusals, no rejected cheques, each settling to zero owed. Cheques cash from a separate machine — the relay must never hold that key. + +| | to date | +|---|---:| +| billed | $0.0003 | +| cashed on-chain | $0.00006 | +| paying clients | 1, and it was me | + +> The mechanism works; nobody is paying it. The browser dApp is the only real traffic and it signs stamps, not cheques. Nothing yet stops a relay billing for chunks it drops. diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md new file mode 100644 index 0000000..644b7d3 --- /dev/null +++ b/docs/pusher-incentives.md @@ -0,0 +1,1909 @@ +# Pusher incentives: paying for relay with SWAP cheques + +Status: **Stages 0 and 1 shipped (soft mode); Stages 2–3 outstanding.** A relay can be paid, but never +refuses: soft mode meters, reports, and accepts cheques, and 402 +enforcement is Stage 2. See §14 for exactly what is and is not in. This doc +specifies an +optional *metered* mode for `hoverfly pusher` in which a client pays the +relay for bandwidth with off-chain SWAP cheques. It is a companion to +`docs/pusher-design.md`, which stays the index for the pusher subsystem; +read §§1–7 there first. + +The short version: **both counterparties are hoverfly.** A client pays a +relay. Bee is not a party to the payment — we borrow SWAP's *contracts and +cheque format*, not its protocol. + +**This is a rewrite.** Three rounds of adversarial review went into the +previous version, and the third round's most important finding was +structural rather than local: the doc was building *two-sided* +cryptographic verification for a *one-sided* trust relationship. A client +**chooses and pins** its relay before sending a byte; a relay gets whoever +POSTs. Roughly half the old document defended the client against the +relay — a party it had already vetted — and it paid for that with a +forgeable billing unit, an unbounded residual, and a ship-blocking +measurement nobody had taken. + +This version points every defence in one direction — **the relay is what +gets protected, from the client** — and picks a billing unit the client +cannot lie about. The result is smaller, has no kill criterion, and closes +the attacks that involve actual money. §16 records the live bugs that +review found in open-mode code; they are the durable artifact of the +process and are all fixed. + +## 1. Why + +Today the relay eats a cost it did not incur. + +In a native upload the user's own machine opens the pushsync streams, and +bee debits *it* for every chunk — `price(po) = (32 − po) × 10 000` +accounting units (`bee/pkg/pricer/pricer.go:34-36`, mirrored at +`src/transport.rs:94-97`). Put a relay in the middle and that debt moves +wholesale to the relay: it is the peer bee sees, so it is the peer bee +charges. The browser client that caused the traffic pays nothing but +postage. + +`docs/pusher-design.md` §6 currently books this as an accepted risk — +*"worst case = the platform's free egress for the month (~70–100 GB) +burned, $0 lost"* — and §10 identifies the thing that is actually scarce: +a dedicated egress IP with an unthrottled dial budget. Free-tier lanes +starve at a fraction of a dedicated host's throughput, and the only way to +get more of them is for someone to volunteer. + +Metering is the answer to both. It puts the cost back on the party that +caused it, converts §6's accepted quota-drain risk into a priced one, and +makes running a dedicated-IP relay a rational act rather than a donation. + +Nobody gets rich. §9's economics are thin: at $0.02/GiB and §9.1's 3.7× +egress, a relay saturating a 2 TB/month bandwidth allowance bills about +**$10**. On hardware whose cost is already sunk — a box running other +things, bandwidth already included — that is close to all margin, since +§9.3's gas turns out to be negligible and the *client* funds the +chequebook, not the relay. But the allowance is a ceiling and crossing it +inverts the economics in one step (§9.2), so what this funds is a +self-sustaining lane federation, not a business. + +## 2. Trust model — read this before anything else + +Every design decision below follows from one asymmetry. + +> **The client chose its relay. The relay did not choose its client.** +> +> A client verifies the signed quote and pins +> `(url, node_eth_address, beneficiary)` (§7.3) before it sends a byte. A +> relay is whatever URL it was pointed at, and a client is whoever POSTs. +> Defences point *from* the relay *at* the client. The security goal is: +> **a client cannot obtain relay service without paying for it, and cannot +> lie to the relay about what it owes.** + +**The asymmetry is pinning, not curation, and the difference is not +cosmetic.** A relay is a plain HTTP service: there is no registry, no +discovery mechanism, and no list anyone has to be admitted to. Anyone can +run `hoverfly pusher`, and any client can point `--pusher` at any URL. +`PUSHER_URLS` is one client's default fleet — the dApp's — not a +federation roster. + +So "relays are known" is only true of a relay a given client has already +chosen. The property the design can actually lean on is that **a client +only ever pays a relay it configured and pinned**, which is a decision it +made with the signed quote in hand. + +The inverse — a relay defrauding a client — is still **out of scope for +cryptographic treatment**, but it does not follow that the client is +unprotected. Four bounds hold without the relay being trustworthy: + +- **The client computes its own bill** from bytes it sent (§8.4), so an + over-reported `kib_admitted` is immediately visible and attributable. +- **The price is pinned** from a signed quote and echoed verbatim in every + 402 (§7.3), so it cannot move under the client. +- **Exposure per lane is capped at the credit line** — `max_outstanding` + at most, and `remaining_value ÷ credit_ratio` for a small batch. +- **Outcomes are measured.** A lane that takes bytes and delivers poorly is + deweighted by the scheduler's existing EWMA. + +The one place a relay's own assertion could exceed that cap is §17.1's +reconcile, where the client adopts the relay's `owed`. It is bounded by +the ceiling in the relay's *signed quote* precisely so that this list stays +true; see `LanePayer::check_reported_debt`. + +Three consequences worth stating so they don't get re-litigated: + +**No confidentiality is claimed or implied.** The pusher sees plaintext +chunks, the stamp, and the batch-owner address. That is inherent to +relaying, and Swarm is not an anonymity system in the first place — chunks +carry a signed postage stamp, addresses are content-derived and publicly +retrievable, and retrieval hands the stamp back +(`src/protocols/retrieval.rs:27`, `:42-80`). Metering changes none of +this. The one operational consequence: **a relay holds every stamp it has +ever relayed**, which makes it the richest possible source of harvestable +stamps and is an argument for §7.2's origin-bound challenge. + +**The client is not asked to verify the relay's work cryptographically.** +It verifies *arithmetic* — it knows exactly how many bytes it sent — and +it measures *outcomes*, which `src/pushsched.rs` already does by +deweighting lanes that underperform. That is the same protection open mode +has today, and open mode works. + +**Receipts are telemetry, not evidence.** The relay forwards the pushsync +receipt in the ack because it is genuinely useful — it is the first +cryptographic signal a relay client has ever had about where its chunk +landed, and it feeds lane weighting — but **it does not enter the +invoice.** An earlier draft billed per verified receipt, which required +anchoring each receipt in the staking registry to stop forgery, and still +left a replay hole that no off-chain check could close. §8 explains why +the billing unit moved. + +## 3. Scope and non-goals + +**In scope:** a client (native CLI first, browser later) pays a +`hoverfly pusher` for relayed bytes, over the existing HTTPS channel, +denominated in BZZ. + +Explicit non-goals, each with its reason: + +- **Not bee's swap protocol.** No `/swarm/swap/1.0.0/swap` stream, no + `Handshake`/`EmitCheque` framing, no `exchange`/`deduction` headers, no + priceoracle. Client and relay already speak HTTP; there is nothing to + gain from tunnelling a libp2p settlement protocol through it. We reuse + the *cheque* (§4), not the transport. +- **Not bee's accounting model.** No `paymentTolerance`, no ghost + balances, no trust ramp, no blocklist state machine. But note §10.2: + bee's *reservation* concept is mandatory and an early draft wrongly + dropped it. +- **Not "the relay uses the income to pay bees."** `PERFORMANCE.md` + measured paid at 160 KiB/s against unpaid at 195 KiB/s ("verdict: not + confirmed at this workload"), and the arithmetic agrees: bee grants + 4 500 000 accounting units/s per peer via pseudosettle, ≈ 18 chunks/s at + PO 8, ≈ 2 400 chunks/s across a 128-session pool — against ~150 chunks/s + actually measured. The relay is session- and RTT-bound, not + credit-bound, so buying credit buys nothing. Relay→bee settlement stays + free pseudosettle. (The pusher does not even wire `SwapConfig` today — + `build_push_state` (`src/pusher.rs:724-772`) never calls `.with_swap`, + so the global `--chequebook` flag has no effect on a relay.) +- **Not confidentiality, not anonymity.** §2. +- **Not defence against a malicious relay.** §2, §14. +- **Not encrypted uploads.** 64-byte references are unsupported throughout + (`src/feed.rs:150`; the erasure coder is non-encrypted-path only at + `src/erasure/mod.rs:145`, `:365`, `:408`) and adding a key-management + surface is out of proportion to the problem. +- **Not a new contract.** See §13. + +## 4. What we borrow from SWAP, and what we drop + +| Borrowed | Where it comes from | Why | +|---|---|---| +| `ERC20SimpleSwap` chequebook + canonical factory | `bee/pkg/config/chain.go:89` (Gnosis `0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420`), `chain.go:66` (Sepolia `0x0fF044F6bB4F684a5A149B46D7eC03ea659F98A1`) | Audited, deployed, in production. Nothing to write. | +| EIP-712 cheque | `bee/pkg/settlement/swap/chequebook/cheque.go:32-38` | Domain `{name:"Chequebook", version:"1.0", chainId}` — no `verifyingContract`, no `salt`. Type `Cheque(address chequebook, address beneficiary, uint256 cumulativePayout)`. Already implemented at `src/signer.rs:309-337`. | +| Cumulative-payout monotonicity | `chequestore.go:133-138` (`ErrChequeNotIncreasing`) | Makes cheques loss-tolerant and replay-proof (§8.3). | +| Funding check | `chequestore.go:172-179` (`ErrBouncingCheque`) | But **use `liquidBalanceFor(us)`, not `balance()` — bee's version is unsound**, see §11.6. | +| Reservation against concurrent issuance | `chequebook.go:163-178` (`reserveTotalIssued`) | Needed on both sides. See §10.2. | +| Payee-only role | `bee/pkg/node/node.go:592-625`, `bee/pkg/node/chain.go:206` | The beneficiary is a plain **EOA**, not a chequebook. A payee needs no contract of its own. | +| Cheque JSON encoding | `src/protocols/swap.rs:84-106` (`encode_signed_cheque_json`) | Byte-exact Go `encoding/json` compatibility already solved. | + +| Dropped | Reason | +|---|---| +| swap libp2p stream, `Handshake`, `EmitCheque` framing | We're on HTTP. | +| priceoracle, `exchange`, `deduction` | Relay quotes PLUR directly. No oracle dependency, no new-peer ramp. | +| accounting-unit indirection | Prices are PLUR. One unit, no conversion. | +| `ErrChequeValueTooLow` as inherited | Bee's floor is "≥ 1 accounting credit" (`chequestore.go:37-38`), denominated in the oracle indirection we dropped. Our `min_cheque_plur` (§10.1) is a *different* mechanism with a different justification; do not treat bee as precedent for its sizing. | +| ghost/shadow balances, tolerance, trust ramp | No analogue in request/response. | +| `StakeRegistry` / staked-set snapshot | An earlier draft needed this to make forged pushsync receipts expensive. With receipts out of the invoice (§8) there is nothing to anchor, and Stage 1 loses its two largest work items. | + +## 5. Relay modes + +A relay runs in exactly one of two modes, advertised in `/v1/status`: + +- **`open`** — today's behaviour, unmetered. The four free-tier lanes in + `apps/upload/src/config.ts` keep running this. Auth stays "stamp signer is + the live batch's on-chain owner" and nothing is billed. +- **`metered`** — every byte admitted is billed (§8). There is no free + allowance. `pusher.browserbzz.link` is the first production lane running + it, with hard enforcement. + +**Mode is per relay, and paying is optional for the client.** A fleet mixes +freely: the same `PUSHER_URLS` holds four `open` lanes and one hard-metered +one, and each client uses the subset it can be served by. The rule is +symmetric on both sides of the wire: + +| relay mode | client has a chequebook | client does not | +| --- | --- | --- | +| `open` | used, nothing billed | used, nothing billed | +| `metered`, soft | used, billed, settles | used, billed, served anyway | +| `metered`, hard | used, billed, settles | **lane retired at startup** | + +Retiring is the load-bearing case. A hard lane answers a push without a +challenge header with 401, and a 401 is not a 402 — it counts against lane +health, so scheduling one anyway spends a retry per chunk rediscovering +something the lane advertised in `/v1/status` before the first byte moved. +Both drivers therefore drop it up front: `src/client.rs` (native) when no +`--chequebook` is configured, and `UploadSession::set_lane_status` +(`src/wasm.rs`) unconditionally, since the browser only stamps and the +chequebook lives in the native client. Soft-metered lanes are *kept* by +both — they bill and serve, so an unpaying client is served exactly as on +`open`. A run whose every lane is hard-metered fails immediately with that +reason rather than stalling. + +Four further consequences worth stating up front: + +**Metering subsumes the deferred `--push-quota`.** Design §6 proposed +capping each batch at its own effective volume per TTL. Under metering the +price *is* the quota, and a better one: it scales continuously instead of +cliff-edging at a volume boundary, and it doesn't require the relay to +model batch TTL semantics. `--push-quota` should be struck from the +watchlist if metered mode ships. + +**Metering makes `--push-challenge` mandatory, not optional** — and it +does double duty as the admission mechanism (§7.2, §11.1). + +**Metered mode requires durable storage.** Not a recommendation. §11.4 +shows that an ephemeral filesystem turns one signature into unlimited free +service. Free-tier hosts with ephemeral disks (Render free, the reference +deployment in design §11) **must** run `open`. + +**Metered mode requires a chain RPC.** Open mode already needs one for +batch-owner resolution; metering adds batch-value reads (§10.3) and cheque +verification (§11.6). Only on the relay — the client needs no chain access +to compute its own bill, which is the point of §8. + +## 6. Identity: the account is the batch owner, the credit line is the batch + +The relay's existing auth already establishes a strong, on-chain identity +for every push (`src/pusher.rs:895-960`): the stamp signature must recover +to the address that the `PostageStamp` contract reports as the batch's +owner, and the batch must be alive. That recovered address is the natural +account key. + +> **Account = the batch-owner EOA.** Cheques, the cumulative and the +> settlement ledger are keyed on it. +> **A cheque is valid for that account iff its chequebook's on-chain +> `issuer()` equals the same EOA.** +> **Credit is keyed one level finer, on the *batch*.** Standing (below), +> the credit line (§10.3) and admission (§7.2) are all per batch, because +> one owner can hold many batches of wildly different value and the +> cheapest of them must not buy the credit of the dearest. + +This binds payment to identity cryptographically, with no session tokens, +no registration step, and no extra protocol message. It lines up with the +CLI's existing semantics: `--chequebook`'s documented precondition is +already *"`issuer()` == `--key`'s address"* (`src/bin/hoverfly.rs:90-105`), +and `--key` is the stamp key. And in a browser it means the session key — +which already owns the batch — is the chequebook issuer, so cheques sign +with **zero wallet prompts**. + +**One *batch* per request.** `run_push` currently re-resolves the owner +whenever a frame's batch id changes (`src/pusher.rs:905-925`), so a single +POST can legally mix batches and owners. Metered mode must forbid this: +standing, the credit line and the reservation are all properties of a +*batch*, not of an owner who may hold many. Every frame in a POST must +carry the exact batch id named in the challenge, and a mismatched frame is +rejected rather than billed to whoever it names. Without this the +admission check in §7.2 is trivially bypassed by prefixing one frame from +a good-standing batch (§11.8). + +Tightening account → batch costs nothing and buys three alignments: the +recent-ack cache is already keyed on `(addr, batch_id)` (§16.1), admission +resolves standing once per POST instead of once per frame, and the +reservation has exactly one credit line to check against. A client mixing +batches splits them across POSTs, which the scheduler already pipelines. + +**The relay never holds spendable key material.** It needs the +beneficiary's *address* only. `cashChequeBeneficiary(recipient, cumulativePayout, signature)` +must be called *by* the beneficiary (`cashout.go:137`, and +`_cashChequeInternal` takes `msg.sender` as the beneficiary), so cashing +out happens later, elsewhere, from a machine that does hold the key. A +relay box holds nothing worth stealing — the property that makes today's +pusher safe (design §1, "not a signer") survives metering intact. The +relay *does* hold its node-identity key (`HOVERFLY_PUSHER_IDENTITY`), +which is not spendable and is used to sign quotes and challenges (§7.3). + +**The factory check is security, not compatibility.** The relay must +verify `factory.deployedContracts(chequebook) == true` +(`bee/pkg/settlement/swap/chequebook/factory.go:101-118`) against a +factory address **hardcoded per chain**, never one supplied by the client. +Skip it and a client presents an arbitrary contract that returns a forged +`issuer()` and `liquidBalanceFor()` and implements `cashChequeBeneficiary` +as a no-op — total compromise for one `eth_call` saved. Optionally also +check the deployed bytecode hash against `AcceptedChequebookBytecodeHashes` +(`bee/pkg/config/chain.go:96-98`). + +**Liveness is not enough: metering needs batch *standing*.** Open-mode +code now caches `batch_id → owner` through a bounded `OwnerCache` with +separate TTLs for successes and definitive rejections +(`src/pusher.rs:181-239`, `:1106`) — an amplification fix (§16.2) that +happens to give metering a staleness bound. Two gaps remain. + +The first is that the TTL is tuned for amplification, not standing. +`OWNER_OK_TTL_SECS = 1800` bounds how stale an aliveness answer can be; +§10.3's Sybil bound needs the *credit line* re-read on the same schedule. +Metered mode extends the cache entry rather than adding a second cache. + +The second is that liveness is a *boolean* and the thing it stands in for +is a *quantity*. "Alive" is satisfied by the cheapest batch that clears +the contract's minimum — minimum depth, minimum validity — which costs a +fraction of a cent. So metered mode reads standing, not liveness: + +``` +standing(B) = (owner, depth, remaining_value_plur) +remaining_value_plur = remainingBalance(B) × 2^depth +``` + +Both reads already exist — `read_batch` returns owner and depth +(`src/batch.rs:248`), `read_remaining_balance` returns PLUR per chunk +still funded (`:288`) — so this is one extra `eth_call` over what open +mode already does, cached per batch under the same TTL. §10.3 turns +`remaining_value_plur` into that account's credit line, which is what +makes the Sybil bound hold by construction rather than by assumption. + +Known limitation: one account = one batch owner = one chequebook. A client +uploading under several batches with distinct owners needs a chequebook +per owner. A signed authorization linking extra batch owners to one +chequebook is possible but deferred (§15). + +## 7. Wire protocol + +Push frames (`src/pushframe.rs`) are **unchanged**. Payment is +out-of-band, for the same reason bee keeps swap on a separate stream from +pushsync: it must not sit on the hot path, and a payment failure must not +fail a push. + +| Endpoint | Shape | +|---|---| +| `GET /v1/status` | new `payment` block, signed (§7.3); `mode: "open" \| "metered"` | +| `GET /v1/challenge?account=&batch=` | `{nonce, expires_ms, max_outstanding_plur}` — stateless MAC, issued only to a batch in good standing (§7.2) | +| `GET /v1/account` | requires the challenge header. `{owed_plur, reserved_plur, outstanding_plur, kib_admitted, kib_dedup, cumulative_received_plur, settle_every_plur, max_outstanding_plur}` | +| `POST /v1/pay` | requires the challenge header; body = `SignedCheque` JSON → `{accepted_plur, cumulative, outstanding_plur}` | +| `POST /v1/push` | requires the challenge header; `402 Payment Required` when over cap | + +`/v1/account` is authenticated because unauthenticated it is a +per-identity volume oracle over on-chain-enumerable batch owners, and a +targeting oracle for tipping a victim into 402 at a chosen moment. + +### 7.1 Rollout + +Two-phase. *Soft mode* first: the relay meters, accepts cheques, and +reports `owed`, but **never answers 402** — an account over its credit line +is recorded and served anyway. + +**Soft mode still requires the challenge.** An earlier draft of this +section said existing clients "keep working", implying an unchallenged +request should be served. That is wrong, and it was caught while trying to +enable metering on a live lane: if a missing header meant "serve for free", +metering would be bypassable by *omitting a header*, which is not a +degraded mode but no mode at all. What soft mode drops is enforcement of +the cap, not authentication. A relay flipping to `--meter` therefore does +break clients that predate the protocol — acceptable, because the only +dApp using these lanes ships alongside them. + +Only once clients in the wild can pay does a relay flip to hard mode. + +Soft mode is an *instrument*, not a migration path for clients: the `done` +line lands after the whole batch (`src/pusher.rs:1090`), so a client +cannot use it to pace itself. Its job is to tell the operator what §14 +Stage 0 needs to know. + +### 7.2 Admission — the challenge carries account, batch and credit line + +An early draft claimed 402 was easy because `/v1/push` spawns its body +(`src/pusher.rs:835`), so the status is committed before any chunk is +processed. That is true and it is the *problem*: at that moment the relay +does not yet know whose account to check. The account only exists after +`stamp::validate` (`:900`) and `resolve_owner` (`:912`, two `eth_call`s on +a miss), both inside the spawned task. Hoisting them means up to 512 +ecrecovers (~40 ms) plus a possible RPC round-trip synchronously in front +of every response — the exact unauthenticated amplification surface §11.6 +spends a section defending, and instantly over budget on the CF Workers +profile (design §9: 10 ms CPU/req). + +**The challenge solves this by moving the chain reads off the POST path +entirely.** Standing (§6) is resolved once when the challenge is *issued*, +and the resulting credit line is baked into the nonce. `/v1/push` +admission then reads no chain state at all: + +``` +GET /v1/challenge?account=A&batch=B + → resolve standing(B) (cached per batch, TTL; §6) + → require owner(B) == A, else 403 — no nonce is issued + → cap = credit_line(standing(B)) (§10.3) + → nonce = HMAC(relay_secret, preimage(A, B, origin, expiry, cap)) + → {nonce, expires_ms, max_outstanding_plur: cap} +``` + +**The MAC preimage is fixed-width and domain-tagged, not a +concatenation.** `origin` is variable-length, so a bare concatenation +makes `("host.a", "bc")` and `("host.ab", "c")` share a preimage, and a +relay serving several hostnames would issue one nonce valid for two of +them. In bytes rather than a typed struct, because only the relay ever +parses it: + +``` +preimage = "hoverfly-pusher-challenge-v1" // domain tag, fixed 28 B + ‖ A (20 B) + ‖ B (32 B) + ‖ expiry_be ( 8 B) + ‖ cap_be (16 B) + ‖ len(origin) ( 2 B, big-endian) + ‖ origin (variable, last) +``` + +Two operational rules come with it. Compare the MAC in **constant time** — +it is the only secret in the exchange and the client controls every other +field, so a byte-wise early exit is a forgery oracle. And **persist +`relay_secret` alongside the ledger** (§11.4): a secret regenerated at +boot invalidates every outstanding challenge, which on a host that sleeps +and cold-starts (design §7 measured a 35.2 s wake) turns every restart +into a 403 storm for clients mid-upload. + +The nonce is therefore a **capability**: possessing one is proof the relay +already checked standing and already priced the credit line. Admission +becomes: + +1. Verify the MAC over `preimage(A, B, origin, expiry, cap)` — symmetric, + no RPC, constant-time compare. +2. Verify `origin` against the relay's **configured** hostname — never + against the request's `Host` or `X-Forwarded-Host` (see below). +3. Verify the client's signature over the challenge struct → recovers `A` + (1 ecrecover, no RPC, no body). +4. `reserve = ceil(Content-Length / 1024) × price_plur_per_kib`; if + `outstanding(A) + reserve > cap` → **402**, before reading the body. +5. Otherwise commit the reservation atomically, return 200, spawn. +6. On completion, convert reservation → `owed` for bytes actually admitted + (§8) and release the remainder. + +**`origin` must be configured, not derived.** This is the single +easiest-to-get-wrong line in the design. The obvious implementation reads +the `Host` header and compares the challenge's `origin` against it — and +that is a no-op, because `Host` is *supplied by the same client that +supplies the challenge*. An attacker replaying a victim's signature at +relay B simply sends `Host: relay-b.example`, the comparison passes, and +§11.1's cross-relay replay is restored in full while the doc claims it is +closed. `X-Forwarded-Host` is worse: attacker-set on any relay not behind +a proxy that overwrites it, which the reference deployment is not. So the +relay takes its hostname from a required `--origin` flag and compares +against that constant. A relay reachable at several hostnames configures +the list; a relay that cannot state its own origin must not run metered. + +**The reservation is exactly the billing unit.** Because §8 bills bytes, +the quantity reserved at admission and the quantity billed at completion +are the same thing, computed the same way — `Content-Length` before, bytes +counted after. No estimation, no conversion, no over-reserve. (An earlier +draft reserved a flat `PUSH_BATCH_MAX × price`, which was ~10× a dust +batch's entire credit line and 402'd every POST from it regardless of +size.) A request with no `Content-Length` is refused; the existing client +always sets one, and `PUSH_MAX_BODY` (`src/pusher.rs:64`, ≈ 2.08 MiB) +caps it anyway. + +**Small batches are the client's job to size.** A 0.01 BZZ dust batch gets +~208 KiB of credit (§10.3), so it cannot push a full 512-frame POST +(~2 125 KiB) in one go. The challenge returns `max_outstanding_plur`, so +the client knows its ceiling before it builds the request and simply sends +smaller POSTs. This is a normal flow-control interaction, not a failure. + +Issuing the challenge only to a batch in good standing also closes a hole +an earlier draft left open: admission previously granted a reservation to +*any* EOA that could sign, since standing was established per frame inside +the spawned task. Free identities could occupy reservation-ledger entries +without ever owning a batch. Now they cannot obtain a nonce at all. + +**Nonces are stateless.** No server-side table, so no unbounded-memory DoS +from a free `GET /v1/challenge`, and no per-POST round trip that would +serialize the pipeline the scheduler exists to exploit (`inflight_max` up +to 8, `src/pusher.rs:517`). Replay within the window is bounded by the +reservation, not by nonce uniqueness. Two limits follow from making the +challenge endpoint do chain work: + +- **Rate-limit and cache `/v1/challenge`.** Amplification is bounded by + *distinct batch ids*, not request count, so cache standing per batch + under the §6 TTL and negative-cache unknown batches. Per-IP limiting on + top (§11.6's inbound limiter). +- **Bound live-reservation cardinality.** The in-memory + `account → reserved` map is attacker-influenced (one entry per batch in + standing), so cap the number of accounts holding live reservations and + shed beyond it. Persist a row only once `owed > 0`. + +**The challenge is EIP-712, not a concatenation.** The account key already +signs postage stamps (EIP-191 over stamp bytes) and cheques (EIP-712, +domain `Chequebook`). A third raw-bytes scheme over the same key invites +cross-scheme confusion. Use a typed struct with its own domain: + +```solidity +// domain: {name: "HoverflyPusher", version: "1", chainId} +struct PushChallenge { + bytes32 nonce; + string origin; // host the client dialled; relay checks its own + address account; + bytes32 batchId; + uint256 expiry; +} +``` + +`quote_valid_secs` governs the quote; the challenge gets its own, much +shorter `challenge_ttl_secs` (~300 s), since re-signing is one local +ecrecover's worth of work and a short window shrinks the replay surface to +near nothing. + +**Browser blocker:** the CORS preflight currently allows exactly one +request header, `content-type` (`src/pusher.rs:468-486`, header list at +`:481`). A custom challenge header fails preflight in every browser until +that list grows. Push *frames* are unchanged, but the challenge is still a +wire change. + +### 7.3 The quote is signed + +`/v1/status` is unsigned JSON today (`src/pusher.rs:488-528`). An unsigned +price is repudiable in both directions: the relay can serve `P` and bill +`10P`, the client can claim it saw `P/10`, and reconciliation can detect +the mismatch but never attribute it. So the `payment` block is signed with +the node-identity key — which the relay already holds and already +publishes as `overlay` (`:499`) — and the signed blob is echoed verbatim +in every 402. + +**The pin is on the node's Ethereum address, not on its overlay.** An +overlay is `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`; verifying a +signature yields the **eth address**, and the nonce is neither transmitted +nor derivable, so "pin `(url, overlay)`" is not implementable — the +recovered address and the pinned overlay are values in different spaces. +The signed block therefore carries `node_eth_address` and `overlay_nonce`, +so any client can recompute `overlay` and check it against what +`/v1/status` advertises, and clients pin +**`(url, node_eth_address, beneficiary)`**. A client already names its +relays somewhere — `PUSHER_URLS` in the dApp, `--pusher` on the CLI — so +carrying two more fields alongside a URL it already had costs nothing. + +```jsonc +// GET /v1/status → new field (the whole object is covered by `sig`) +"payment": { + "mode": "metered", + "beneficiary": "0x…", // EOA that must appear in Cheque.beneficiary + "node_eth_address": "0x…", // recovers from `sig`; client pins this + "overlay_nonce": "0x…", // 32 B; lets the client recompute `overlay` + "origin": "relay-a.example", // must equal the configured --origin (§7.2) + "chain_id": 100, + "factory": "0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420", + "price_plur_per_kib": "480000000", + "min_cheque_plur": "3900000000000", + "settle_every_plur": "15600000000000", + "max_outstanding_plur": "62200000000000", // ceiling; actual cap is per + // batch, see §10.3 + "credit_ratio": 1000, // credit line = batch value ÷ this + "quote_valid_secs": 86400, + "challenge_ttl_secs": 300, + "sig": "0x…" // node-identity key over the above +} +``` + +Parameters are derived in §9 and §10.1; the ordering +`min_cheque < settle_every < max_outstanding` is a hard invariant (§10.1). + +## 8. The billing unit: bytes admitted + +> **`owed = (kib_admitted − kib_dedup) × price_plur_per_kib`**, where +> `kib_admitted` is the body bytes the relay accepted under a valid +> challenge, rounded up to KiB. + +That is the whole billing rule. It has one property that matters more than +everything else in this document: + +> **The client cannot lie about it, because the client is the one who +> produced the bytes and the relay is the one who counted them.** + +There is no third-party attestation to forge, no signature to replay, no +chain state to disagree about, and no relay assertion the client has to +take on faith. The client committed to `Content-Length` when it built the +request; the relay counted what arrived. Both numbers are known to both +parties before any push work happens, and they must agree or the request +was malformed. + +Compare what this replaces. An earlier draft billed per *verified pushsync +receipt*, which forced a five-clause predicate anchored in the staking +registry (to stop trivial receipt forgery), a set-valued invoice (to stop +self-replay), a sampled retrieval audit (to catch fabrication), a staked-set +log sweep on both sides, and a shared predicate module that had to be +byte-identical between client and relay or it would generate disputes with +no adjudicator. All of that existed to make a *third party's* signature +into a billing input. Removing the receipt from the invoice removes the +entire apparatus and every attack against it. + +### 8.1 Why bytes rather than successful pushes + +Because bytes are what the relay spends money on. §9.1's cost basis is +egress, and egress is incurred on *attempts* — the 3-way peer race and the +shallow retries happen whether or not a chunk ultimately lands. Billing +successes would mean the relay eats the cost of every failure, which is +what made an earlier draft's "shallow-cascade arbitrage" (§11.5) an +unresolvable tension between verifiable billing and cost recovery. Billing +attempts dissolves it: the relay charges for what it spends, and the +*client* protects itself against a lane that spends without succeeding by +deweighting it in the scheduler, which is a mechanism that already exists +and already works. + +Two mechanisms, each doing what it is good at, instead of one mechanism +trying to do both and failing at the second. + +### 8.2 Dedup hits are billed at zero + +A frame served from the recent-ack cache (`src/pusher.rs:928-948`) does no +push work, so its bytes are subtracted. The ack already reports dedup, and +the `done` line carries the count (`:969-971`), so the client can +reconstruct the invoice exactly. + +This is the one place a relay assertion enters the bill — the relay claims +"this was a dedup hit". It is safe because the claim only ever *lowers* +the amount owed, so a relay has no incentive to make it falsely, and a +client that disagrees is disagreeing in its own favour. + +### 8.3 Cumulative cheques, and why not per-chunk + +A cheque is a cumulative running total for one `(chequebook, beneficiary)` +pair. Three properties fall out, all of which we want: + +- **Loss-tolerant.** A dropped or failed `/v1/pay` costs nothing; the next + cheque supersedes it. No retry state machine. +- **Replay-proof within a live relay.** Strict monotonicity means a + re-presented cheque credits zero. (Across a *restart*, see §11.4.) +- **Cheap, and gas-amortizing by construction.** One signature per + `settle_every_plur`, and the relay cashes only the *latest* cumulative, + so on-chain gas is paid once per account, not once per cheque. This is + why the dust floor (§10.1) is about RPC cost, not gas. + +**Attaching cheques to chunks or frames is rejected.** ~137 bytes per +4 KiB chunk, an EIP-712 signature on the per-chunk hot path, and — fatally +— cumulative payouts are *serial* per `(issuer, beneficiary)` pair, so +per-chunk cheques would force a total order on chunks within a lane, +destroying the concurrent multi-POST pipelining the scheduler depends on +(design §7). + +**Two lanes may share one beneficiary, and the client must handle it.** A +cumulative is per `(chequebook, beneficiary)` — the beneficiary is what +the contract keys `paidOut` on — while a *lane* is a URL, and one operator +running four lane URLs behind one beneficiary EOA is the obvious +deployment. If the client tracks cumulatives per lane, that configuration +**bricks**: `src/cheques.rs` keys `payouts` on the peer *overlay*, with a +comment explaining that the overlay is the only stable cross-run identity +(`:64-76`) — correct for bee peers, wrong for relay beneficiaries. Lane 1 +issues cumulative 10; lane 2, counting from its own zero, issues 8; the +relay applies `ErrChequeNotIncreasing` and rejects it, forever. + +> **Key the client's cumulative store on `(chequebook, beneficiary)`, not +> on lane or overlay.** Two lanes advertising one beneficiary are one +> settlement channel; sum their `owed` before signing, and send the same +> cheque to both. + +Detecting the sharing is free — the beneficiary is in the signed quote +(§7.3), so the client sees it before its first push. + +Independently, lanes with *distinct* beneficiaries still **share one +chequebook balance**, so the client must track the sum. `src/cheques.rs` +needs a `total_issued` mirroring bee's `reserveTotalIssued` +(`chequebook.go:163-178`). Without it a cheque to the second lane silently +exceeds the balance and bounces. + +### 8.4 Reconciliation + +`GET /v1/account` returns `kib_admitted` and `kib_dedup`. The client +compares them against what it sent. Because the client's byte count comes +from its own request construction and not from anything the relay returns, +this is immune to the failure mode that made an earlier draft's +reconciliation useless: ack sends are fire-and-forget +(`src/pusher.rs:855-860`), so a client that hangs up mid-stream legitimately +receives fewer acks than the relay emitted — but it still knows exactly how +many bytes it sent. There is nothing to dispute. + +A relay over-reporting `kib_admitted` is therefore immediately visible and +attributable. The client's response is to withhold the next cheque and +deweight the lane. That is not social enforcement — it needs no operator, +no list and no reputation — it is simply the client declining to sign for +a number it can check itself. What it costs the client to detect the +disagreement is bounded by the credit line, since nothing above that can +be admitted before the next settlement. + +## 9. Pricing + +### 9.1 Cost basis + +Design §9's *"egress ≈ payload × 1.4× at race=1; in-pusher racing is off +by design"* is wrong on its premise. **In-pusher racing is on:** +`CHUNK_PEER_PARALLELISM = 3` (`src/client.rs:4086`), and three peers are +seeded concurrently at dispatch, each writing a full Delivery before any +receipt is read. Design §7 says so in its own words — *"the deliberate +3-way peer race"*, measured at 2.2 PO of receipt depth for 2–3× +throughput. + +Real cost per 4 KiB chunk relayed: + +| | | +|---|---| +| Delivery on the wire | addr 32 + stamp 113 + span/data ≤ 4104 ≈ 4249 B, +~5 % protobuf/yamux/noise/TCP ≈ **4.4 KiB** | +| Peer race | **×3** | +| Shallow retries at pool 128 | **×1.15** (design §10: 0.44 shallow/chunk) | +| **Egress per chunk relayed** | **≈ 15 KiB** | +| **Egress per GiB of payload** | **≈ 3.7 GiB** (262 144 chunks/GiB) | + +**The Stage 0 counter does not check this, and currently cannot.** +`/v1/meter` → `egress.attempts_per_frame` exists to test the ×3.45 model +against reality, and the production relay reports **1.077**. That is not a +refutation of the model, it is an instrument fault. `PUSH_OUTCOME_*` is +incremented *after* the await inside the racing future +(`src/client.rs:5818-5870`), and the dispatcher cancels the losing racers +the moment it accepts a receipt — `src/client.rs:4806-4809` says so in as +many words. A cancelled racer has already written its Delivery to the +wire, so the relay pays that egress, and then the future is dropped before +it ever reaches the counter. + +The relay's own diagnostics show the shape plainly: `ok = 3756` against +`frames_admitted = 3756` — exactly one counted success per chunk, with +only 285 shallow retries and 4 errors above it. A metric that counts +completions cannot see a race it loses on purpose, so it floors near 1.0 +by construction. + +Counting at *dispatch* (where `inflight_pushes` is already incremented, +`src/client.rs:4818-4820`) rather than at completion would fix it. Until +then §9.1's model stands unverified in both directions, and **nothing here +should be repriced on `attempts_per_frame`** — which was the one job Stage +0 was supposed to do for §9.2. + +### 9.2 Price + +| | per GiB of payload | +|---|---| +| Cheap VPS bandwidth (~€1/TB) | ≈ €0.0037 | +| AWS egress ($0.09/GB) | ≈ $0.36 | +| **Suggested price** | **$0.02** | +| Postage (buys a *year* of storage) | orders of magnitude more | + +$0.02/GiB is ~5× a VPS's raw bandwidth cost — a real margin covering CPU, +the dedicated IP, and gas — and ~18× cheaper than AWS egress. Against what +the user already paid for postage it is a rounding error. + +**Read that AWS row as a constraint, not a favourable comparison.** At +§9.1's 3.7 GiB of real egress per GiB of payload, per-GB-billed clouds +cost the relay ~$0.36/GiB against $0.02 of revenue. Metered mode is only +rational on **flat-rate or included bandwidth** — the same class of host +§5 already requires for durable storage, and the same class §1 says +metering exists to fund. A relay on metered egress should run `open` and +eat the quota, or not run at all. + +At $0.40/BZZ: `$0.02 / $0.40 = 0.05 BZZ/GiB`; `0.05 × 10¹⁶ = 5e14` PLUR +per GiB, ÷ 1 048 576 KiB = + +> **`price_plur_per_kib ≈ 4.8 × 10⁸`** (1 BZZ = 10¹⁶ PLUR). + +A full push frame is `HEADER_LEN(147) + wire(4104) = 4251` B ≈ 4.15 KiB, +so a 4 KiB chunk costs ≈ `2.0 × 10⁹` PLUR all in. Framing overhead is +billed because the relay received those bytes. + +Flat per KiB, deliberately. Any pricing curve steeper than flat +re-introduces a per-item number for the two sides to disagree about, and +the entire point of §8 is that there is exactly one number and both +parties measure it directly. + +### 9.3 Gas, and who is actually profitable + +Cashout's gas *limit* is `GetGasLimitWithDefault(ctx, 300_000)` +(`cashout.go:145`), but a limit is not a spend. The two real +`cashChequeBeneficiary` calls on Gnosis mainnet used **75 378** and +**109 590** gas at **169** and **1 292** wei/gas — fees of `1.3e-11` and +`1.4e-10` xDAI: + +| | gas used | gas price | fee | +|---|---:|---:|---:| +| 2026-08-07 | 109 590 | 1 292 wei | 1.4e-10 xDAI | +| 2026-08-08 | 75 378 | 169 wei | 1.3e-11 xDAI | + +Both are `cashChequeBeneficiary` on chequebook +`0x17c89DE40f5ec07343AB095bfDa9dE1A5c095Fc1` (txs `0xb33bd199…` and +`0x9d4a8b29…`), settling `1.46e12` PLUR in total to the relay beneficiary. + +An earlier draft of this section carried **$0.0005**, assuming the full +300 k limit at gwei-scale prices. Gnosis base fee during these runs was +50–1 300 wei, so that estimate was high by roughly **six orders of +magnitude**. Issuing a cheque still costs nothing +(`chequebook.go:190-250` sends no transaction); only cashing out touches +the chain, and because cheques are cumulative that gas is paid **once per +account**. + +**This removes the gas floor, not the price floor.** At ~`1e-10` xDAI a +cashout, essentially any non-zero cheque is worth collecting. Design §11's +flagship 71 MB browser upload is worth `71/1024 × $0.02 ≈ $0.0014`, which +clears its own settlement fee by about seven orders of magnitude. The +conclusion this section used to draw — *"a one-shot user is not +profitable"* — was an artifact of the stale gas number and is withdrawn. + +What survives is an *operational* floor rather than an economic one: an +RPC round trip, a pending transaction to watch, and a chequebook that has +to stay funded. So read the 0.25 BZZ threshold as a batching convenience, +not as a break-even point — and note that Gnosis gas is not permanently +this cheap. A floor computed from `eth_gasPrice` at cashout time holds +under a price spike in a way a hardcoded constant does not, in either +direction. + +## 10. Credit and settlement + +**Postpaid.** The client accrues `owed` and settles when it crosses +`settle_every_plur`. Not prepaid: an anonymous browser prepaying a relay +is exposed to outright theft, whereas postpaid exposes the relay to at +most one cap of bandwidth. The asymmetry is correct — the relay's risk is +a fraction of a cent, the client's would be its deposit. + +### 10.1 Parameters, and the invariant to hold + +> **Invariant: `min_cheque_plur ≤ settle_every_plur < max_outstanding_plur`.** +> A client that is 402'd must always be able to clear it with a cheque for +> exactly what it owes. + +An early draft published `min_cheque_plur` 87× larger than +`settle_every_plur`. Every metered account would have bricked: accrue → +cross `settle_every` → sign a cheque → **rejected as dust** → keep +accruing → 402 → the only cheque that clears the 402 is 21× what is owed, +which the no-prepayment rule forbids. No exit. The error came from sizing +the dust floor against *cashout gas*, which cumulative cheques amortize +separately (§8.3). + +`min_cheque_plur` exists only to bound RPC cost per unit of value (§11.6 +lists up to 4 `eth_call`s per cheque, of which `liquidBalanceFor()` and +`paidOut()` cannot be cached). A quarter of `settle_every` is sufficient. + +At $0.02/GiB → 4.8e8 PLUR/KiB: + +| parameter | value (PLUR) | in payload | rationale | +|---|---|---|---| +| `price_plur_per_kib` | 4.8e8 | 1 KiB | §9.2 | +| `min_cheque_plur` | 3.9e12 | ~8 MiB | ≥ 4 `eth_call`s' worth of value | +| `settle_every_plur` | 1.56e13 | ~32 MiB | ~2–3 cheques per 71 MB upload | +| `max_outstanding_plur` | 6.22e13 | ~127 MiB | 4 × settle_every — a *ceiling*, not the cap; §10.3 | +| cashout threshold | 2.5e15 | ~5 GiB | §9.3 | + +The unsecured credit at risk per account is at most ~127 MiB ≈ **$0.0024**, +and less for any batch worth under ~6.2 BZZ, since §10.3 caps it at a +thousandth of the batch's remaining on-chain value. `settle_every` at +32 MiB also keeps the honest path off §11.6's amplification profile: ~2–3 +`/v1/pay` calls per 71 MB upload, not ~71. + +### 10.2 Reservation: bee's `reserve` was needed after all + +A monotone debit counter per account is not sufficient. `/v1/push` +deliberately does not serialize (`src/pusher.rs:785-846`: *"serializing +them … forced needless failover churn"*), so N concurrent POSTs each read +`outstanding` before any of them debits. + +N is now bounded — §16.3 added a 256-permit semaphore acquired before +`accept()` (`src/pusher.rs:377-415`) — but 256 concurrent full-size POSTs +is still `256 × 2 125 KiB × 4.8e8 = 2.6e14` PLUR, **4× +`max_outstanding_plur`**. The connection cap turns an unbounded overshoot +into a merely large one; it does not remove the need for a reservation. A +*polite* client at the relay's own advertised `inflight_max` of 8 +overshoots a 4-×-`settle_every` cap on its own. + +Fix, per §7.2: reserve `ceil(Content-Length / 1024) × price_plur_per_kib` +atomically at admission and release the unused remainder at completion — +bee's `reserveTotalIssued` applied to the receiving side. + +There are four coupled per-account quantities, not one: `owed`, +`reserved`, `last_cumulative[chequebook]`, and the `chequebook → account` +binding. They are written by N spawned tasks and read by admission, so +they need one lock. + +**But `reserved` must not be persisted.** A reservation belongs to an +in-flight POST, and no in-flight POST survives a restart — there is no +task left to release it. Restoring `reserved` from disk leaks credit +permanently and can brick an account into 402 with no way out, which is +exactly what §10.1's invariant exists to prevent. + +> **Persist `owed`, `last_cumulative` and the chequebook binding +> atomically. Reconstruct `reserved` as zero at boot.** + +The exposure from zeroing is one body's worth of over-admission +immediately after a restart — cents of egress, against an accounting +corruption that never self-heals. `owed` is written at batch completion, +so a crash forfeits at most the batch in flight, the same safe direction. + +### 10.3 Sybil bound: derive the credit line, don't assert it + +"An account is a batch owner, a live batch costs real BZZ, so the margin +is three orders of magnitude" is false. The relay checks **liveness**, and +liveness is satisfied by the cheapest batch the `PostageStamp` contract +will accept — minimum depth, minimum validity — which costs a fraction of +a cent. At a flat $0.0024 credit line the real margin is of order **1×**: +a throwaway batch buys roughly its own value in free relay. + +A hard floor on batch depth and TTL would fix the arithmetic at the cost +of excluding small legitimate users, which is the wrong trade for a system +whose flagship case is a 71 MB browser upload. So instead of gating on +batch size, **scale the credit line to it**: + +> **`max_outstanding(A, B) = min(remaining_value_plur(B) ÷ credit_ratio, +> max_outstanding_plur)`** + +with `credit_ratio = 1000` and `remaining_value_plur` from §6's standing +read. The Sybil margin is then **1000× by construction and independent of +batch size**: any attacker, holding any mix of batches, obtains total +credit equal to one thousandth of the on-chain value they actually funded. +There is no cheap corner of the parameter space, because the ratio is the +invariant rather than a consequence of a particular batch being expensive. + +Concretely: a batch needs ~6.2 BZZ (≈ $2.49) of remaining value to +saturate the global $0.0024 ceiling; a 0.01 BZZ dust batch earns +`1e11 ÷ 4.8e8 ≈ 208 KiB` of credit — enough to be useful, far too little +to farm. The line decays on its own as the batch is spent down or +approaches expiry, and §6's TTL is what makes that decay visible. + +This is the same move as §8's: replace an asserted constant with a +quantity read from chain, so the property holds by construction rather +than by assumption about what attackers will bother to buy. + +## 11. Attack surface + +Everything here is **a client attacking a relay**, per §2. Severity is +from the relay's point of view. "Inherited" means the issue exists in +bee's SWAP too; "introduced" means metering creates it. + +### 11.1 CRITICAL — stamp replay becomes billing griefing (introduced) + +**Swarm stamps are public.** Design §6 says so — *"anyone who saw your +chunks holds valid `(addr, stamp, wire)` triples"* — and they are +recoverable from the network, since retrieval returns the stamp alongside +the data (`src/protocols/retrieval.rs:27`, `:42-80`). A relay holds every +stamp it has ever relayed, so it is the densest source of all (§2). + +In open mode this is harmless: re-pushing is idempotent. **Under metering +it is an attack.** An attacker harvests a victim's stamps and replays them +at a metered relay. Auth passes — the stamps genuinely recover to the +batch owner — so the work is billed to the *victim's* account. Cost to the +attacker: zero. + +**Mitigation, mandatory:** the challenge of §7.2, plus §6's rule that +every frame must carry the exact batch id named in the challenge. The +challenge must be *signed by the account*, so possession of a harvested +stamp is not enough — the attacker would need the batch owner's key, which +is the same key that signs the stamps it is replaying. + +**The signed payload binds the relay's origin, not its beneficiary.** An +earlier draft signed `(nonce ‖ beneficiary ‖ account)` so a challenge could +not be replayed across relays. That fails: nothing authenticates a +beneficiary, so relay A can advertise honest relay B's beneficiary *and* +serve B's nonce, collect a victim's signature during a normal upload +through A, and replay it at B alongside V's stamps. Bind **`origin`** — +the host the client dialled — and have the relay verify it against its own +**configured** hostname (§7.2). Deriving it from a request header makes +the binding a no-op. + +### 11.2 CRITICAL — the withdraw race: cheques are unsecured (inherited) + +bee deploys chequebooks with a hard-deposit timeout of zero +(`init.go:169` passes `big.NewInt(0)` to `Deploy`, whose parameter is +`defaultHardDepositTimeoutDuration`, `factory.go:35`, `:62`) and never +places hard deposits, so the whole balance stays liquid and the issuer can +`withdraw()` at any time. The relay's funding check is therefore true **at +acceptance time, not at cashout time**. + +Attack: accrue debt → sign a covering cheque → relay verifies and keeps +serving → withdraw the chequebook → the relay's cashout collects less than +it is owed. Bee has the identical exposure; it is inherent to +SimpleSwap-as-deployed. + +**What reading `ERC20SimpleSwap` directly established:** + +*Cashout does not revert.* `_cashChequeInternal` pays +`totalPayout = Math.min(requestPayout, liquidBalanceFor(beneficiary))`, +credits `paidOut[beneficiary]`, and only then, if +`requestPayout != totalPayout`, sets `bounced = true` and emits +`ChequeBounced()`. An underfunded cashout **succeeds partially** and takes +whatever is there. The exposure is the shortfall, not the whole claim, and +`bounced` is *readable contract state* set contract-wide, so one bounce +marks the chequebook permanently. + +*Hard deposits work, but are inert as bee deploys them.* `withdraw()` is: + +```solidity +function withdraw(uint amount) public { + require(msg.sender == issuer, "not issuer"); + require(amount <= liquidBalance(), "liquidBalance not sufficient"); + require(token.transfer(issuer, amount), "transfer failed"); +} +``` + +with `liquidBalance() = balance() − totalHardDeposit`, so a deposit +genuinely locks funds. But decreasing uses + +```solidity +uint timeout = hardDeposit.timeout == 0 ? defaultHardDepositTimeout : hardDeposit.timeout; +hardDeposit.canBeDecreasedAt = block.timestamp + timeout; +``` + +and `decreaseHardDeposit` requires +`block.timestamp >= canBeDecreasedAt && canBeDecreasedAt != 0`. With bee's +`defaultHardDepositTimeout = 0` and a fresh deposit's `timeout == 0`, both +conditions hold in the *same block*: prepare, decrease, withdraw. A hard +deposit on a bee chequebook is this attack with two extra calls in front +of it. + +`setCustomHardDepositTimeout(beneficiary, timeout, beneficiarySig)` is the +way out — issuer-submitted, but requiring an EIP-712 signature **from the +beneficiary**, and the hash binds `address(this)`, the client's chequebook, +so it cannot be pre-signed. Secured mode therefore costs one interaction +with the cold beneficiary key per client chequebook. Deferred to §14 +Stage 2 as an account-tier feature, not a default. + +**Mitigations, in force:** + +1. **The per-account cap** (`max_outstanding_plur`), bounding the yield to + $0.0024 per account and one postage batch per account. This is the real + defence — **and it only works if §10.2's reservation ships.** +2. **Cash out at the threshold and treat sub-threshold value as at risk.** + At 5 GiB/account (§9.3) most accounts are never cashed, so the exposure + is chronic rather than acute. Bounded by mitigation 1, not by + promptness. +3. **Blocklist bouncing issuers.** Read the `bounced` flag — a cheap + `eth_call`, not a log subscription you had to be watching at the time. + +Do not treat an accepted cheque as settled revenue. It is a claim. + +### 11.3 HIGH — Sybil beneficiaries and aggregate exposure (introduced) + +Nothing proves a relay controls the beneficiary EOA it advertises; that +would require holding the key, destroying the property in §6. Since a +cumulative is per `(chequebook, beneficiary)`, **one operator running N +lane URLs with N beneficiary EOAs presents N independent accounts**, and +the client has no aggregate view until `src/cheques.rs` grows +`total_issued` (§8.3). + +Under §2 this is not a relay-fraud concern — it is a client-side budgeting +concern, and the fix is client-side: `total_issued` as a hard ceiling +across all beneficiaries, and pinning `(url, node_eth_address, beneficiary)` +in config rather than reading the beneficiary from `/v1/status` at runtime. + +### 11.4 HIGH — relay state loss is an unbounded free-service loop (introduced) + +If the relay loses `last_cumulative[chequebook]`, a client re-presents its +most recent cheque and is credited the full cumulative instead of the +delta. + +Seeding `last_cumulative` from on-chain `paidOut(beneficiary)` is inert at +§9.3's parameters: with a 5 GiB cashout threshold, almost no account is +ever cashed, so `paidOut` is permanently 0 and the seed is 0. The attack +is then: pay one cheque, consume service, wait for a restart, re-present +**the same cheque**, repeat — unbounded free service from a single +signature at zero incremental cost. Restarts are not incidental on free +tiers that sleep and cold-start (design §7 measured a 35.2 s wake). + +**Mitigation: metered mode requires durable storage** (§5). Persist +`owed`, `last_cumulative`, the `chequebook → account` binding **and +`relay_secret`** (§7.2) atomically together — losing only +`last_cumulative` over-credits, losing only `owed` under-bills, losing +`relay_secret` 403s every live client. Seed from `paidOut` on first run as +a floor, never as the primary source. A relay that cannot guarantee +durable state must run `open`. + +**`reserved` is deliberately excluded** and reconstructed as zero at boot +(§10.2). + +### 11.5 MEDIUM — shallow-cascade cost griefing (introduced) + +For a chunk nothing will store: 3 peers are raced immediately +(`src/client.rs:4086`); candidates are topped up to +`cap = max_retries.min(order.len())` with `PUSH_MAX_RETRIES = 20` +(`src/pusher.rs:94`, applied at `src/client.rs:4315`); if every outcome is +shallow or overdraft with no hard error the dispatcher **walks past `cap`** +over the whole eligible pool (`:4573-4602`, up to 128 sessions); +shallow-only then returns a *retryable* outcome and the outer loop +re-dispatches up to `MAX_CHUNK_RETRIES = 60` (`:4104`). The attacker aims: +the relay publishes its overlay (`src/pusher.rs:499`), `pool.live` +(`:511-514`) and the `pool_po` mean inside `diag` (`:525`), so addresses +can be mined into arcs the pool covers poorly. Design §10 measured 8.1 +shallow receipts/chunk at pool 32 as the *accidental* rate. + +**Bytes-admitted billing prices most of this away.** The attacker pays for +every byte it sends regardless of outcome, so it can no longer buy cheap +egress by aiming at badly-covered arcs — the arbitrage that made this +economically rational under receipt billing is gone. What remains is pure +griefing: the attacker burns its own credit to make the relay burn more. +The amplification factor is the cost control that matters, so metered mode +caps delivery *attempts* per chunk well below open mode's fallback and +disables the past-`cap` walk. + +### 11.6 HIGH — RPC amplification on `/v1/pay` (introduced) + +Accepting a cheque costs up to 4 `eth_call`s. Order the checks +cheapest-first and reach the chain only after every free check passes: + +1. Parse and length-check. +2. **Reject non-canonical signatures**: length ≠ 65, + `s > secp256k1n/2`, or `v ∉ {27, 28}`. Free, and mandatory — see below. +3. EIP-712 recover (local, ~80 µs). +4. `beneficiary == ours`, `chain_id == ours`. +5. `cumulative > last_cumulative[chequebook]` and + `amount ≥ min_cheque_plur`. +6. *Then* RPC: `deployedContracts` (cache forever per chequebook), + `issuer()` (**cache per chequebook** — bee refetches it on every cheque + at `chequestore.go:149` despite its own comment saying it never + changes; do not copy that), `liquidBalanceFor(ourBeneficiary)` and + `paidOut(ourBeneficiary)` (uncacheable — they are the §11.2 check). + Read `bounced` in the same batch and refuse the chequebook if set. + +**That ordering bounds nothing on its own.** Every "free" check passes for +a cheque an attacker synthesizes at zero cost: it reads `beneficiary` and +`chain_id` from the public quote, picks a random chequebook address, signs +with a throwaway key — step 3 recovers *some* address, and nothing +compares it to anything until `issuer()` — and sets `cumulative` above +`min_cheque_plur`, which clears step 5 trivially because `last_cumulative` +for an unseen chequebook is zero. Each garbage POST costs one +`deployedContracts` call. Cheapest-first is a latency optimisation; the +bound must come from somewhere the attacker cannot reach: + +- **No debt, no cheque.** Refuse any cheque for an account with + `owed == 0`, before parsing. Postpaid (§10) means an honest client + always has debt by the time it settles, so this costs nothing legitimate + and makes the endpoint useless to anyone who has not first done billable + work under a challenge. +- **Require the challenge header on `/v1/pay`**, as on `/v1/push`. The + caller must hold a batch in standing to reach any code path, pricing the + amplifier at one live postage batch. +- **Negative-cache non-deployed chequebooks** — bounded LRU with a TTL, + the structure the owner cache grew in §16.2. +- **Rate-limit per account.** **`src/ratelimit.rs` cannot be reused** — it + is a per-peer *outbound libp2p dial* GCRA pacer that parks rather than + refuses (`src/ratelimit.rs:1-30`), with no inbound, per-account or HTTP + concept. Stage 1 needs a new inbound limiter, also covering + `/v1/challenge`. + +**On step 2 — this is a validity check, not a malleability nicety.** +`ERC20SimpleSwap.recoverEIP712` calls `ECDSA.recover` from +`@openzeppelin/contracts/cryptography/ECDSA.sol` at `^3.4.1-solc-0.7-2` +(the repo's `package.json`), which is unconditionally strict: + +```solidity +require(uint256(s) <= 0x7FFF…5D576E7357A4501DDFE92F46681B20A0, "ECDSA: invalid signature 's' value"); +require(v == 27 || v == 28, "ECDSA: invalid signature 'v' value"); +``` + +plus a 65-byte length check and a zero-address check. `alloy` recovers +high-`s` and `v ∈ {0,1}` happily, so **a client can issue cheques that +verify off-chain, buy service, and revert at cashout** — free relay, with +the relay's own acceptance as the evidence it was paid. OZ's comment notes +the alternative of rewriting as `(n − s, v^1)`, but there is no reason to +accept a client that cannot produce a canonical signature, and `alloy` +always does, so rejection costs honest clients nothing. + +### 11.7 MEDIUM — hedging costs money (introduced) + +The scheduler hedges stragglers to a rank-#2 lane (design §7). Both relays +receive the bytes, so both bill. The recent-ack cache does not absorb it: +`push.recent` is populated only **after the whole batch drains** +(`src/pusher.rs:1050-1065`), not in the per-chunk callback (`:1000-1025`), +so two concurrent POSTs containing the same address — including the +design's own *"re-POST unacked frames"* retry model (design §3) — are not +deduped even on the same relay. + +Under §8 this is not a dispute: the client knows it sent those bytes +twice, and it did. **`hedge_fraction` becomes an economic parameter, not +just a latency one**, and clients should lower it against metered lanes. + +### 11.8 MEDIUM — mixed-account batches bypass admission (introduced) + +`run_push` handles frames from different batches with different owners in +one POST (`src/pusher.rs:905-925`), while §7.2's admission necessarily +picks one account. Prefix one frame from a good-standing account and fill +the remaining 511 from a 402'd account: admission passes, billing lands on +the wrong account. Closed by §6's one-batch-per-request rule. + +### 11.9 LOW + +| Threat | Assessment | +|---|---| +| Cross-relay cheque replay | Impossible: `beneficiary` is inside the signed EIP-712 struct. | +| Cross-chain replay | Impossible: `chainId` is in the domain separator; the relay pins its own. | +| Reorg invalidating `deployedContracts`/`liquidBalanceFor` | Negligible on Gnosis at these values; re-verified at cashout. | +| Paying off someone else's debt | Requires their chequebook key. Not a threat. | +| Price change mid-flight | Bill at the signed quote in force when the work was done; `quote_valid_secs` must exceed one settlement period (§10.1's sizing gives 24 h against a ~32 MiB period). | +| `/v1/account` as a volume/targeting oracle | Closed by requiring the challenge header (§7). | +| Challenge replay inside its own window | Bounded by the reservation (§7.2), a 300 s `challenge_ttl_secs`, and origin binding. Only the relay sees the header over TLS, so it must never be logged or echoed back. | +| Client under-reports bytes | Not possible. The relay counts what it received; `Content-Length` is a commitment the client made before the relay did any work. | +| Client disputes the invoice | Nothing to dispute — §8.4. | + +## 12. Client scheduler integration + +- `LaneInfo` (`src/pushsched.rs:101-112`) gains `price_plur_per_kib` and + `mode`. All five existing fields are already `Option`, so parsing is + free — but *scheduling* is not: `weight()` (`:257-278`) multiplies + `rate × budget × concurrency` against a `MIN_WEIGHT` floor (`:65-68`), + and an unknown price has no safe default. Treating it as free prefers + unadvertised lanes; treating it as expensive pushes warming lanes under + `MIN_WEIGHT`, where they can never revise their EWMA — the exact failure + `MIN_WEIGHT` exists to prevent. Probably "unknown price ⇒ treat as the + median of known lanes". +- **A 402 needs its own outcome.** `BatchOutcome` is + `Answered | Failed(String)` (`:304-310`), so a 402 maps to `Failed` → + `fail_streak++` → `Backoff` → `Retired` after 5 doublings (`:60-63`, + `:88-91`). "Pay, then retry" would cost lane health on every routine + settlement and retire a healthy lane mid-upload. Add + `BatchOutcome::PaymentRequired`, which pauses the lane without touching + the streak. +- **"Cannot pay" needs a non-terminal ineligible state.** `eligible()` + (`:228-234`) is a pure function of `LaneHealth`, and the only ineligible + terminal state is `Retired`, which is permanent for the run — wrong for + "temporarily out of chequebook balance". +- **POST sizing must respect `max_outstanding_plur`.** The challenge + returns the cap; the client sizes its body to fit rather than + discovering the ceiling as a 402 (§7.2). +- Lower `hedge_fraction` against metered lanes (§11.7). +- Lane weighting should prefer `open` lanes when both are eligible and the + client has no chequebook — otherwise a browser client with no payment + path spends its first POST discovering a 402. +- **Lane health is the client's protection**, per §2. A lane that takes + bytes and delivers poorly shows up in the existing EWMA and gets + deweighted. No new mechanism. + +## 13. Rejected alternatives + +**Billing per verified pushsync receipt.** The previous design. Rejected +in §8: it makes a third party's signature into a billing input, which +forces a staking-registry anchor, a set-valued invoice, a sampled +retrieval audit, and a shared predicate module that must not drift between +the parties — and after all of it, a receipt still carries no freshness, +so a relay could replay one for content already in the swarm and no +off-chain check could tell. Bytes-admitted deletes the input and the +entire apparatus with it. + +**Per-chunk / per-frame cheques.** §8.3. Serial by construction; kills +multi-lane pipelining. + +**Billing per ack.** Every field of an ack is a relay assertion +(`src/pusher.rs:1005-1020`). Under §2's trust model this is *nearly* +acceptable — but bytes-admitted is strictly better at the same cost, since +the client can verify it without trusting anything. + +**Relay earns like a bee forwarder.** Bee's ledger does pay forwarders — a +forwarder earns `(PO_next − PO_self) × 10 000` per chunk — but cheques are +issued *only* against `originatedBalance` +(`bee/pkg/accounting/accounting.go:472-484`). Non-originated debt settles +purely in free, rate-capped refreshments, so the spread is never +monetised. + +**Postage in kind** (client stamps chunks the relay names). Pays in +storage for a service whose cost is bandwidth. Cute, weak. + +**Direct on-chain BZZ prepay.** Simpler — no chequebook deployment — but +gas on every top-up and no off-chain micropayments. Kept as a fallback if +chequebook onboarding proves fatal in the browser. + +**A shared multi-tenant escrow contract.** Would remove per-user chequebook +deployment: one contract, many depositors, cumulative vouchers to any +beneficiary. Attractive UX, but a new contract to write, audit and +maintain, forfeiting reuse of an audited deployment. Not worth it unless +per-user deployment measurably kills conversion. + +**Cheque-as-credential, never cashed.** The relay accepts cheques purely +as funded, non-replayable proof of identity and never cashes them. Zero +cashout machinery, zero gas, no §11.2 exposure, no §9.3 profitability +problem — and it subsumes `--push-challenge`. The cost is that it prices +nothing: it gates abuse but does not fund the dedicated egress IP §1 says +is actually scarce. Kept as the fallback if Stage 0 shows consumption too +thin to bother metering. + +## 14. Staged plan + +**Stage 0 — metering instrument, zero protocol change. Shipped: +`src/meter.rs`.** Shadow accounting in the relay: per-`(owner, batch)` +bytes admitted, what they would owe at §9.2's candidate price, and the +credit line §10.3 would have granted each batch. Nothing is billed, no +request is refused, and no client-visible behaviour changes. + +Where to read it: + +- **`GET /v1/status` → `meter`** — aggregates only. Accounts and batches + seen, KiB admitted and deduped, total owed in PLUR and USD, how many + accounts would reach one settlement or the cashout threshold, the + observed egress multiplier, and the credit-line distribution. Names + nobody, so it stays public alongside the `bytes_pushed` total already + published there. +- **`GET /v1/meter`** — per-account and per-batch rows. **Open by default**; + set `HOVERFLY_PUSH_METER_TOKEN` to require an `Authorization: Bearer` + header. Everything in it derives from state that is already public — + batch owners and balances are on-chain and enumerable from + `BatchCreated`, and the stamp on every relayed chunk names its batch, + which retrieval hands back (§2) — so it adds only relay attribution and + timing. + + **This changes at Stage 1, for a reason worth stating precisely.** §7 + authenticates `/v1/account` on two grounds, and only the second is load + bearing: "per-identity volume oracle" is weak against a network where + stamps are public anyway, but "targeting oracle for tipping a victim into + 402 at a chosen moment" is real — knowing an account's *outstanding* + balance lets an attacker time a stamp replay (§11.1) to break a victim's + upload at a chosen instant. That is an active attack enabler, not a + privacy leak, and it does not exist in Stage 0 because there is no 402 + and no cap enforcement. Do not carry the Stage 1 conclusion backwards. + +Three properties worth knowing when reading the output: + +- **Costs no extra RPC.** `resolve_owner` already reads both `read_batch` + and `read_remaining_balance` to check for expiry, so the batch's total + value (`remainingBalance × 2^depth`) simply rides along on the cached + success rather than being discarded. +- **Costs one lock per POST**, not per frame: a request accumulates into a + stack-local `PostTally` and merges once at completion. +- **Bounded and in-memory.** 4 096 `(owner, batch)` rows with FIFO + eviction, for the same reason the owner cache is bounded (§16.2) — the + key is attacker-chosen. Evicted rows keep contributing to the totals, so + headline volume stays honest; only their detail is lost. State resets on + restart, which biases "do accounts return?" downward — check + `window_secs` before drawing conclusions. + +**What gates Stage 1:** the **distribution of batch remaining value across +accounts**, reported as `credit.batches_below_one_full_post` and the +`credit_kib` percentiles. It calibrates `credit_ratio` (§10.3) — +specifically, what fraction of real users would be capped below the global +ceiling, and whether §7.2's POST-sizing interaction bites in practice. + +Stage 0 was also meant to settle §9.1's cost basis, which the doc *models* +rather than measures: `egress.attempts_per_frame` against the modelled +3.45. **It does not, and the claim that justified it was wrong.** The +counter was believed to see every per-stream attempt including losing +racers; it is bumped after the await inside a future the dispatcher +cancels, so it sees completions only. §9.1 has the detail. This gate is +**not met** and needs the counter moved to dispatch before any repricing. + +*(The previous design had four gating measurements including a kill +criterion — the staked fraction of receipt signers, which could have +vetoed metering outright. Bytes-admitted removes the dependency.)* + +**Stage 1 — the relay can be paid (native client only), soft mode.** +**Relay side shipped.** Enabled with `--meter --origin --beneficiary +<0x…> --state-dir `; without `--meter` nothing below is reachable and +the relay behaves exactly as it does today. Modules: `src/ledger.rs`, +`src/metered.rs`, `src/inbound_limit.rs`, plus cheque recovery in +`src/signer.rs`, a `SignedCheque` decoder in `src/protocols/swap.rs` and +chequebook bindings in `src/batch.rs`. Endpoints: `GET /v1/challenge`, +`GET /v1/account`, `POST /v1/pay`, and metered admission on +`POST /v1/push`. + +Those four sit behind the `pusher` cargo feature, which exists to pull in +hyper — it is the *relay*. The shared and client-side pieces +(`src/challenge.rs`, `src/meter.rs`, `src/payer.rs`) deliberately do not: +a client that pays a metered relay needs the challenge wire format, the +pricing arithmetic and the payer, but no server. `challenge.rs` owns the +whole challenge protocol including the `x-hoverfly-challenge` codec, so +the two ends cannot drift on a format only one of them defines. + +**Client side shipped** (`src/payer.rs`): signed-quote verification with +lane pinning on `(url, node_eth_address, beneficiary)`, challenge parsing +and signing, POST sizing against the returned cap, and local `owed` +tracking computed from bytes sent. `BatchOutcome::PaymentRequired` and a +non-terminal `LaneHealth::Unfunded` are in `src/pushsched.rs`, and +`src/cheques.rs` gained `total_issued` plus a `relay:` key namespace. + +**Chequebook deployment is an explicit user action, never automatic.** It +belongs in its own command (`hoverfly chequebook deploy`, alongside +`batch create`), not on the upload path. Deploying a contract is +irreversible and spends real funds, so an upload that silently deployed one +because some lane quoted a price would fire on the *first* metered lane a +user ever met — before they had decided they wanted to pay at all. This is +the same shape §14 Stage 3 gives the browser, where the wallet deploys the +chequebook as a deliberate opt-in. hoverfly has no deploy path today: it +consumes a chequebook via `--chequebook` and documents "already deployed by +bee's official factory" as a precondition. + +Two things remain before hard mode: the driver does not yet *act* on a 402 +by issuing a cheque and calling `Scheduler::fund_lane` (the pieces exist +and are tested; the loop that connects them does not), and the `/v1/pay` +chain reads have never been exercised against a real chequebook. Soft mode +is what makes that ordering safe: a relay can run `--meter` against today's +clients without breaking them, because nothing is refused. + + +- Byte accounting per account with a **body-bounded reservation** (§7.2, + §10.2), durably persisted except `reserved` (§11.4). +- EIP-712 **recovery** in `src/signer.rs`, mirroring `sign_cheque` + (`:309-337`). The `sol!` `Cheque` type exists at `:21-27`; the + `Eip712Domain` is built inline at `:321-327` and needs extracting to be + shared. +- `SignedCheque` JSON **decoder** in `src/protocols/swap.rs` — only the + encoder exists (`encode_signed_cheque_json`, `:84-106`). It must reject + non-canonical signatures at parse time (§11.6): it is the one place + every cheque passes through. +- `ERC20SimpleSwap` + factory `sol!` bindings — `issuer()`, + `liquidBalanceFor(address)`, `paidOut(address)`, `bounced()`, + `deployedContracts(address)`. `EthRpc::call_view` (`src/batch.rs:726`) + is **private**, so these either live in `batch.rs` or it gains + visibility. +- **Re-key `src/cheques.rs` on `(chequebook, beneficiary)`** instead of + peer overlay (`:64-76`), and add `total_issued` (§8.3). +- Stateless MAC challenge with standing baked in, over a length-prefixed + domain-tagged preimage compared in constant time; EIP-712 + `PushChallenge` verification; `relay_secret` persisted with the ledger + (§7.2). CORS `allow-headers` update (`src/pusher.rs:468-486`). +- Signed quote carrying `node_eth_address`, `overlay_nonce` and `origin` + (§7.3); a new inbound per-account rate limiter (§11.6), also covering + `/v1/challenge` and `/v1/pay`. +- `/v1/pay` hardening: challenge header required, `owed == 0` refused + before parsing, non-deployed chequebooks negative-cached (§11.6). +- One-**batch**-per-request enforcement; batch *standing* reads with a TTL + on the owner cache (§6); credit line derivation (§10.3). +- Metered retry policy (§11.5). +- Forward the pushsync receipt into the ack as **telemetry** (§2). Not + billing-relevant, but independently the first cryptographic evidence a + relay client has ever had, and it feeds lane weighting. + +Pusher flags: `--meter`, `--origin` (**required under `--meter`**; §7.2, +§11.1), `--beneficiary`, `--price-plur-per-kib`, `--settle-every-plur`, +`--max-outstanding-plur`, `--min-cheque-plur`, `--credit-ratio`, +`--state-dir`. + +Client: the native path already has `--chequebook` / `--cheques-file` / +`--chequebook-chain-id`, so it needs the 402 handler, the challenge +signature, POST sizing against the returned cap, lane pinning on +`(url, node_eth_address, beneficiary)` (§7.3), and `total_issued` (§8.3). + +**Stage 2 — hard mode and cashout.** 402 enforcement; scheduler changes +(§12). + +**`hoverfly cashout` shipped**, run from a machine holding the beneficiary +key, never the relay. It reads the relay's ledger, prices each held cheque +on-chain, and presents the ones worth collecting. Two things it forced: + +- **The ledger has to keep the signature.** The first cut stored only the + cumulative, which made the whole thing a record of money that could not + be collected — the relay could prove nothing about a number it held + alone. On-disk format is now version 2; a version 1 entry loads (losing + its cumulative would let the client replay, §11.4) but is skipped at + cashout rather than submitted for the contract to reject. +- **Cashing is naturally idempotent**, because the contract tracks + `paidOut(beneficiary)`. A repeated run sees `unclaimed 0` and presents + nothing, which matters for a command that will be run on a timer. + +`--min-amount` defaults to 0.25 BZZ (§9.3's threshold). The reasoning it +was given — cashing costs ~300k gas whatever the amount, so a smaller +cheque is worth less than collecting it — does not survive §9.3's measured +fees; at ~`1e-10` xDAI the amount that is *not* worth collecting is far +below any cheque this system will produce. Keep the default as batching +convenience, and see §9.3 on deriving it from live gas. Verified on Gnosis +mainnet — a 145,920,000,000 PLUR cheque presented, +`paidOut` moved by exactly that, and the beneficiary's BZZ balance moved by +exactly that. Cashout reads `bounced` and +`liquidBalanceFor` rather than `balance` (§11.2), and optionally offers +**secured mode** for high-volume accounts: the beneficiary signs a +`setCustomHardDepositTimeout` for that client's chequebook, the client +calls `increaseHardDeposit`, and the cheque stops being an unsecured +claim. Interactive by nature — it needs the cold key once per client +chequebook — so it is an account-tier feature, not a default. + +**Stage 3 — browser.** Deferred. Unblockers: export `sign_cheque` through +`#[wasm_bindgen]` (it already compiles on wasm — `src/signer.rs` has no +`cfg` gates — it is simply not bound in `src/wasm.rs`); an IndexedDB or +localStorage cumulative store, anticipated at `src/cheques.rs:24-26`; and +an opt-in dApp flow where the wallet deploys a chequebook with +`issuer = sessionKey` and funds it — two transactions on top of today's +approve + `createBatch`, ≈ $0.001 of Gnosis gas, one-time per user and +reused across every upload and lane. The dApp's four public lanes stay in +`open` mode throughout, so nothing here is on the settlement path at all. + +One Stage 3 hazard Stage 1 does not have: **a session-key chequebook can +strand funds permanently.** `withdraw()` is issuer-only, and the issuer +here is a key living in localStorage. Clearing site data destroys the only +key that can recover the remaining balance. Fund in small increments, warn +on deposit, and offer an explicit "drain chequebook" action before the key +is discarded. A wallet-owned chequebook with the session key as a mere +*signer* would avoid this, but SimpleSwap has no such split. + +## 15. Residual risks and open questions + +Carried knowingly: + +- **§11.2** — cheques are unsecured claims, bounded by the per-account cap. + Hard deposits work but are inert as bee deploys them; secured mode costs + a cold-key signature per client chequebook. +- **§9.3** — the cashout threshold is a hardcoded constant sized against a + gas cost that no longer holds. At today's Gnosis prices it is a batching + convenience, not a break-even, and a gas spike moves the real floor + without moving the constant. Deriving it from `eth_gasPrice` at cashout + time is the fix; until then the number is stale in whichever direction + gas has drifted. +- **§11.3** — aggregate exposure across beneficiaries is invisible to the + client until `total_issued` ships. +- **§11.5** — griefing by aiming at badly-covered arcs still forces the + relay to spend more than the attacker pays, even though the attacker now + pays. Bounded by attempt caps, not eliminated. +- **A relay can take payment and drop chunks.** Out of scope for + cryptographic treatment by §2. What bounds it is that the client pinned + the lane, caps its exposure at one credit line, and deweights a lane + whose acks do not arrive — so the loss is at most `max_outstanding` and + is detected within one settlement window. Adequate for a lane a client + chose; **not** a proof of delivery, which pushsync cannot give us. + +Open: + +- What would it take to open the relay set? The honest answer is a + freshness-bearing proof of delivery, which pushsync does not provide — + the storer signs the bare chunk address + (`bee/pkg/pushsync/pushsync.go:277`), so a receipt is valid forever, for + everybody, and a relay can replay one for content already in the swarm. + Two upstream asks would fix it, in increasing order of difficulty: + **(a)** a read-only "do you hold `(address, batch)`?" query — bee's + reserve is *already* indexed exactly that way + (`BatchRadiusItem.ID() = BatchID ‖ Bin ‖ Address ‖ StampHash`, + `pkg/storer/internal/reserve/items.go:35-37`), so this is a lookup on an + existing index; **(b)** having the storer sign + `keccak(chunk_address ‖ stamp_hash)` instead of the bare address. Both + have value well beyond metering — any uploader could prove *its own* + stamp reached a neighbourhood — and (a) is a much easier sell. +- Price discovery across a federation: fixed per-operator quotes, or + something the client aggregates? +- Should a relay accept cheques from a chequebook whose `issuer()` is not + the batch owner, via a signed authorization (§6)? Needed by anyone + uploading under multiple batch owners. +- Is the batch owner the right account key at all, given §6's TTL problem + and the fact that batches expire? +- Is `credit_ratio = 1000` (§10.3) right? Chosen for a comfortable margin, + not derived. Stage 0's batch-value distribution should justify it or + move it. + +## 16. Found during review: six live open-mode bugs (all fixed) + +All independent of metering, all in code running in production. They are +recorded here rather than in a commit message because the design argument +keeps citing them: §6, §10.2 and §11.6 all describe relay behaviour these +fixes changed. + +### 16.1 The recent-ack cache let one uploader substitute another's stamp + +**Status: fixed** — `RecentAcks` is now keyed on `(addr, batch_id)` +(`src/pusher.rs:286`, applied at `:928-948` and `:1050-1065`). + +Dedup was keyed on `chunk.addr` alone. On a hit the frame is acked +`{"s":"ok"}` and is **never added to `accepted`** — the submitted stamp +never reaches the swarm. Swarm addresses are content-derived, so two +uploaders of the same bytes collide by construction. + +Attack: push chunk X stamped with a dust batch expiring tomorrow. Within +`RECENT_ACK_TTL_SECS = 120` (`:129`) the victim pushes X stamped with its +year-long batch, is told `ok`, and the chunk lives under the attacker's +dying stamp. When that batch expires the chunk is garbage-collected and +the victim's manifest 404s. The victim has a successful upload record and +no way to know. The same thing happens *accidentally* between two honest +users uploading the same file within 120 s on one lane. + +Two notes on the applied fix, neither a defect today: + +- The completion path resolves the batch via `batch_of.get(a)` with an + all-zero sentinel fallback (`:1050-1065`). No admitted chunk currently + reaches it unmapped, but a sentinel that could in principle alias a + batch id is worth replacing with an `if let Some` that skips caching. +- If one POST carries the same address under two different batches, only + the last-inserted mapping is cached. That under-dedups rather than + mis-dedups — the safe direction — and metered mode forbids the case + outright under §6's one-batch-per-request rule. + +### 16.2 `/v1/push` was an unauthenticated RPC amplifier + +**Status: fixed.** `resolve_owner` cached only *successes*, in an unbounded +`HashMap`, with no per-request budget. A batch id that failed to resolve — +not found, or zero remaining balance — was re-resolved on every mention, +so one anonymous POST of `PUSH_BATCH_MAX` frames naming bogus batches +became that many serial `eth_call`s against the operator's RPC endpoint. +Nothing about the request needed to be valid: the amplification happens +*before* any push work. `EthRpc::new` made it worse by building a fresh +`reqwest::Client` per read, so every call also paid a new connection pool +and TLS handshake. + +Fixed by a bounded `OwnerCache` (`src/pusher.rs:181-239`, `:1106`) that +caches definitive rejections under a shorter TTL than successes but never +caches transport errors — an RPC blip must not blacklist a live batch — is +capped at 4 096 entries with FIFO eviction, and by a per-request budget of +`PUSH_MAX_BATCH_LOOKUPS = 8` distinct lookups. `EthRpc` now shares one +process-wide client (`src/batch.rs`). + +This is the same defect §11.6 identifies in metered `/v1/pay`, which is +why that section stopped treating cheapest-first ordering as a bound. + +### 16.3 The accept loop had no connection limit + +**Status: fixed.** The loop spawned per connection with no semaphore and +no cap anywhere, despite design §3 listing one as table stakes. Now a +`PUSH_MAX_CONNS_DEFAULT = 256` semaphore permit is acquired **before** +`accept()` and held for the connection's life (`src/pusher.rs:377-415`, +`HOVERFLY_PUSH_MAX_CONNS` to override). Acquiring before accepting is what +applies backpressure to the kernel queue instead of admitting everything +and queueing internally. + +§10.2's reservation argument is written against the fixed behaviour: 256 +is a bound, and it is still 4× `max_outstanding_plur`. + +### 16.4 One transient accept error killed the relay + +**Status: fixed.** `listener.accept().await?` propagated out of `run`, so a +single `EMFILE` or `ECONNABORTED` — both transient and both routine under +load — terminated the process. The loop now logs, backs off exponentially +to `ACCEPT_BACKOFF_MAX_MS = 1000`, and continues. + +### 16.5 There were no HTTP timeouts at all + +**Status: fixed.** The pre-existing comment said hyper's defaults were +fine. They are not, and the reason is worth recording: `header_read_timeout` +is **silently inert unless a timer is installed**. Without `.timer(...)` +hyper's `Time` is `Empty`, its `check()` returns `None`, and the +configured timeout never fires — so the code read as protected while +accepting connections that could hold a slot forever sending one header +byte at a time. + +Now `TokioTimer` is installed, `HEADER_READ_TIMEOUT_SECS = 30` applies, +and the `/v1/push` body read is wrapped in a +`PUSH_BODY_READ_TIMEOUT_SECS = 120` `tokio::time::timeout` returning +`408 Request Timeout`. Combined with §16.3's cap, a slowloris now costs an +attacker 256 connections for 30 seconds rather than indefinitely. + +### 16.6 Pushsync receipt addresses were never checked + +**Status: fixed** (`src/protocols/pushsync.rs:142-148`, with regression +tests). + +`receipt.address` was copied straight off the wire and never compared to +the address that was pushed. Two things followed, and any peer in the pool +could reach both — pool membership comes from hive gossip and the seed +list, so it is not a trusted set: + +- **Remote panic.** Callers build a `[u8; 32]` from it with + `copy_from_slice` (`src/client.rs:5380`, `:5398`), which panics on any + other length. A 0- or 33-byte address unwound the push task. +- **Address substitution.** A peer could accept the Delivery, store + nothing, and sign a receipt for some *other* address deep inside its own + neighbourhood. `is_shallow` and the `po` computation both read + `r.address`, so the forged receipt looked like a perfect deep delivery + and the chunk was acked `ok` having never been stored. + +Receipts no longer carry money under this design, but they still carry the +`po` and shallow signals the client schedules on, so a peer able to forge +them could steer traffic. Checking at the protocol boundary means every +`PushsyncReceipt` in the codebase carries exactly the 32 bytes that were +pushed. + +## 17. Found by running a metered relay: six bugs (all fixed) + +None of these is reachable from a single upload against a fresh relay, +which is why all six survived the test suite and the Stage 1 round-trip. +They need a relay whose ledger *persists across client runs* — the shipped +configuration — enough concurrency to have several POSTs on the wire at +once, and a batch that has been spent down far enough for its credit line +to bind. + +The last of those is the one to take seriously as a *method* point: §17.3 +is not a coding mistake but an invariant checked against the wrong +quantity, and nothing short of running a real batch until its value +decayed would have surfaced it. + +### 17.1 Debt the relay carried across sessions could not be paid + +**Status: fixed** (`LaneAccount::adopt_relay_debt`, `LanePayer::reconcile`, +with regression tests). + +§10.2's dust floor guarantees a run ends owing something: the residual +below `min_cheque_plur` is left unpaid because a cheque for it would be +refused. The relay is right to keep counting that against the credit line +— forgiving it would make "stay under the floor" a way to be served for +free. But the client's books are per-process, so the *next* run starts +believing it owes nothing, and the relay's `owed` only ever grows. + +Once the carry crosses `max_outstanding_plur` the account is refused on +its first POST, and the refusal is unpayable: `next_cumulative()` computes +the cheque from the client's own `owed`, which is zero. Observed live as +a second upload failing 151/151 against a relay carrying 290,400,000,000 +PLUR. With the shipped defaults the per-run residual is under 3.9e12 +against a 62.2e12 cap, so it takes roughly sixteen runs rather than two — +a slow-motion deadlock, not a corner case. + +The fix is to ask rather than to remember: `GET /v1/account` already +reports the relay's own figure, so a 402 the client cannot pay triggers a +reconcile and it pays what the relay says it owes. Three things about +which number, each of which was wrong in a draft: + +- **`owed`, not the `outstanding_plur` in the 402 body.** `reserve()` + adds the reservation *before* computing what it reports, so the body's + figure includes the request just refused. Adopting it over-pays by + exactly that body and the next cheque bounces as an overpayment. +- **Reservations excluded on our side too**, for the same reason: bytes + still in flight are already held in `pending_plur` and would be billed + twice when those POSTs land. Under-counting is safe and self-correcting; + over-counting is a refused cheque. +- **Bounded by the quote's ceiling, not by the per-batch credit line — and + then by the chequebook balance.** The line is the wrong bound: it shrinks + as the batch is spent down, so it falls below debt properly incurred when + the batch was worth more, and rejecting on that basis refuses a real bill + and preserves the deadlock. `max_outstanding_plur` from the *signed* quote + is the right one: admission refuses above the per-batch cap, and every cap + is `min(value / ratio, ceiling)`, so a figure above the ceiling describes + debt that cannot have been incurred. The balance check stays as a second, + looser gate — `settle` enforces it exactly across all lanes (§8.3), and + stating it here turns a confusing failure at signing time into a legible + one at reconcile time. + +This makes the client trust the relay's arithmetic about its own +receivable, but only within the credit that relay granted — which is what +pinning buys (§2). The first version bounded on the chequebook balance +alone, which would let any lane a client pointed at name the whole balance +and be signed for it. That is not something §2 grants: a relay is pinned, +not vetted, and there is no list to be thrown off. + +The alternative — persisting the +residual client-side — keeps better books but has no recovery when that +state is lost, and a client permanently locked out of a relay with no way +to clear it is the worse failure. + +### 17.2 The headroom guard admitted a frame, then sent a batch + +**Status: fixed** (`LanePayer::has_headroom` prices a real POST; +`dispatch_ok` re-evaluated per dispatch). + +The pre-dispatch guard asked whether one more *frame* fit inside the +credit line and then dispatched a full `batch_max` POST, and its answer +was computed once for an unbounded run of dispatches. Several concurrent +POSTs were each waved through against the same headroom; the relay +reserved every one of them, their sum crossed the line, and it answered +402 to a client whose own books said it had room. + +That refusal is the unpayable kind — the bytes are in flight, so nothing +is owed yet and settling changes nothing — and the batch returns having +spent an attempt per chunk, which is why handing batches back was +measured to make things worse rather than better. §7.2's whole point is +that the client sizes to fit instead of discovering the ceiling as a 402. + +The guard now prices the POST the scheduler will actually build. Where the +line holds only one POST this serialises the lane, which is the honest +answer: a credit line that fits 1.8 POSTs cannot have eight in flight. +Measured on a hard-mode relay, per-upload 402s went 11 → 1 and unpayable +refusals 6 → 0; the one remaining 402 is §17.1's carried debt, paid on +reconcile. + +`LaneAccount::max_body_bytes` had the same confusion — sizing against +`owed` while `has_headroom` and `would_exceed` both bind on `outstanding`. +It is only reached from tests today, but it would have reintroduced the +bug at its next caller. + +The first attempt at this fix gated dispatch on whether a *full* POST +would fit, which is worse: see §17.3, which it caused. + +### 17.3 §10.1's invariant does not hold at the line that binds + +**Status: fixed** (`Params::effective`, applied on both sides, with tests). + +`Params::validate` checks `min_cheque <= settle_every < max_outstanding`. +But `max_outstanding_plur` is only the *ceiling* on a credit line; the +line that actually binds an account is per batch, +`min(remaining_value / credit_ratio, ceiling)` (§10.3). For any batch +whose remaining value is under `min_cheque_plur * credit_ratio` — about +0.39 BZZ at the shipped defaults — the configured floor sits above +everything that account can ever owe, and the invariant quietly stops +holding. + +What follows is a permanent refusal. The account accrues to its cap, is +answered 402, and cannot write a cheque large enough to be accepted: +`next_cumulative()` returns `None` below the floor, and `/v1/pay` rejects +anything below it as dust. Nothing on either side is broken or dishonest +— the parameters simply cannot be satisfied. Observed live at a credit +line of 679,783,122,862 against a 3,900,000,000,000 floor, 5.7× short, +which halted an upload at 16 of 219 chunks with the relay reporting +`err=0`. + +This is exactly §10.3's small batch — the case the value-scaled credit +line exists to keep serving — so refusing it defeats the purpose of +scaling the line at all. + +The thresholds are now resolved against the line before they are applied: +settle at half of it, and never demand a cheque larger than that. Both +sides derive this from `(params, cap)`, and `cap` is already in the +challenge, so they agree without exchanging anything new. A generous line +keeps the configured values unchanged; only a line that cannot reach them +scales them down. + +Accepting a smaller cheque costs the relay nothing, which is what makes +this safe: cheques are cumulative, so a small one now does not force a +small cash-out later, and `hoverfly cashout --min-amount` already declines +to spend gas on a claim that is not worth collecting. The dust floor +belongs at redemption, where the gas is, not at acceptance. + +### 17.4 A lane refused for bytes in flight was parked for good + +**Status: fixed** (`unpayable_402` in the driver). + +The relay refuses on `owed + reserved`, but only `owed` is payable — +reservations are bodies it is still reading. A lane can therefore be over +its line while its *debt* is under the dust floor, and that is neither a +disagreement nor something a cheque fixes: those bytes simply have to +land. + +A 402 marks the lane `Unfunded`, and only a successful settle re-funds it +(§12). With nothing payable there is no settle, so the lane stayed parked +and the upload ended with chunks pending and the relay reporting `err=0` — +2 MiB stopping at 311 of 567 frames. The bigger the upload, the more +certain this is, because more POSTs are in flight when the line fills. + +The driver now distinguishes the two cases. Bytes in flight means busy: +re-fund the lane and let it resume when they clear. Nothing in flight and +nothing payable is the genuine disagreement, and still warns. Re-funding +cannot spin, because `has_headroom` binds on `outstanding` and so +dispatches nothing until the in-flight bytes actually clear. + +The same run surfaced the mirror of §17.1 in the reconcile itself: +adopting the relay's `owed` while a POST it had already booked was still +`pending` locally counted that body twice once the response closed, and +the cheque was refused as an overpayment (`credits 535680000000 but only +510240000000 is owed`). Reconciliation now deducts what is in flight. +Under-adopting is safe — the remainder is still owed and the next settle +collects it. + +### 17.5 A broken response stream made every later cheque bounce + +**Status: fixed** (`LaneAccount::sync_relay_debt`, one bounded re-present +in `LanePayer::pay`). + +§7.3's ack-tail cuts both ways. A POST whose response stream breaks was +still *read*, so the client bills it — but if the relay's task is +cancelled before it commits, the `Admitted` guard releases the reservation +and it books nothing. The client cannot tell that case from a clean one, +so it over-counts. + +The overshoot is not self-correcting: it rides on the cumulative, so every +later cheque carries it and is refused for the same reason, and the lane +never settles again. Seen the first time the relay was reached through a +real reverse proxy — two broken streams, then `cheque credits +212640000000 but only 148800000000 is owed`. + +The relay is the party deciding what it will accept, so the client yields +to it: on a rejection naming a smaller figure, re-read `/v1/account`, take +that number, and re-present once. Nothing was issued, so there is no +cumulative to be inconsistent with. The retry is bounded to a single +attempt, so a relay that keeps refusing cannot loop the client. + +This is the same yielding as §17.1 in the opposite direction, and +`forgive_phantom_debt` was already the total-loss case of it. + +### 17.6 The first POST of a run was sized before the debt was known + +**Status: fixed** (reconcile once per lane at setup). + +§17.1 made carried debt *recoverable*, via a 402 the client answers by +reconciling. It did not stop the client from walking into it. A fresh +process starts believing it owes nothing, so it sizes its first POST +against the entire credit line while the relay is already holding part of +it — and is refused immediately. + +Worse, the recovery could land in a hole: if the carried debt happens to +sit below the dust floor, the client reconciles, finds it cannot write an +acceptable cheque, and stops with the lane over its cap. Observed at 16 of +567 frames with 152,160,000,000 carried against a 416,771,800,039 line. + +Reading `/v1/account` once per lane at setup costs one GET and makes every +subsequent size correct, so the refusal never happens. §17.1's path +remains as the recovery for debt that appears mid-run. + +The same run showed that POST sizing has to be recomputed per *dispatch* +rather than per pass of the driver loop: the inner loop hands out several +assignments in a row, and a ceiling refreshed only on the outer pass sizes +the second and third of them as if each were alone on the lane. + +After all six fixes, uploads of 128 KiB through 4 MiB against a hard-mode +relay complete every frame, with no unpayable refusals and no rejected +cheques: + +| payload | frames acked | 402s | stuck | rejected cheques | +|--------:|-------------:|-----:|------:|-----------------:| +| 128 KiB | 43/43 | 0 | 0 | 0 | +| 256 KiB | 76/76 | 0 | 0 | 0 | +| 512 KiB | 151/151 | 0 | 0 | 0 | +| 1 MiB | 290/290 | 2 | 0 | 0 | +| 2 MiB | 567/567 | 2 | 0 | 0 | +| 4 MiB | 1122/1122 | 4 | 0 | 0 | + +The remaining 402s are the intended kind: the line genuinely fills, the +client pays or waits, and the lane resumes. + +Repeated over public HTTPS through a reverse proxy, with §17.6 in place so +the client knows its carried debt before sizing anything, the 402s go away +entirely — the client never reaches its cap because it never builds a body +that would cross it: + +| run | payload | frames acked | 402s | rejected cheques | +|----:|--------:|-------------:|-----:|-----------------:| +| 1 | 2 MiB | 567/567 | 0 | 0 | +| 2 | 2 MiB | 567/567 | 0 | 0 | +| 3 | 2 MiB | 567/567 | 0 | 0 | + +Each run settles to `owed: 0` on the relay, so the next carries nothing. +That is the intended steady state: 402 is the recovery path, not the +mechanism. diff --git a/src/batch.rs b/src/batch.rs index aa215ee..f12de64 100644 --- a/src/batch.rs +++ b/src/batch.rs @@ -655,14 +655,30 @@ struct EthRpc { http: reqwest::Client, } +/// One process-wide HTTP client for every `eth_call`. +/// +/// `EthRpc::new` is called per read (`read_batch`, `read_remaining_balance`, +/// `read_last_price`, …), and building a fresh `reqwest::Client` each time +/// means a fresh connection pool, so every read paid a full TLS handshake +/// and none were ever reused. On the pusher's hot path that is one handshake +/// per batch resolution; under a flood of unresolvable batch ids it was one +/// per frame. `reqwest::Client` is an `Arc` internally, so cloning it shares +/// the pool. +fn shared_http() -> &'static reqwest::Client { + static HTTP: std::sync::OnceLock = std::sync::OnceLock::new(); + HTTP.get_or_init(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("reqwest client") + }) +} + impl EthRpc { fn new(url: String) -> Self { Self { url, - http: reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .build() - .expect("reqwest client"), + http: shared_http().clone(), } } @@ -1053,3 +1069,533 @@ mod tests { assert_eq!(amount_for_duration(1, 86400), U256::from(17290u64)); } } + +// ────────────────────────────────────────────────────────────────────── +// Chequebook reads (metered relay — docs/pusher-incentives.md Stage 1) +// ────────────────────────────────────────────────────────────────────── + +sol! { + // ERC20SimpleSwap / SimpleSwapFactory, from + // github.com/ethersphere/swap-swear-and-swindle. + function issuer() external view returns (address); + function paidOut(address beneficiary) external view returns (uint256); + function bounced() external view returns (bool); + // NOT `balance()`. `liquidBalance() = balance() - totalHardDeposit`, and + // `liquidBalanceFor(b) = liquidBalance() + hardDeposits[b].amount`, which + // is what `_cashChequeInternal` actually pays against. Bee checks + // `balance()`, counting *other* beneficiaries' hard deposits as our + // coverage — unsound in general, invisible only because bee never places + // any (incentives §11.2). + function liquidBalanceFor(address beneficiary) external view returns (uint256); + function deployedContracts(address who) external view returns (bool); +} + +/// Canonical `SimpleSwapFactory` per chain (`bee/pkg/config/chain.go:66,89`). +/// **Hardcoded on purpose.** A factory address supplied by the client lets it +/// present a contract that returns a forged `issuer()` and +/// `liquidBalanceFor()` and implements `cashChequeBeneficiary` as a no-op — +/// total compromise for one `eth_call` saved (incentives §6). +pub const GNOSIS_SWAP_FACTORY: &str = "0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420"; +pub const SEPOLIA_SWAP_FACTORY: &str = "0x0fF044F6bB4F684a5A149B46D7eC03ea659F98A1"; + +/// The factory for a chain, or `None` if we have no vetted address for it — +/// in which case metered mode must not run, rather than fall back to +/// something the client names. +pub fn swap_factory_for_chain(chain_id: u64) -> Option
{ + let s = match chain_id { + 100 => GNOSIS_SWAP_FACTORY, + 11155111 => SEPOLIA_SWAP_FACTORY, + _ => return None, + }; + s.parse().ok() +} + +/// What the relay needs to know about a chequebook to accept a cheque +/// drawn on it. +#[derive(Debug, Clone, Copy)] +pub struct ChequebookState { + pub issuer: Address, + /// Everything this beneficiary could actually cash right now. + pub liquid_for_us: U256, + /// Already cashed by this beneficiary; the cheque's cumulative must + /// exceed it or there is nothing left to draw. + pub paid_out_to_us: U256, + /// Set permanently, contract-wide, the first time any cheque could not + /// be paid in full. Readable state rather than an event you had to have + /// been watching for (incentives §11.2). + pub bounced: bool, +} + +/// Is this address a chequebook the canonical factory deployed? +/// +/// Cache this forever per address on a `true`, and negative-cache a `false` +/// — it is the first chain read an unauthenticated `/v1/pay` can reach, so +/// an uncached miss is a one-RPC-per-request amplifier (incentives §11.6). +pub async fn is_deployed_chequebook( + rpc_url: &str, + factory: Address, + chequebook: Address, +) -> Result { + EthRpc::new(rpc_url.to_string()) + .call_view(factory, deployedContractsCall { who: chequebook }) + .await +} + +/// Read the four values that decide whether a cheque is worth anything — +/// in **one** JSON-RPC round trip. +/// +/// They used to be four sequential `eth_call`s, which put four round trips +/// on the critical path of every `/v1/pay`. They are all reads against the +/// same block, so there is no reason for them to be sequential: batching +/// them makes the miss case one request, and the caller (`Metered`) caches +/// on top so the honest path is usually zero. +pub async fn read_chequebook_state( + rpc_url: &str, + chequebook: Address, + beneficiary: Address, +) -> Result { + let rpc = EthRpc::new(rpc_url.to_string()); + let calls = [ + issuerCall {}.abi_encode(), + liquidBalanceForCall { beneficiary }.abi_encode(), + paidOutCall { beneficiary }.abi_encode(), + bouncedCall {}.abi_encode(), + ]; + let out = rpc.batch_call_view(chequebook, &calls).await?; + let dec = |i: usize| -> Result<&Vec, BatchError> { + out.get(i) + .ok_or_else(|| BatchError::Rpc("short batch response".into())) + }; + Ok(ChequebookState { + issuer: issuerCall::abi_decode_returns(dec(0)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + liquid_for_us: liquidBalanceForCall::abi_decode_returns(dec(1)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + paid_out_to_us: paidOutCall::abi_decode_returns(dec(2)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + bounced: bouncedCall::abi_decode_returns(dec(3)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + }) +} + +impl EthRpc { + /// Several `eth_call`s to one contract in a single JSON-RPC batch. + /// + /// Results come back keyed by request id rather than in order — the + /// spec permits a server to reorder them, and some do — so they are + /// re-sorted before being returned. + async fn batch_call_view( + &self, + to: Address, + calls: &[Vec], + ) -> Result>, BatchError> { + #[derive(Serialize)] + struct BatchReq<'a> { + jsonrpc: &'a str, + id: usize, + method: &'a str, + params: (CallObj, &'a str), + } + let to_hex = format!("0x{}", hex::encode(to)); + let reqs: Vec = calls + .iter() + .enumerate() + .map(|(i, data)| BatchReq { + jsonrpc: "2.0", + id: i, + method: "eth_call", + params: ( + CallObj { + from: format!("0x{}", hex::encode(Address::ZERO)), + to: to_hex.clone(), + data: format!("0x{}", hex::encode(data)), + }, + "latest", + ), + }) + .collect(); + let resp: Vec = self + .http + .post(&self.url) + .json(&reqs) + .send() + .await? + .json() + .await?; + if resp.len() != calls.len() { + return Err(BatchError::Rpc(format!( + "batch eth_call: sent {} requests, got {} responses", + calls.len(), + resp.len() + ))); + } + let mut out = vec![Vec::new(); calls.len()]; + for item in resp { + if let Some(err) = item.get("error") { + return Err(BatchError::Rpc(format!("batch eth_call: {err}"))); + } + let id = item + .get("id") + .and_then(|i| i.as_u64()) + .ok_or_else(|| BatchError::Rpc("batch eth_call: missing id".into()))? + as usize; + let hex_str = item + .get("result") + .and_then(|r| r.as_str()) + .ok_or_else(|| BatchError::Rpc("batch eth_call: missing result".into()))?; + let slot = out + .get_mut(id) + .ok_or_else(|| BatchError::Rpc(format!("batch eth_call: bad id {id}")))?; + *slot = hex::decode(hex_str.trim_start_matches("0x"))?; + } + Ok(out) + } +} + +#[cfg(test)] +mod chequebook_binding_tests { + use super::*; + + /// Selectors are what the node dispatches on, so a wrong one silently + /// reads a *different* function rather than failing. Pinned against + /// `keccak256(signature)[..4]`. + #[test] + fn selectors_match_the_solidity_signatures() { + use alloy_sol_types::SolCall; + for (got, sig) in [ + (issuerCall::SELECTOR, "issuer()"), + (paidOutCall::SELECTOR, "paidOut(address)"), + (bouncedCall::SELECTOR, "bounced()"), + (liquidBalanceForCall::SELECTOR, "liquidBalanceFor(address)"), + ( + deployedContractsCall::SELECTOR, + "deployedContracts(address)", + ), + ] { + let want: [u8; 32] = ::digest(sig.as_bytes()).into(); + assert_eq!(got, want[..4], "selector drift for {sig}"); + } + } + + /// A relay must never accept a factory address from the wire, and must + /// refuse to run metered on a chain it has no vetted factory for. + #[test] + fn only_known_chains_have_a_factory() { + assert!(swap_factory_for_chain(100).is_some(), "gnosis"); + assert!(swap_factory_for_chain(11155111).is_some(), "sepolia"); + assert!( + swap_factory_for_chain(1).is_none(), + "mainnet: no vetted factory" + ); + assert!(swap_factory_for_chain(31337).is_none(), "local devnet"); + } +} + +// ────────────────────────────────────────────────────────────────────── +// Chequebook deployment (docs/pusher-incentives.md §14) +// ────────────────────────────────────────────────────────────────────── + +sol! { + // SimpleSwapFactory.deploySimpleSwap(issuer, defaultHardDepositTimeoutDuration, salt) + function deploySimpleSwap( + address issuer, + uint256 defaultHardDepositTimeoutDuration, + bytes32 salt + ) external returns (address); + + event SimpleSwapDeployed(address contractAddress); + + function transfer(address to, uint256 amount) external returns (bool); +} + +#[derive(Debug, Clone)] +pub struct DeployChequebookParams { + pub rpc_url: String, + pub chain_id: u64, + pub factory: Address, + /// Must be the **batch owner's** address (§6): a relay only accepts a + /// cheque whose chequebook `issuer()` equals the account it billed. + pub issuer: Address, + pub receipt_timeout: std::time::Duration, +} + +#[derive(Debug, Clone)] +pub struct DeployedChequebook { + pub address: Address, + pub tx: B256, + /// True when the address already held code before we sent anything — + /// the deploy is deterministic in `(issuer, timeout, salt)`, so a repeat + /// with the same salt would revert rather than make a second one. + pub already_deployed: bool, +} + +/// Deploy a chequebook through bee's canonical factory. +/// +/// **Deliberately its own command, never a side effect of an upload.** +/// Deploying a contract is irreversible and spends real funds; an upload +/// that did it silently because some lane quoted a price would fire on the +/// first metered lane a user ever met, before they had decided they wanted +/// to pay at all. +/// +/// The hard-deposit timeout is **0**, matching what bee deploys +/// (`init.go:169`). That means deposits are not actually locked — see +/// §11.2 — but it keeps us byte-compatible with every bee chequebook, and +/// a non-zero timeout can be set later per beneficiary via +/// `setCustomHardDepositTimeout` if secured mode is ever wanted. +pub async fn deploy_chequebook( + signer: &PrivateKeySigner, + params: DeployChequebookParams, +) -> Result { + let rpc = EthRpc::new(params.rpc_url.clone()); + + let mut salt_bytes = [0u8; 32]; + getrandom::fill(&mut salt_bytes).map_err(|e| BatchError::Rpc(format!("getrandom: {e}")))?; + let salt = B256::from(salt_bytes); + + let call = deploySimpleSwapCall { + issuer: params.issuer, + defaultHardDepositTimeoutDuration: U256::ZERO, + salt, + }; + + // Simulate first. The factory returns the address it *would* create, so + // a revert (bad issuer, salt collision) surfaces here for free instead + // of as a burnt transaction. + let predicted: Address = rpc.call_view(params.factory, call.clone()).await?; + let existing = rpc.code_len(predicted).await?; + if existing > 0 { + return Ok(DeployedChequebook { + address: predicted, + tx: B256::ZERO, + already_deployed: true, + }); + } + + let tx = rpc + .send_signed(signer, params.chain_id, params.factory, &call.abi_encode()) + .await?; + rpc.wait_for_success(tx, params.receipt_timeout).await?; + + // Trust the chain, not the simulation: read the address back out of the + // receipt's `SimpleSwapDeployed` log. + let deployed = rpc.find_deployed_chequebook(tx).await?.unwrap_or(predicted); + + // The relay checks this before accepting any cheque (§6), so checking it + // here turns "your cheques are silently refused" into a deploy-time + // error. + let issuer: Address = rpc.call_view(deployed, issuerCall {}).await?; + if issuer != params.issuer { + return Err(BatchError::Rpc(format!( + "deployed chequebook {deployed} has issuer {issuer}, expected {}", + params.issuer + ))); + } + Ok(DeployedChequebook { + address: deployed, + tx, + already_deployed: false, + }) +} + +/// Move BZZ into a chequebook. Plain ERC-20 transfer — the contract holds +/// whatever balance the token says it does, and `liquidBalanceFor` is +/// derived from it. +pub async fn fund_chequebook( + signer: &PrivateKeySigner, + rpc_url: &str, + chain_id: u64, + bzz_token: Address, + chequebook: Address, + amount: U256, + receipt_timeout: std::time::Duration, +) -> Result { + let rpc = EthRpc::new(rpc_url.to_string()); + let from = signer.address(); + let balance: U256 = rpc + .call_view(bzz_token, balanceOfCall { account: from }) + .await?; + if balance < amount { + return Err(BatchError::InsufficientBalance { + have: balance, + need: amount, + }); + } + // Refuse to fund something that is not a chequebook: a mistyped address + // sends BZZ somewhere unrecoverable. + let issuer: Address = rpc + .call_view(chequebook, issuerCall {}) + .await + .map_err(|e| { + BatchError::Rpc(format!( + "{chequebook} does not answer issuer() — is it a chequebook? ({e})" + )) + })?; + if issuer != from { + return Err(BatchError::Rpc(format!( + "chequebook {chequebook} is issued by {issuer}, not {from}: only the issuer \ + can ever withdraw, so funding it would strand the deposit" + ))); + } + let call = transferCall { + to: chequebook, + amount, + } + .abi_encode(); + let tx = rpc.send_signed(signer, chain_id, bzz_token, &call).await?; + rpc.wait_for_success(tx, receipt_timeout).await?; + Ok(tx) +} + +impl EthRpc { + /// `eth_getCode` length, for "is there already a contract here". + async fn code_len(&self, addr: Address) -> Result { + let hex_str: String = self + .raw( + "eth_getCode", + (format!("0x{}", hex::encode(addr)), "latest"), + ) + .await?; + Ok(hex_str.trim_start_matches("0x").len() / 2) + } + + /// Pull the chequebook address out of a deploy receipt's logs. + async fn find_deployed_chequebook(&self, tx: B256) -> Result, BatchError> { + let receipt: serde_json::Value = self + .raw( + "eth_getTransactionReceipt", + (format!("0x{}", hex::encode(tx)),), + ) + .await?; + let topic = SimpleSwapDeployed::SIGNATURE_HASH; + let Some(logs) = receipt.get("logs").and_then(|l| l.as_array()) else { + return Ok(None); + }; + for log in logs { + let topics: Vec = log + .get("topics") + .and_then(|t| t.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default(); + if topics + .first() + .map(|t| t.trim_start_matches("0x").to_lowercase()) + != Some(hex::encode(topic)) + { + continue; + } + // Non-indexed single address parameter: it is the data word. + if let Some(data) = log.get("data").and_then(|d| d.as_str()) { + let raw = hex::decode(data.trim_start_matches("0x")) + .map_err(|e| BatchError::Rpc(format!("log data hex: {e}")))?; + if raw.len() >= 32 { + return Ok(Some(Address::from_slice(&raw[12..32]))); + } + } + } + Ok(None) + } +} + +// ────────────────────────────────────────────────────────────────────── +// Cashing out (docs/pusher-incentives.md §14 Stage 2) +// ────────────────────────────────────────────────────────────────────── + +sol! { + function cashChequeBeneficiary( + address recipient, + uint256 cumulativePayout, + bytes issuerSig + ) external; +} + +/// What a cheque is actually worth right now, before spending gas on it. +#[derive(Debug, Clone, Copy)] +pub struct CashoutQuote { + /// `cumulative - paidOut(beneficiary)`: what is still unclaimed. + pub requested_plur: U256, + /// `min(requested, liquidBalanceFor(beneficiary))` — what the contract + /// would actually transfer. + pub payable_plur: U256, + /// True when the chequebook cannot cover the whole claim. The cashout + /// still succeeds and takes what is there — `_cashChequeInternal` does + /// not revert — but it sets `bounced` permanently. + pub would_bounce: bool, + pub already_bounced: bool, +} + +/// Price a cheque without sending anything. +/// +/// Worth doing first because gas is spent whether or not the cheque is +/// worth cashing, and a cumulative at or below `paidOut` transfers nothing +/// at all. +pub async fn quote_cashout( + rpc_url: &str, + chequebook: Address, + beneficiary: Address, + cumulative_plur: U256, +) -> Result { + let st = read_chequebook_state(rpc_url, chequebook, beneficiary).await?; + let requested_plur = cumulative_plur.saturating_sub(st.paid_out_to_us); + let payable_plur = requested_plur.min(st.liquid_for_us); + Ok(CashoutQuote { + requested_plur, + payable_plur, + would_bounce: payable_plur < requested_plur, + already_bounced: st.bounced, + }) +} + +/// Present a cheque on-chain. +/// +/// **Must be sent by the beneficiary**: `cashChequeBeneficiary` passes +/// `msg.sender` as the beneficiary into `_cashChequeInternal`, so the +/// signing key here is the EOA the cheques were made out to — never the +/// relay's, which is why this is a separate command run somewhere else +/// (§6). `recipient` is where the BZZ lands and may differ. +pub async fn cash_cheque( + beneficiary_signer: &PrivateKeySigner, + rpc_url: &str, + chain_id: u64, + chequebook: Address, + recipient: Address, + cumulative_plur: U256, + signature: &[u8; 65], + receipt_timeout: std::time::Duration, +) -> Result { + // The contract verifies through OpenZeppelin's ECDSA, which rejects + // high-s and v ∉ {27,28}. A non-canonical signature reverts and burns + // the gas, so refuse it here where it costs nothing. + crate::signer::check_canonical_signature(signature) + .map_err(|e| BatchError::Rpc(format!("stored cheque is not cashable: {e}")))?; + let rpc = EthRpc::new(rpc_url.to_string()); + let call = cashChequeBeneficiaryCall { + recipient, + cumulativePayout: cumulative_plur, + issuerSig: signature.to_vec().into(), + } + .abi_encode(); + let tx = rpc + .send_signed(beneficiary_signer, chain_id, chequebook, &call) + .await?; + rpc.wait_for_success(tx, receipt_timeout).await?; + Ok(tx) +} + +#[cfg(test)] +mod cashout_tests { + use super::*; + + #[test] + fn the_cashout_selector_matches_the_contract() { + use alloy_sol_types::SolCall; + let want: [u8; 32] = ::digest( + "cashChequeBeneficiary(address,uint256,bytes)".as_bytes(), + ) + .into(); + assert_eq!(cashChequeBeneficiaryCall::SELECTOR, want[..4]); + } +} diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index a18dc40..07afcd5 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -507,6 +507,70 @@ enum Commands { value_name = "URL" )] rpc_url: String, + + // ── Metered mode (docs/pusher-incentives.md Stage 1) ────────── + /// Bill clients for relayed bytes with off-chain SWAP cheques. + /// Off by default: `open` is today's unmetered behaviour and is + /// what the production lanes run. + #[arg(long)] + meter: bool, + + /// Hostname(s) this relay is reached at. **Required with + /// `--meter`.** The admission challenge binds the origin a client + /// dialled, and the relay compares it against *this* value — never + /// against the `Host` header, which the same client supplies and + /// which would make the check a no-op that silently reopens + /// cross-relay replay. + #[arg(long, value_name = "HOST")] + origin: Vec, + + /// EOA that cheques are made out to. The relay holds this address + /// only, never the key: cashing out happens elsewhere. + #[arg(long, value_name = "0xADDR")] + beneficiary: Option, + + /// Settlement chain. Pins the EIP-712 domain and selects the + /// hardcoded SimpleSwapFactory; a chain with no vetted factory + /// cannot run metered. + #[arg(long, default_value_t = 100, value_name = "ID")] + chequebook_chain: u64, + + /// Directory for the ledger and relay secret. **Required with + /// `--meter`**: without durable state a client re-presents its + /// last cheque after every restart and is credited the full + /// cumulative again, which is unlimited free service from one + /// signature. + #[arg(long, value_name = "DIR")] + state_dir: Option, + + /// PLUR per KiB of body admitted. + #[arg(long, value_name = "PLUR")] + price_plur_per_kib: Option, + + /// Smallest cheque accepted, to bound RPC cost per unit of value. + #[arg(long, value_name = "PLUR")] + min_cheque_plur: Option, + + /// Debt at which a client is expected to settle. + #[arg(long, value_name = "PLUR")] + settle_every_plur: Option, + + /// Global ceiling on a credit line. The actual cap is per batch: + /// `min(batch_remaining_value / credit_ratio, this)`. + #[arg(long, value_name = "PLUR")] + max_outstanding_plur: Option, + + /// Credit line = batch remaining on-chain value ÷ this. The Sybil + /// margin is this ratio by construction, independent of batch size. + #[arg(long, value_name = "N")] + credit_ratio: Option, + + /// Enforce 402 when an account is over its cap. Default is soft + /// mode: meter, report, and serve anyway — which is what Stage 1 + /// ships, so the overshoot rate can be measured against live + /// traffic before anyone is refused. + #[arg(long)] + meter_hard: bool, }, /// Run a long-lived daemon that holds a warm session pool across @@ -738,6 +802,70 @@ enum Commands { action: BatchAction, }, + /// Cash cheques a metered relay has accepted. + /// + /// Reads the relay's ledger, prices each held cheque on-chain, and + /// presents the ones worth collecting. **Run this somewhere other than + /// the relay**: `cashChequeBeneficiary` must be sent *by* the + /// beneficiary, so it needs the beneficiary's private key — which is + /// exactly the key a relay box is designed never to hold. Copy the + /// ledger file (or point `--state-dir` at a snapshot of it) and run + /// this from a machine that does. + /// + /// A cheque is cumulative, so only the newest one per chequebook is + /// ever presented; gas is paid once per chequebook, not once per + /// cheque received. + /// + /// Behind the `pusher` feature because it reads the *relay's* ledger: + /// without a relay there are no cheques to cash. + #[cfg(all(unix, feature = "pusher"))] + Cashout { + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] + rpc_url: String, + /// The **beneficiary's** private key — the EOA cheques were made + /// out to, and the only address the contract will pay. + #[arg(long, value_name = "KEY")] + key: String, + /// Relay state directory containing `ledger.json`. + #[arg(long, value_name = "DIR")] + state_dir: std::path::PathBuf, + /// Where the BZZ should land. Defaults to the beneficiary. + #[arg(long, value_name = "ADDR")] + recipient: Option, + /// Skip cheques worth less than this in BZZ. Cashing costs ~300k + /// gas whatever the amount, so tiny cheques are worth less than + /// collecting them (§9.3). + #[arg(long, default_value = "0.25", value_name = "BZZ")] + min_amount: String, + /// Only one chequebook, rather than everything in the ledger. + #[arg(long, value_name = "ADDR")] + chequebook: Option, + /// Price everything and print it, sending nothing. + #[arg(long)] + dry_run: bool, + #[arg(long, default_value_t = 100, value_name = "ID")] + chain_id: u64, + #[arg(long, default_value_t = 300, value_name = "SECS")] + timeout: u64, + }, + + /// Manage a SWAP chequebook — the contract that pays metered pusher + /// relays (`docs/pusher-incentives.md`). + /// + /// Deliberately its own command rather than something an upload does + /// for you: deploying a contract is irreversible and spends real funds, + /// and an upload that deployed one silently because a lane quoted a + /// price would fire before you had decided you wanted to pay at all. + #[cfg(unix)] + Chequebook { + #[command(subcommand)] + action: ChequebookAction, + }, + /// Bridge funds from another chain to xDAI + BZZ on Gnosis via Relay. /// /// Solves the setup chicken-and-egg: `batch create` needs the signer's @@ -837,6 +965,79 @@ enum Commands { } #[cfg(unix)] +#[derive(Subcommand)] +#[cfg(unix)] +enum ChequebookAction { + /// Deploy a chequebook through bee's canonical SimpleSwapFactory. + /// + /// The issuer is `--key`'s address, and it must be the **batch owner**: + /// a metered relay only accepts a cheque whose chequebook `issuer()` + /// equals the account it billed. The issuer is also the only address + /// that can ever `withdraw()`, so never deploy with a key you do not + /// exclusively control. + /// + /// Deploys with a hard-deposit timeout of 0, matching every bee + /// chequebook. Deposits are therefore not locked; see §11.2. + Deploy { + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] + rpc_url: String, + /// Private key (hex, 32 bytes). Its address becomes the issuer. + #[arg(long, value_name = "KEY")] + key: String, + #[arg(long, default_value_t = 100, value_name = "ID")] + chain_id: u64, + /// Seconds to wait for the deploy receipt. + #[arg(long, default_value_t = 300, value_name = "SECS")] + timeout: u64, + }, + /// Deposit BZZ into an existing chequebook. + /// + /// Separate from `deploy` so topping up later does not mean pretending + /// to redeploy. Refuses to send to an address that does not answer + /// `issuer()`, or whose issuer is not `--key` — only the issuer can + /// withdraw, so funding someone else's chequebook strands the deposit. + Fund { + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] + rpc_url: String, + #[arg(long, value_name = "KEY")] + key: String, + #[arg(long, value_name = "ADDR")] + chequebook: String, + /// BZZ to deposit (decimal, e.g. `0.05`). BZZ has 16 decimals. + #[arg(long, value_name = "BZZ")] + amount: String, + #[arg(long, default_value_t = 100, value_name = "ID")] + chain_id: u64, + #[arg(long, default_value_t = 300, value_name = "SECS")] + timeout: u64, + }, + /// Print a chequebook's on-chain state: issuer, balance liquid to a + /// given beneficiary, what it has already paid out, and whether it has + /// ever bounced. + Status { + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] + rpc_url: String, + #[arg(long, value_name = "ADDR")] + chequebook: String, + /// Beneficiary to report liquidity for. Defaults to the zero + /// address, which reports the plain liquid balance. + #[arg(long, value_name = "ADDR")] + beneficiary: Option, + }, +} + #[derive(Subcommand)] enum BatchAction { /// Create a new postage stamp batch on-chain. @@ -1782,9 +1983,60 @@ async fn main() -> Result<(), Box> { }; let n_chunks = streamer.total_chunks(); let progress = make_progress_bar(); - let root = - hoverfly::client::push_stream_via_pushers(&pusher, streamer, progress.as_ref()) + // Metered lanes: pay only when the user has explicitly given + // a chequebook. Without one we push exactly as before and a + // metered lane simply meters us in soft mode — nothing here + // deploys or funds anything on its own. + let pay_cfg = match cli.chequebook.as_ref() { + Some(cb_hex) => { + let cb = + parse_address_hex(cb_hex).map_err(|e| format!("--chequebook: {e}"))?; + // The relay checks funding against `liquidBalanceFor` + // at accept time; we check total issuance against the + // same balance before signing, so we never hand over a + // cheque that cannot be cashed. + let st = hoverfly::batch::read_chequebook_state( + &rpc_url, + alloy_primitives::Address::from(cb), + alloy_primitives::Address::ZERO, + ) .await?; + let store = hoverfly::cheques::ChequeStore::load_or_create( + &cli.cheques_file, + cb, + ) + .map_err(|e| format!("loading {}: {e}", cli.cheques_file.display()))?; + eprintln!( + "metered: chequebook=0x{} liquid={} PLUR", + hex::encode(cb), + st.liquid_for_us + ); + // `batch` is the hex string the user passed; the + // challenge binds the raw 32 bytes. + let batch_raw = hex::decode(batch.trim_start_matches("0x")) + .map_err(|e| format!("--batch: {e}"))?; + let batch_id: [u8; 32] = batch_raw + .as_slice() + .try_into() + .map_err(|_| "--batch must be 32 bytes".to_string())?; + Some(hoverfly::payer::PaymentConfig { + signer: signer.clone(), + batch: batch_id, + chequebook: cb, + chain_id: cli.chequebook_chain_id, + cheques: std::sync::Arc::new(std::sync::Mutex::new(store)), + balance_plur: u128::try_from(st.liquid_for_us).unwrap_or(u128::MAX), + }) + } + None => None, + }; + let root = hoverfly::client::push_stream_via_pushers_paid( + &pusher, + streamer, + progress.as_ref(), + pay_cfg.as_ref(), + ) + .await?; drop(progress); let elapsed = upload_started.elapsed(); let root_hex = hex::encode(root.as_bytes()); @@ -2063,6 +2315,17 @@ async fn main() -> Result<(), Box> { peerlist, probe, rpc_url, + meter, + origin, + beneficiary, + chequebook_chain, + state_dir, + price_plur_per_kib, + min_cheque_plur, + settle_every_plur, + max_outstanding_plur, + credit_ratio, + meter_hard, } => { // Premined overlay nonce. On ephemeral-FS hosts (Render, // Lambda) there is no persistent `--nonce-file`, so a random @@ -2092,6 +2355,81 @@ async fn main() -> Result<(), Box> { let node_identity = std::env::var("HOVERFLY_PUSHER_IDENTITY") .ok() .filter(|s| !s.trim().is_empty()); + // Metered mode is all-or-nothing: every precondition is checked + // here and in `build_metered`, and a failure refuses to start + // rather than serving a half-configured meter that a paying + // client would discover the hard way. + // Env fallbacks so a deployed relay can be switched to metered + // without changing its start command — the same pattern + // HOVERFLY_PUSH_POOL and HOVERFLY_PUSHER_IDENTITY already use, + // and the only lever available on hosts where the command line + // is fixed by the platform. + let envs = |k: &str| std::env::var(k).ok().filter(|s| !s.trim().is_empty()); + let meter = meter || envs("HOVERFLY_METER").is_some_and(|v| v != "0"); + let meter_hard = meter_hard || envs("HOVERFLY_METER_HARD").is_some_and(|v| v != "0"); + let origin = if origin.is_empty() { + envs("HOVERFLY_METER_ORIGIN") + .map(|v| v.split(',').map(|s| s.trim().to_string()).collect()) + .unwrap_or_default() + } else { + origin + }; + let beneficiary = beneficiary.or_else(|| envs("HOVERFLY_METER_BENEFICIARY")); + let state_dir = state_dir.or_else(|| envs("HOVERFLY_METER_STATE_DIR").map(Into::into)); + let parse_env_plur = |k: &str| -> Result, String> { + match envs(k) { + Some(v) => v.parse().map(Some).map_err(|e| format!("{k}: {e}")), + None => Ok(None), + } + }; + let price_plur_per_kib = + price_plur_per_kib.or(parse_env_plur("HOVERFLY_METER_PRICE_PLUR_PER_KIB")?); + let min_cheque_plur = + min_cheque_plur.or(parse_env_plur("HOVERFLY_METER_MIN_CHEQUE_PLUR")?); + let settle_every_plur = + settle_every_plur.or(parse_env_plur("HOVERFLY_METER_SETTLE_EVERY_PLUR")?); + let max_outstanding_plur = + max_outstanding_plur.or(parse_env_plur("HOVERFLY_METER_MAX_OUTSTANDING_PLUR")?); + let credit_ratio = credit_ratio.or(parse_env_plur("HOVERFLY_METER_CREDIT_RATIO")?); + let meter_opts = if meter { + let mut params = hoverfly::meter::Params::default(); + if let Some(v) = price_plur_per_kib { + params.price_plur_per_kib = v; + } + if let Some(v) = min_cheque_plur { + params.min_cheque_plur = v; + } + if let Some(v) = settle_every_plur { + params.settle_every_plur = v; + } + if let Some(v) = max_outstanding_plur { + params.max_outstanding_plur = v; + } + if let Some(v) = credit_ratio { + params.credit_ratio = v; + } + let beneficiary = beneficiary + .as_deref() + .ok_or("--meter requires --beneficiary")?; + let raw = hex::decode(beneficiary.trim_start_matches("0x")) + .map_err(|e| format!("--beneficiary: {e}"))?; + if raw.len() != 20 { + return Err("--beneficiary must be a 20-byte address".into()); + } + let mut b = [0u8; 20]; + b.copy_from_slice(&raw); + Some(hoverfly::pusher::MeterOpts { + origins: origin, + beneficiary: b, + chain_id: chequebook_chain, + params, + state_dir: state_dir + .ok_or("--meter requires --state-dir (durable state is mandatory)")?, + hard_mode: meter_hard, + }) + } else { + None + }; hoverfly::pusher::run(hoverfly::pusher::PusherOpts { listen, peerlist, @@ -2101,6 +2439,7 @@ async fn main() -> Result<(), Box> { rpc_url, node_identity, transport: cfg, + meter: meter_opts, }) .await?; } @@ -2528,6 +2867,217 @@ async fn main() -> Result<(), Box> { } #[cfg(unix)] + #[cfg(all(unix, feature = "pusher"))] + Commands::Cashout { + rpc_url, + key, + state_dir, + recipient, + min_amount, + chequebook, + dry_run, + chain_id, + timeout, + } => { + let signer = parse_signer(&key)?; + let beneficiary = signer.address(); + let recipient: alloy_primitives::Address = match recipient { + Some(r) => r.parse().map_err(|e| format!("--recipient: {e}"))?, + None => beneficiary, + }; + let floor = parse_bzz_amount(&min_amount)?; + let only: Option = match chequebook { + Some(c) => Some(c.parse().map_err(|e| format!("--chequebook: {e}"))?), + None => None, + }; + let ledger_path = state_dir.join("ledger.json"); + let ledger = hoverfly::ledger::Ledger::load_or_create(&ledger_path) + .map_err(|e| format!("reading {}: {e}", ledger_path.display()))?; + let held = ledger.held_cheques(); + println!( + "beneficiary 0x{} recipient 0x{} ({} cheque(s) held)", + hex::encode(beneficiary), + hex::encode(recipient), + held.len() + ); + if held.is_empty() { + println!("nothing to cash"); + return Ok(()); + } + let mut cashed = 0usize; + let mut skipped = 0usize; + for (account, cb, cheque) in held { + let cb_addr = alloy_primitives::Address::from(cb); + if only.is_some_and(|o| o != cb_addr) { + continue; + } + let cumulative = alloy_primitives::U256::from(cheque.cumulative_plur); + let q = hoverfly::batch::quote_cashout( + &rpc_url, + cb_addr, + alloy_primitives::Address::from(beneficiary), + cumulative, + ) + .await?; + println!(); + println!( + "chequebook 0x{} (account 0x{})", + hex::encode(cb), + hex::encode(account) + ); + println!( + " cumulative {cheque_cumulative}", + cheque_cumulative = cheque.cumulative_plur + ); + println!(" unclaimed {}", q.requested_plur); + println!( + " payable now {}{}", + q.payable_plur, + if q.would_bounce { + " <- chequebook cannot cover the claim" + } else { + "" + } + ); + if q.already_bounced { + println!(" NOTE: this chequebook has bounced before"); + } + if cheque.signature == [0u8; 65] { + println!(" SKIP: no stored signature (ledger predates signature persistence)"); + skipped += 1; + continue; + } + if q.payable_plur < floor { + println!( + " SKIP: below --min-amount ({min_amount} BZZ); gas would cost more than this collects" + ); + skipped += 1; + continue; + } + if dry_run { + println!(" DRY RUN: would cash {}", q.payable_plur); + continue; + } + let tx = hoverfly::batch::cash_cheque( + &signer, + &rpc_url, + chain_id, + cb_addr, + recipient, + cumulative, + &cheque.signature, + std::time::Duration::from_secs(timeout), + ) + .await?; + println!(" CASHED tx 0x{}", hex::encode(tx)); + cashed += 1; + } + println!(); + println!("{cashed} cashed, {skipped} skipped"); + } + #[cfg(unix)] + Commands::Chequebook { action } => match action { + ChequebookAction::Deploy { + rpc_url, + key, + chain_id, + timeout, + } => { + let signer = parse_signer(&key)?; + let issuer = signer.address(); + let factory = + hoverfly::batch::swap_factory_for_chain(chain_id).ok_or_else(|| { + format!( + "no vetted SimpleSwapFactory for chain {chain_id} — a factory address \ + must never be guessed, since a fake one can return a forged issuer()" + ) + })?; + println!("deploying chequebook: issuer 0x{}", hex::encode(issuer)); + println!(" factory 0x{}", hex::encode(factory)); + let out = hoverfly::batch::deploy_chequebook( + &signer, + hoverfly::batch::DeployChequebookParams { + rpc_url, + chain_id, + factory, + issuer, + receipt_timeout: std::time::Duration::from_secs(timeout), + }, + ) + .await?; + if out.already_deployed { + println!(" already deployed at 0x{}", hex::encode(out.address)); + } else { + println!(" tx 0x{}", hex::encode(out.tx)); + println!(" deployed 0x{}", hex::encode(out.address)); + } + println!(); + println!("Fund it before it can pay anything:"); + println!( + " hoverfly chequebook fund --key --chequebook 0x{} --amount 0.05", + hex::encode(out.address) + ); + } + ChequebookAction::Fund { + rpc_url, + key, + chequebook, + amount, + chain_id, + timeout, + } => { + let signer = parse_signer(&key)?; + let cb: alloy_primitives::Address = chequebook + .parse() + .map_err(|e| format!("--chequebook: {e}"))?; + let plur = parse_bzz_amount(&amount)?; + let bzz: alloy_primitives::Address = hoverfly::batch::MAINNET_BZZ_TOKEN + .parse() + .map_err(|e| format!("bzz token: {e}"))?; + println!( + "funding 0x{} with {amount} BZZ ({plur} PLUR)", + hex::encode(cb) + ); + let tx = hoverfly::batch::fund_chequebook( + &signer, + &rpc_url, + chain_id, + bzz, + cb, + plur, + std::time::Duration::from_secs(timeout), + ) + .await?; + println!(" tx 0x{}", hex::encode(tx)); + } + ChequebookAction::Status { + rpc_url, + chequebook, + beneficiary, + } => { + let cb: alloy_primitives::Address = chequebook + .parse() + .map_err(|e| format!("--chequebook: {e}"))?; + let ben: alloy_primitives::Address = match beneficiary { + Some(b) => b.parse().map_err(|e| format!("--beneficiary: {e}"))?, + None => alloy_primitives::Address::ZERO, + }; + let st = hoverfly::batch::read_chequebook_state(&rpc_url, cb, ben).await?; + println!("chequebook 0x{}", hex::encode(cb)); + println!(" issuer 0x{}", hex::encode(st.issuer)); + println!(" liquid for 0x{} {}", hex::encode(ben), st.liquid_for_us); + println!(" paid out to it {}", st.paid_out_to_us); + println!( + " bounced {}{}", + st.bounced, + if st.bounced { + " <- a relay will refuse this chequebook" + } else { + "" + } + ); + } + }, Commands::Batch { action } => match action { BatchAction::Create { rpc_url, @@ -2769,3 +3319,74 @@ fn whole_to_smallest_unit(whole: f64, decimals: u8) -> Option Result> { + let raw = hex::decode(key.trim_start_matches("0x"))?; + if raw.len() != 32 { + return Err(format!("key must be 32 bytes hex, got {}", raw.len()).into()); + } + Ok(alloy_signer_local::PrivateKeySigner::from_slice(&raw)?) +} + +/// Decimal BZZ → PLUR. BZZ has **16** decimals, not 18: getting this wrong +/// by two orders of magnitude is the kind of mistake that silently deposits +/// 100× too little and makes every cheque bounce. +#[cfg(unix)] +fn parse_bzz_amount(s: &str) -> Result> { + const DECIMALS: usize = 16; + let s = s.trim(); + let (whole, frac) = match s.split_once('.') { + Some((w, f)) => (w, f), + None => (s, ""), + }; + if frac.len() > DECIMALS { + return Err(format!("BZZ has {DECIMALS} decimals; '{s}' has more").into()); + } + if whole.is_empty() && frac.is_empty() { + return Err("empty amount".into()); + } + let mut digits = String::from(whole); + digits.push_str(frac); + digits.push_str(&"0".repeat(DECIMALS - frac.len())); + if !digits.chars().all(|c| c.is_ascii_digit()) { + return Err(format!("'{s}' is not a decimal amount").into()); + } + Ok(alloy_primitives::U256::from_str_radix(&digits, 10)?) +} + +#[cfg(all(test, unix))] +mod chequebook_cli_tests { + use super::*; + + /// BZZ has 16 decimals. An 18-decimal assumption would under-fund by + /// 100×, and every cheque drawn on it would then fail the relay's + /// funding check for no visible reason. + #[test] + fn bzz_amounts_use_sixteen_decimals() { + let one = parse_bzz_amount("1").expect("1 BZZ"); + assert_eq!(one.to_string(), "10000000000000000"); + assert_eq!( + parse_bzz_amount("0.05").expect("0.05").to_string(), + "500000000000000" + ); + assert_eq!( + parse_bzz_amount("0.0001").expect("small").to_string(), + "1000000000000" + ); + assert_eq!( + parse_bzz_amount("1.5").expect("1.5").to_string(), + "15000000000000000" + ); + } + + #[test] + fn malformed_amounts_are_refused() { + for bad in ["", ".", "abc", "1.2.3", "-1", "0.00000000000000001"] { + assert!(parse_bzz_amount(bad).is_err(), "{bad} must be refused"); + } + } +} diff --git a/src/challenge.rs b/src/challenge.rs new file mode 100644 index 0000000..ba0c219 --- /dev/null +++ b/src/challenge.rs @@ -0,0 +1,429 @@ +//! Admission challenge for metered relay — `docs/pusher-incentives.md` §7.2. +//! +//! A challenge is a **capability**. Holding one proves the relay already +//! resolved the batch's standing on-chain and already priced its credit +//! line, so `/v1/push` admission reads no chain state at all — which is the +//! whole reason the design can afford to check standing before accepting a +//! body (§7.2's amplification argument). +//! +//! It is stateless: the relay keeps no table of issued nonces, so a free +//! `GET /v1/challenge` cannot exhaust memory. The nonce is a MAC over the +//! fields it authorises, and the fields travel back with the request. +//! +//! Two independent checks run at admission, and conflating them is the +//! easiest way to get this wrong: +//! +//! 1. **The relay's MAC over the nonce** proves *this relay* issued the +//! capability, with these exact fields. Symmetric, no RPC. +//! 2. **The client's EIP-712 signature** over the same fields proves the +//! caller holds the account key — and, because `origin` is inside the +//! signed struct, that it signed for *this* relay. That is what stops a +//! signature gathered during a normal upload through relay A being +//! replayed at relay B alongside the victim's harvested stamps (§11.1). +//! +//! Check 2 is only worth anything if `origin` is compared against a +//! **configured** hostname. Comparing it to the request's `Host` header +//! compares one attacker-supplied value to another and silently restores +//! the replay — see [`verify`]. + +use sha3::{Digest, Keccak256}; + +/// Domain tag, fixed 28 bytes. First field after the secret so no other +/// scheme using the same key can collide with this one. +pub const DOMAIN_TAG: &[u8; 28] = b"hoverfly-pusher-challenge-v1"; + +/// Default lifetime of a challenge (§7.2). Short on purpose: re-issuing +/// costs one local ecrecover, and a narrow window shrinks the replay +/// surface to near nothing. +pub const CHALLENGE_TTL_SECS: u64 = 300; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChallengeFields { + pub account: [u8; 20], + pub batch: [u8; 32], + /// The host the client dialled. Compared against configuration, never + /// against a request header. + pub origin: String, + pub expiry_unix: u64, + /// The credit line §10.3 granted this batch, in PLUR. Inside the MAC so + /// a client cannot present a nonce issued for a rich batch alongside a + /// cheap one's id. + pub cap_plur: u128, +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum ChallengeError { + #[error("challenge nonce is not ours")] + BadMac, + #[error("challenge expired {0}s ago")] + Expired(u64), + #[error("challenge origin {got:?} is not this relay ({want:?})")] + OriginMismatch { got: String, want: String }, + #[error("challenge origin is empty")] + EmptyOrigin, + #[error("origin too long: {0} bytes (max 65535)")] + OriginTooLong(usize), +} + +/// Bytes the MAC covers. +/// +/// **Fixed-width and length-prefixed, not a concatenation.** `origin` is +/// variable-length, so a bare `A ‖ B ‖ origin ‖ …` makes `("host.a", "bc")` +/// and `("host.ab", "c")` share a preimage — a relay serving several +/// hostnames would issue one nonce valid for two of them. Every fixed field +/// comes first at a known width, then a 2-byte length, then `origin` last. +pub fn preimage(f: &ChallengeFields) -> Result, ChallengeError> { + if f.origin.is_empty() { + return Err(ChallengeError::EmptyOrigin); + } + let olen = f.origin.len(); + if olen > u16::MAX as usize { + return Err(ChallengeError::OriginTooLong(olen)); + } + let mut out = Vec::with_capacity(28 + 20 + 32 + 8 + 16 + 2 + olen); + out.extend_from_slice(DOMAIN_TAG); + out.extend_from_slice(&f.account); + out.extend_from_slice(&f.batch); + out.extend_from_slice(&f.expiry_unix.to_be_bytes()); + out.extend_from_slice(&f.cap_plur.to_be_bytes()); + out.extend_from_slice(&(olen as u16).to_be_bytes()); + out.extend_from_slice(f.origin.as_bytes()); + Ok(out) +} + +/// `keccak256(secret ‖ preimage)`. +/// +/// A prefix-MAC rather than HMAC, which is sound here specifically because +/// Keccak is a sponge and has no length-extension weakness — the property +/// that forces HMAC's nested construction on Merkle–Damgård hashes like +/// SHA-256. The secret is a fixed 32 bytes and every field after it is +/// fixed-width or length-prefixed, so no two distinct inputs share a +/// preimage. (This is the same reasoning KMAC is built on.) +pub fn nonce(secret: &[u8; 32], f: &ChallengeFields) -> Result<[u8; 32], ChallengeError> { + let mut h = Keccak256::new(); + h.update(secret); + h.update(preimage(f)?); + Ok(h.finalize().into()) +} + +/// Verify a presented nonce against the fields it claims to authorise. +/// +/// `origins` is the relay's **configured** hostname list (`--origin`). It +/// must never be derived from `Host` or `X-Forwarded-Host`: those are +/// supplied by the same client supplying the challenge, so comparing them +/// is a no-op that leaves §11.1's cross-relay replay wide open while +/// appearing to close it. +pub fn verify( + secret: &[u8; 32], + f: &ChallengeFields, + presented: &[u8], + now_unix: u64, + origins: &[String], +) -> Result<(), ChallengeError> { + if !origins.iter().any(|o| o == &f.origin) { + return Err(ChallengeError::OriginMismatch { + got: f.origin.clone(), + want: origins.join(","), + }); + } + let want = nonce(secret, f)?; + if !constant_time_eq(presented, &want) { + return Err(ChallengeError::BadMac); + } + // Expiry last: a caller holding a valid-but-stale capability learns + // only that it is stale, while a forged one learns nothing about which + // field was wrong. + if now_unix > f.expiry_unix { + return Err(ChallengeError::Expired(now_unix - f.expiry_unix)); + } + Ok(()) +} + +/// Equal-length, data-independent comparison. A byte-wise early exit on the +/// MAC would let a nonce be ground out one byte at a time, which is exactly +/// the forgery this whole construction exists to prevent. +pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// Seconds since the Unix epoch, saturating at 0 on a clock before 1970. +pub fn now_unix() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +// ---- the header, on the wire ---- +// +// Both ends of the challenge need this codec: the relay issues and decodes, +// the client encodes and presents. It lives here rather than in `metered.rs` +// because `metered.rs` is the relay's state machine — reserving, billing, +// crediting — and a client that pays a relay must be buildable without it. + +/// Header carrying the capability plus the client's proof it holds the +/// account key. One custom header, so the CORS preflight allow-list grows +/// by exactly one entry (§7.2's browser blocker). +pub const CHALLENGE_HEADER: &str = "x-hoverfly-challenge"; + +/// Cap on the header itself. The fields are fixed-width apart from +/// `origin`, so anything larger is not a challenge. +pub const MAX_CHALLENGE_HEADER: usize = 2048; + +/// A capability the relay minted: the authorised fields plus our MAC over +/// them. +pub struct IssuedChallenge { + pub fields: ChallengeFields, + pub nonce: [u8; 32], +} + +impl IssuedChallenge { + pub fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "nonce": format!("0x{}", hex::encode(self.nonce)), + "account": format!("0x{}", hex::encode(self.fields.account)), + "batch": format!("0x{}", hex::encode(self.fields.batch)), + "origin": self.fields.origin, + "expiry": self.fields.expiry_unix, + "max_outstanding_plur": self.fields.cap_plur.to_string(), + "expires_ms": self.fields.expiry_unix.saturating_mul(1000), + }) + } +} + +/// What the client sends back: the capability it was issued plus its +/// signature over the same fields. +pub struct PresentedChallenge { + pub fields: ChallengeFields, + pub nonce: [u8; 32], + pub sig: [u8; 65], +} + +impl PresentedChallenge { + /// `base64(json)` in one header, so the CORS allow-list grows by one. + pub fn decode(raw: &str) -> Result { + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(raw.trim()) + .map_err(|e| format!("challenge header base64: {e}"))?; + let v: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| format!("challenge header json: {e}"))?; + let get = |k: &str| -> Result { + v.get(k) + .and_then(|x| x.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| format!("challenge header: missing {k}")) + }; + let fixed = |k: &str, n: usize| -> Result, String> { + let raw = hex::decode(get(k)?.trim_start_matches("0x")) + .map_err(|e| format!("challenge {k} hex: {e}"))?; + if raw.len() != n { + return Err(format!( + "challenge {k} must be {n} bytes, got {}", + raw.len() + )); + } + Ok(raw) + }; + let mut account = [0u8; 20]; + account.copy_from_slice(&fixed("account", 20)?); + let mut batch = [0u8; 32]; + batch.copy_from_slice(&fixed("batch", 32)?); + let mut nonce = [0u8; 32]; + nonce.copy_from_slice(&fixed("nonce", 32)?); + let mut sig = [0u8; 65]; + sig.copy_from_slice(&fixed("sig", 65)?); + let expiry_unix = v + .get("expiry") + .and_then(|x| x.as_u64()) + .ok_or("challenge header: missing expiry")?; + let cap_plur: u128 = get("max_outstanding_plur")? + .parse() + .map_err(|e| format!("challenge cap: {e}"))?; + Ok(Self { + fields: ChallengeFields { + account, + batch, + origin: get("origin")?, + expiry_unix, + cap_plur, + }, + nonce, + sig, + }) + } +} + +/// Encode a challenge plus signature into the header value. Client side, +/// and used by the tests to drive the relay path end to end. +pub fn encode_challenge_header(issued: &IssuedChallenge, sig: &[u8; 65]) -> String { + use base64::Engine; + let body = serde_json::json!({ + "nonce": format!("0x{}", hex::encode(issued.nonce)), + "account": format!("0x{}", hex::encode(issued.fields.account)), + "batch": format!("0x{}", hex::encode(issued.fields.batch)), + "origin": issued.fields.origin, + "expiry": issued.fields.expiry_unix, + "max_outstanding_plur": issued.fields.cap_plur.to_string(), + "sig": format!("0x{}", hex::encode(sig)), + }); + base64::engine::general_purpose::STANDARD.encode(body.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SECRET: [u8; 32] = [7u8; 32]; + + fn fields() -> ChallengeFields { + ChallengeFields { + account: [1u8; 20], + batch: [2u8; 32], + origin: "relay-a.example".into(), + expiry_unix: 1_000_000, + cap_plur: 62_200_000_000_000, + } + } + + fn origins() -> Vec { + vec!["relay-a.example".into()] + } + + #[test] + fn a_nonce_we_issued_verifies() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + verify(&SECRET, &f, &n, 999_999, &origins()).expect("must verify"); + } + + #[test] + fn a_nonce_from_another_relays_secret_does_not() { + let f = fields(); + let n = nonce(&[9u8; 32], &f).expect("nonce"); + assert_eq!( + verify(&SECRET, &f, &n, 999_999, &origins()), + Err(ChallengeError::BadMac) + ); + } + + /// Every field is inside the MAC, so tampering with any of them after + /// issue must fail. The `cap` case is the sharp one: without it a client + /// could present a rich batch's credit line alongside a dust batch's id. + #[test] + fn every_field_is_covered_by_the_mac() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + let mut cases = Vec::new(); + + let mut g = f.clone(); + g.account = [0xAA; 20]; + cases.push(("account", g)); + let mut g = f.clone(); + g.batch = [0xBB; 32]; + cases.push(("batch", g)); + let mut g = f.clone(); + g.expiry_unix += 1; + cases.push(("expiry", g)); + let mut g = f.clone(); + g.cap_plur *= 1000; + cases.push(("cap", g)); + + for (what, g) in cases { + assert_eq!( + verify(&SECRET, &g, &n, 999_999, &origins()), + Err(ChallengeError::BadMac), + "{what} must be covered by the MAC" + ); + } + } + + /// The ambiguity the doc rejects for the client's signature and then + /// very nearly repeated in the relay's own MAC. `("host.a","bc")` and + /// `("host.ab","c")` must not collide. + #[test] + fn the_preimage_is_unambiguous_across_a_variable_length_origin() { + let mut a = fields(); + a.origin = "host.a".into(); + let mut b = fields(); + b.origin = "host.ab".into(); + assert_ne!( + preimage(&a).expect("a"), + preimage(&b).expect("b"), + "a shorter origin must not be a prefix-collision of a longer one" + ); + assert_ne!(nonce(&SECRET, &a).unwrap(), nonce(&SECRET, &b).unwrap()); + } + + /// The origin the relay compares against is configuration. A nonce + /// issued for another host must not verify here even though the MAC + /// itself is intact — this is the check that stops §11.1. + #[test] + fn an_origin_outside_the_configured_set_is_refused() { + let mut f = fields(); + f.origin = "relay-b.example".into(); + let n = nonce(&SECRET, &f).expect("nonce"); + let got = verify(&SECRET, &f, &n, 999_999, &origins()); + assert!( + matches!(got, Err(ChallengeError::OriginMismatch { .. })), + "got {got:?}" + ); + } + + #[test] + fn a_relay_serving_several_hostnames_accepts_each_of_them() { + let all: Vec = vec!["relay-a.example".into(), "alias.example".into()]; + for host in &all { + let mut f = fields(); + f.origin = host.clone(); + let n = nonce(&SECRET, &f).expect("nonce"); + verify(&SECRET, &f, &n, 999_999, &all).expect("each configured host verifies"); + } + } + + #[test] + fn an_expired_challenge_is_refused() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + verify(&SECRET, &f, &n, f.expiry_unix, &origins()).expect("valid up to the instant"); + assert_eq!( + verify(&SECRET, &f, &n, f.expiry_unix + 1, &origins()), + Err(ChallengeError::Expired(1)) + ); + } + + #[test] + fn a_truncated_or_padded_nonce_is_refused() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + assert_eq!( + verify(&SECRET, &f, &n[..31], 1, &origins()), + Err(ChallengeError::BadMac) + ); + let mut long = n.to_vec(); + long.push(0); + assert_eq!( + verify(&SECRET, &f, &long, 1, &origins()), + Err(ChallengeError::BadMac) + ); + assert_eq!( + verify(&SECRET, &f, &[], 1, &origins()), + Err(ChallengeError::BadMac) + ); + } + + #[test] + fn an_empty_origin_is_rejected_rather_than_hashed() { + let mut f = fields(); + f.origin = String::new(); + assert_eq!(preimage(&f), Err(ChallengeError::EmptyOrigin)); + } + + #[test] + fn constant_time_eq_agrees_with_equality() { + assert!(constant_time_eq(b"abc", b"abc")); + assert!(!constant_time_eq(b"abc", b"abd")); + assert!(!constant_time_eq(b"abc", b"ab")); + assert!(constant_time_eq(b"", b"")); + } +} diff --git a/src/cheques.rs b/src/cheques.rs index 4d435cc..d87cfd3 100644 --- a/src/cheques.rs +++ b/src/cheques.rs @@ -175,3 +175,144 @@ impl ChequeStore { Ok(()) } } + +// ────────────────────────────────────────────────────────────────────── +// Metered relays (docs/pusher-incentives.md §8.3) +// ────────────────────────────────────────────────────────────────────── + +/// Key for a metered relay's cumulative. +/// +/// **Namespaced by beneficiary, not by lane or overlay**, and deliberately +/// distinct from the bare-overlay keys the bee settlement path uses. +/// +/// A cumulative is per `(chequebook, beneficiary)` — the beneficiary is what +/// `paidOut` is keyed on in the contract — while a *lane* is a URL. One +/// operator running four lane URLs behind one beneficiary EOA is the obvious +/// deployment, and keying per lane would deadlock it: lane 1 issues +/// cumulative 10, lane 2 counts from its own zero and issues 8, the relay +/// applies `ErrChequeNotIncreasing`, and the client recomputes 8 from the +/// same local counter forever. Two lanes sharing a beneficiary collapse to +/// one key here, which is exactly right — they are one settlement channel. +/// +/// The overlay key stays correct for bee peers, where the overlay *is* the +/// stable cross-run identity, so the two namespaces coexist without a +/// migration. +pub fn relay_key(beneficiary: &[u8; 20]) -> String { + format!("relay:{}", hex::encode(beneficiary)) +} + +impl ChequeStore { + /// Everything promised against this chequebook, across every payee. + /// + /// Lanes with distinct beneficiaries are independent claims on **one** + /// balance, so without this a cheque to the second lane silently + /// exceeds it and bounces. Mirrors bee's `reserveTotalIssued` + /// (`chequebook.go:163-178`) on the issuing side. + pub fn total_issued(&self) -> u128 { + self.payouts + .values() + .copied() + .fold(0u128, u128::saturating_add) + } + + /// Would raising `key` to `cumulative` push the total past `balance`? + /// + /// Checked *before* signing: an over-committed cheque is not refused by + /// the relay, it is accepted and then fails at cashout, which looks like + /// the relay's fault and costs the lane's trust rather than the + /// client's. + pub fn would_exceed_balance(&self, key: &str, cumulative: u128, balance: u128) -> bool { + let others = self.total_issued().saturating_sub(self.cumulative(key)); + others.saturating_add(cumulative) > balance + } + + /// Set an absolute cumulative, for payees where the client computes the + /// running total itself (metered relays) rather than accruing deltas. + /// Refuses to move backwards — that would produce a cheque the payee + /// rejects as non-increasing. + pub fn set_cumulative(&mut self, key: &str, cumulative: u128) -> Result<(), ChequeStoreError> { + let k = key.to_lowercase(); + let cur = self.payouts.get(&k).copied().unwrap_or(0); + if cumulative < cur { + return Err(ChequeStoreError::Overflow); + } + self.payouts.insert(k, cumulative); + Ok(()) + } +} + +#[cfg(test)] +mod metered_tests { + use super::*; + + const CB: [u8; 20] = [1u8; 20]; + const BEN_A: [u8; 20] = [0xAA; 20]; + const BEN_B: [u8; 20] = [0xBB; 20]; + + /// The deployment that would otherwise deadlock: several lane URLs, one + /// beneficiary. They must share a single running cumulative. + #[test] + fn two_lanes_behind_one_beneficiary_share_a_cumulative() { + let mut s = ChequeStore::new(CB); + let k = relay_key(&BEN_A); + s.set_cumulative(&k, 1000).expect("lane 1 settles"); + // Lane 2, same operator, same beneficiary — must continue from 1000 + // rather than starting over at its own zero. + assert_eq!(s.cumulative(&k), 1000); + s.set_cumulative(&k, 1600).expect("lane 2 settles"); + assert_eq!(s.cumulative(&k), 1600); + } + + #[test] + fn a_cumulative_never_moves_backwards() { + let mut s = ChequeStore::new(CB); + let k = relay_key(&BEN_A); + s.set_cumulative(&k, 500).expect("set"); + s.set_cumulative(&k, 400) + .expect_err("a lower cumulative would be rejected as non-increasing"); + assert_eq!(s.cumulative(&k), 500); + } + + /// Distinct beneficiaries are distinct channels but one balance. + #[test] + fn total_issued_sums_every_payee() { + let mut s = ChequeStore::new(CB); + s.set_cumulative(&relay_key(&BEN_A), 600).expect("a"); + s.set_cumulative(&relay_key(&BEN_B), 300).expect("b"); + // A bee peer drawing on the same chequebook counts too. + s.bump_and_get("abc123", 100).expect("bee peer"); + assert_eq!(s.total_issued(), 1000); + } + + #[test] + fn over_committing_the_balance_is_caught_before_signing() { + let mut s = ChequeStore::new(CB); + s.set_cumulative(&relay_key(&BEN_A), 600).expect("a"); + s.set_cumulative(&relay_key(&BEN_B), 300).expect("b"); + let k = relay_key(&BEN_A); + assert!( + !s.would_exceed_balance(&k, 700, 1000), + "raising A to 700 alongside B's 300 exactly fits" + ); + assert!( + s.would_exceed_balance(&k, 701, 1000), + "one PLUR more does not" + ); + } + + /// Relay and bee keys must not collide: an overlay is 32 bytes of hex + /// and a beneficiary is 20, but the namespace makes it explicit rather + /// than incidental. + #[test] + fn relay_keys_are_namespaced_away_from_bee_overlays() { + let k = relay_key(&BEN_A); + assert!(k.starts_with("relay:")); + let mut s = ChequeStore::new(CB); + s.set_cumulative(&k, 42).expect("relay"); + assert_eq!( + s.cumulative(&hex::encode(BEN_A)), + 0, + "a bare-hex key must not read the relay's entry" + ); + } +} diff --git a/src/client.rs b/src/client.rs index 00d606a..d4914d0 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2668,6 +2668,7 @@ pub async fn push_via_pushers( total, move |_| Ok(once.take().unwrap_or_default()), progress, + None, ) .await } @@ -2684,6 +2685,10 @@ async fn drive_pushers

( total_hint: usize, mut produce: P, progress: Option<&ProgressFn>, + // Metered lanes only (`docs/pusher-incentives.md` §12). `None` means + // pay nothing, which is correct for every `open` lane and is what all + // callers pass until a lane actually advertises a price. + payment: Option<&crate::payer::PaymentConfig>, ) -> Result<(), ClientError> where P: FnMut(usize) -> Result, ClientError>, @@ -2709,8 +2714,13 @@ where // *itself* a default-weighted start, never degrade routing for the // others — which is exactly what the previous all-or-nothing overlay // collection did. - let infos: Vec = - futures::future::join_all(pusher_urls.iter().map(|u| fetch_lane_info(&http, u))).await; + let infos: Vec = futures::future::join_all(pusher_urls.iter().map(|u| { + // A lane quoting more than this is refused rather than paid. + // Priced off the shipped default so a lane cannot quietly + // charge an order of magnitude more than the design assumes. + fetch_lane_info(&http, u, crate::meter::PRICE_PLUR_PER_KIB * 8) + })) + .await; for (i, (u, info)) in pusher_urls.iter().zip(&infos).enumerate() { info!(target: "hoverfly::upload", "lane {i} {u}: pool={:?} batch_max={:?} inflight_max={:?} budget_gb={:?}", @@ -2733,7 +2743,110 @@ where .iter() .map(|u| std::sync::Arc::new(format!("{}/v1/push", u.trim_end_matches('/')))) .collect(); + // One payer per metered lane. Lanes that quote no price get none, so an + // `open` lane costs nothing and needs no special case below. + let mut payers: Vec> = Vec::with_capacity(pusher_urls.len()); + for (i, url) in pusher_urls.iter().enumerate() { + let quote = payment.and_then(|_| infos.get(i).and_then(|inf: &LaneInfo| inf.quote.clone())); + payers.push(match (payment, quote) { + (Some(pc), Some(q)) => { + let key = crate::cheques::relay_key(&q.beneficiary); + let cumulative = pc + .cheques + .lock() + .expect("cheque store poisoned") + .cumulative(&key); + info!(target: "hoverfly::upload", + "lane {i} is metered at {} PLUR/KiB; resuming cumulative {cumulative}", + q.params.price_plur_per_kib); + Some(crate::payer::LanePayer::new(url.clone(), q, cumulative)) + } + _ => None, + }); + } + // Size POSTs to the credit line before scheduling anything (§7.2). A + // body larger than the whole line can never be admitted however + // promptly we settle, so building one guarantees a 402 the client + // cannot pay its way out of — it would owe nothing, having had nothing + // accepted. + let mut infos = infos; + // Paying is optional, so a lane that *requires* payment we cannot make is + // simply not ours to use. Scheduling it anyway means every chunk sent + // there is refused for a missing capability, and since that is not a 402 + // it counts against lane health — so the fleet spends one attempt per + // chunk discovering, repeatedly, something the lane advertised up front. + // + // Soft-metered lanes are kept: they bill and serve, so an unpaying client + // is served exactly like an `open` one. + let unusable: Vec = (0..pusher_urls.len()) + .filter(|&i| { + infos + .get(i) + .is_some_and(|inf: &LaneInfo| inf.hard_enforcement) + && payers[i].is_none() + }) + .collect(); + for &i in &unusable { + warn!(target: "hoverfly::upload", + "lane {i} {} enforces payment and this client has none configured; skipping it", + pusher_urls[i]); + } + if !unusable.is_empty() && unusable.len() == pusher_urls.len() { + return Err(ClientError::Pusher( + "every lane requires payment and no chequebook is configured — \ + pass --chequebook, or use a fleet with free lanes" + .into(), + )); + } + let mut lane_frame_ceiling: Vec = vec![usize::MAX; infos.len()]; + if let Some(pc) = payment { + for (i, payer) in payers.iter_mut().enumerate() { + let Some(payer) = payer.as_mut() else { + continue; + }; + if let Err(e) = payer.header(&http, pc).await { + warn!(target: "hoverfly::upload", + "lane {i}: no challenge ({e}); scheduling it unpaid"); + continue; + } + let fits = payer.max_frames(); + let before = infos[i].batch_max.unwrap_or(usize::MAX); + if fits < before { + info!(target: "hoverfly::upload", + "lane {i}: credit line allows {fits} frames/POST (was {before})"); + infos[i].batch_max = Some(fits); + } + // Remember the lane's own ceiling: the per-dispatch resize below + // must clamp to it and never raise a lane above what it + // advertised. + lane_frame_ceiling[i] = fits.min(before); + // Learn any debt carried from an earlier run *before* sizing the + // first POST. The relay's ledger outlives ours (§17.1), so a + // fresh process starts believing it owes nothing and builds a + // body against the whole credit line — while the relay is + // already holding part of it. The first POST is then refused, + // and if the carried debt happens to sit below the dust floor + // the refusal cannot be paid off either. Reconciling here costs + // one GET per lane and makes every later size correct. + match payer.reconcile(&http, pc).await { + Ok(true) => info!(target: "hoverfly::upload", + "lane {i}: relay carries {} PLUR from an earlier run", + payer.account.owed()), + Ok(false) => {} + Err(e) => warn!(target: "hoverfly::upload", + "lane {i}: could not read carried debt ({e}); \ + sizing may be optimistic until the first 402"), + } + } + } + // batch id -> body bytes, so a completed POST can be billed for what + // it actually sent. The relay bills what it *admits*, so a refused POST + // must not become debt on this side either. + let mut in_flight_bytes: HashMap = HashMap::new(); let mut sched = Scheduler::new(infos, cfg); + for &i in &unusable { + sched.retire_lane(i); + } // Frames are held by address, not by index: `admit` de-duplicates, so an // index-parallel Vec would silently skew. Keying by address also lets a // frame be dropped the moment its chunk is acked, which is what keeps a @@ -2791,8 +2904,61 @@ where refill(&mut sched, &mut frames, &mut total, &mut exhausted)?; } - // Hand out everything the scheduler is willing to dispatch. - while let Some(a) = sched.next(now_ms()) { + // Don't ask for work a metered lane cannot currently pay for. + // Settling first is the way out; if nothing is owed yet, the debt is + // still in flight and completing it is what frees the line. Taking + // an assignment and returning it would cost every chunk a retry + // attempt per bounce and fail the upload instead of pausing it. + if let Some(pc) = payment { + for (lane, payer) in payers.iter_mut().enumerate() { + let Some(payer) = payer.as_mut() else { + continue; + }; + if !payer.has_headroom() + && payer.account.owed() > 0 + && let Err(e) = payer.settle(&http, pc).await + { + warn!(target: "hoverfly::upload", "lane {lane}: settle failed: {e}"); + } + // POST sizing happens per dispatch, in the loop below. + } + } + // Hand out everything the scheduler is willing to dispatch, resizing + // each metered lane to its *live* headroom immediately before every + // single assignment. + // + // Both halves have to happen per dispatch, not per pass of the outer + // loop. Each POST adds its body to `pending`, so one up-front answer + // authorises a whole run of them against headroom only the first + // actually had — and since the body is built from the lane's + // `batch_max`, a stale ceiling means the second and third POSTs are + // each sized as if they were alone on the lane. That is what earns a + // 402 the client cannot pay: the bytes are in flight, so nothing is + // owed yet and no cheque changes anything. + while let Some(a) = { + let mut anyone_can_take_work = payment.is_none(); + if payment.is_some() { + for (lane, p) in payers.iter().enumerate() { + let Some(p) = p.as_ref() else { + // Unmetered lane in a metered run: nothing to size. + anyone_can_take_work = true; + continue; + }; + let affordable = p.affordable_frames(); + if affordable > 0 { + sched.set_lane_batch_max(lane, affordable.min(lane_frame_ceiling[lane])); + } + if p.has_headroom() { + anyone_can_take_work = true; + } + } + } + if anyone_can_take_work { + sched.next(now_ms()) + } else { + None + } + } { let batch: Vec = a .chunks .iter() @@ -2806,8 +2972,62 @@ where debug!(target: "hoverfly::upload", "hedging {} straggler(s) onto lane {lane}", batch.len()); } + // Metered lane: attach the capability, and bill ourselves for + // the body by the same arithmetic the relay uses (§8). We count + // what we *send*; nothing the relay reports enters this number, + // which is why the two sides can agree without exchanging + // anything. + let mut challenge: Option = None; + if let (Some(pc), Some(payer)) = (payment, payers[lane].as_mut()) { + let body_bytes: u64 = batch + .iter() + .map(|c| (crate::pushframe::HEADER_LEN + c.wire.len()) as u64) + .sum(); + // Pay first if this body would not fit. Settling on the + // window alone is not enough: several POSTs dispatch + // concurrently, so the cap can be crossed before any of + // them completes and reports back. + if payer.would_exceed(body_bytes) { + if let Err(e) = payer.settle(&http, pc).await { + warn!(target: "hoverfly::upload", + "lane {lane}: pre-dispatch settle failed: {e}"); + } + // Settling may not have helped: debt only exists for + // POSTs the relay has already admitted, so a client that + // has saturated its line with *in-flight* bytes has + // nothing to pay yet. Dispatching anyway earns a 402 and, + // if it keeps happening, fails the chunks outright. + // Hand the batch back instead and let it be re-dispatched + // once something completes — the same pause a 402 would + // cause, without the round trip. + } + match payer.header(&http, pc).await { + Ok(h) => challenge = Some(h.to_string()), + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {lane}: cannot obtain a challenge ({e}); pausing it"); + sched.on_batch_result( + batch_id, + crate::pushsched::BatchOutcome::PaymentRequired, + now_ms(), + ); + continue; + } + } + payer.account.record_sent(body_bytes); + in_flight_bytes.insert(batch_id, body_bytes); + } tokio::spawn(async move { - post_batch_streaming(&http, url.as_str(), batch_id, lane, &batch, &tx).await; + post_batch_streaming( + &http, + url.as_str(), + batch_id, + lane, + &batch, + &tx, + challenge.as_deref(), + ) + .await; }); } @@ -2901,9 +3121,135 @@ where acked, elapsed_ms, outcome, + dedup_bytes, } => { sched.on_batch_timing(lane, acked, elapsed_ms); + let needs_payment = + matches!(outcome, crate::pushsched::BatchOutcome::PaymentRequired); + // Turn the in-flight bytes into debt only if the relay + // actually took them. + if let Some(sent) = in_flight_bytes.remove(&batch) + && let Some(payer) = payers[lane].as_mut() + { + // The question is whether the relay *read the body*, not + // whether we got a clean answer back. A 402 is refused at + // admission, before a byte is read, so it costs nothing + // and is not billed by either side. Anything else means + // the bytes arrived and were billed — including a stream + // that broke halfway through, which is exactly §7.3's + // ack-tail case. Billing only clean answers made the + // client silently under-count every interrupted POST + // while the relay booked it. + let reached_relay = + !matches!(outcome, crate::pushsched::BatchOutcome::PaymentRequired); + payer.account.record_answered(sent, reached_relay); + if reached_relay && dedup_bytes > 0 { + payer.account.refund_dedup(dedup_bytes); + } + } sched.on_batch_result(batch, outcome, now_ms()); + // Settle when the relay asks (402) or when we have crossed + // the lane's own settlement window. Paying on the window + // keeps an upload from ever reaching its cap in the first + // place; paying on 402 is the recovery path when it does. + if let (Some(pc), Some(payer)) = (payment, payers[lane].as_mut()) + && (needs_payment || payer.account.should_settle()) + { + match payer.settle(&http, pc).await { + Ok(Some(c)) => { + info!(target: "hoverfly::upload", + "lane {lane}: paid, cumulative now {c}"); + // Restores the lane to the health it had before + // it ran out of credit — a 402 never charged the + // failure streak, so this is not a re-warm. + sched.fund_lane(lane); + } + Ok(None) if needs_payment => { + // 402 with nothing owed above the dust floor + // means the two sides disagree about the + // ledger, and retrying the POST cannot fix it. + // Almost always this is debt carried from an + // earlier run: the relay's ledger is durable + // and ours is not, so it is still counting a + // residual we settled below the dust floor and + // then forgot. Ask what it thinks we owe and + // pay that. + match payer.reconcile(&http, pc).await { + Ok(true) => match payer.settle(&http, pc).await { + Ok(Some(c)) => { + info!(target: "hoverfly::upload", + "lane {lane}: reconciled with relay ledger, \ + paid cumulative {c}"); + sched.fund_lane(lane); + } + Ok(None) => { + unpayable_402(&mut sched, lane, payer); + } + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {lane}: payment after reconcile failed: {e}"); + } + }, + Ok(false) => { + unpayable_402(&mut sched, lane, payer); + } + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {lane}: reconcile failed: {e}"); + } + } + } + Ok(None) => {} + Err(e) => { + warn!(target: "hoverfly::upload", "lane {lane}: payment failed: {e}"); + } + } + } + } + } + } + + // Settle whatever is left before returning. Without this the relay is + // left holding debt for work it did and the client walks away — and on + // the next run that debt still counts against the credit line, so the + // account starts closer to its cap for no reason. Only the residual + // below the lane's dust floor is left over, which is a cheque the relay + // would refuse anyway. + // The run can finish with POSTs still draining: the loop exits once + // every *chunk* is acked, which happens before every *response* has + // closed. Those bodies were read by the relay and billed, so they are a + // debt — leaving them in `pending` meant the final settlement saw + // nothing to pay and the relay kept the balance forever. + if payment.is_some() { + for (_, sent) in in_flight_bytes.drain() { + // Lane is unknown here, but a single-lane run is the only case + // that can strand bytes this way; charge every payer's share by + // attributing to the lane that still has pending debt. + for payer in payers.iter_mut().flatten() { + if payer.account.outstanding() > payer.account.owed() { + payer.account.record_answered(sent, true); + break; + } + } + } + } + if let Some(pc) = payment { + for (lane, payer) in payers.iter_mut().enumerate() { + let Some(payer) = payer.as_mut() else { + continue; + }; + match payer.settle(&http, pc).await { + Ok(Some(c)) => info!(target: "hoverfly::upload", + "lane {lane}: final settlement, cumulative {c}"), + Ok(None) => { + if payer.account.owed() > 0 { + info!(target: "hoverfly::upload", + "lane {lane}: {} PLUR left unsettled (below the lane's dust floor)", + payer.account.owed()); + } + } + Err(e) => warn!(target: "hoverfly::upload", + "lane {lane}: final settlement failed: {e}"), } } } @@ -2980,9 +3326,58 @@ pub async fn push_via_pusher( /// prepare-everything-first route and would OOM on large files. #[cfg(not(target_arch = "wasm32"))] pub async fn push_stream_via_pushers( + pusher_urls: &[String], + streamer: UploadStreamer, + progress: Option<&ProgressFn>, +) -> Result { + push_stream_via_pushers_paid(pusher_urls, streamer, progress, None).await +} + +/// A 402 arrived that no cheque can clear. Decide whether the lane is +/// merely busy or genuinely stuck. +/// +/// The relay refuses on `owed + reserved`, and only `owed` is payable — +/// reservations are bodies it is still reading. So a lane can be over its +/// line with a debt below the dust floor, which is not a disagreement and +/// not something paying fixes: those bytes simply have to land. Leaving +/// the lane `Unfunded` in that case parks it for good, because only a +/// successful settle re-funds it and no settle is possible. That is an +/// upload that stops with chunks pending and the relay reporting no errors. +/// +/// Re-funding is safe against a busy loop: `has_headroom` binds on +/// `outstanding`, so nothing is dispatched until the in-flight bytes +/// actually clear. +#[cfg(not(target_arch = "wasm32"))] +fn unpayable_402( + sched: &mut crate::pushsched::Scheduler, + lane: usize, + payer: &crate::payer::LanePayer, +) { + if payer.account.pending() > 0 { + debug!(target: "hoverfly::upload", + "lane {lane}: over its line on bytes in flight, not debt; \ + resuming once they land"); + sched.fund_lane(lane); + return; + } + warn!(target: "hoverfly::upload", + "lane {lane}: 402 with nothing in flight and nothing payable — \ + ledger disagreement"); +} + +/// As [`push_stream_via_pushers`], but able to pay metered lanes. +/// +/// `payment` is `None` for every `open` lane, which is the default and the +/// only thing production runs today. When present, the driver fetches a +/// challenge per lane, bills itself by the same arithmetic the relay uses, +/// and settles with a cumulative cheque when the lane's window is crossed +/// or it answers 402 (`docs/pusher-incentives.md` §12). +#[cfg(not(target_arch = "wasm32"))] +pub async fn push_stream_via_pushers_paid( pusher_urls: &[String], mut streamer: UploadStreamer, progress: Option<&ProgressFn>, + payment: Option<&crate::payer::PaymentConfig>, ) -> Result { let root = streamer.root(); let total = streamer.total_chunks(); @@ -2991,6 +3386,7 @@ pub async fn push_stream_via_pushers( total, move |want| streamer.next_batch(want), progress, + payment, ) .await?; Ok(root) @@ -3018,6 +3414,9 @@ enum LaneEv { acked: usize, elapsed_ms: u64, outcome: crate::pushsched::BatchOutcome, + /// Body bytes the relay served from its recent-ack cache and billed + /// at zero (§8.2), so the client can subtract them and stay in step. + dedup_bytes: u64, }, } @@ -3035,6 +3434,10 @@ async fn post_batch_streaming( lane: usize, batch: &[StampedChunk], tx: &tokio::sync::mpsc::UnboundedSender, + // Metered lanes only: the signed admission capability + // (`docs/pusher-incentives.md` §7.2). `None` on an `open` lane, which + // is every production lane today. + challenge: Option<&str>, ) { use crate::pushsched::BatchOutcome; use futures::StreamExt; @@ -3043,36 +3446,53 @@ async fn post_batch_streaming( let body = crate::pushframe::encode_batch(batch); let mut acked = 0usize; - let finish = |outcome: BatchOutcome, acked: usize| { + let mut dedup_bytes: u64 = 0; + let finish = |outcome: BatchOutcome, acked: usize, dedup_bytes: u64| { let _ = tx.send(LaneEv::Done { batch: batch_id, lane, acked, elapsed_ms: t0.elapsed().as_millis() as u64, outcome, + dedup_bytes, }); }; - let resp = match http.post(push_url).body(body).send().await { + let mut req = http.post(push_url).body(body); + if let Some(c) = challenge { + req = req.header(crate::challenge::CHALLENGE_HEADER, c); + } + let resp = match req.send().await { Ok(r) => r, Err(e) => { warn!(target: "hoverfly::upload", "lane {push_url} POST failed: {e}"); - finish(BatchOutcome::Failed(e.to_string()), 0); + finish(BatchOutcome::Failed(e.to_string()), 0, 0); return; } }; if !resp.status().is_success() { let code = resp.status(); let txt = resp.text().await.unwrap_or_default(); + // A 402 is a bill, not a fault. Reporting it as `Failed` would + // charge lane health for a routine settlement and retire a healthy + // lane after five of them (§12); `PaymentRequired` pauses it + // instead, and `Scheduler::fund_lane` restores it once a cheque + // clears. + if code == reqwest::StatusCode::PAYMENT_REQUIRED { + info!(target: "hoverfly::upload", + "lane {push_url} requires payment: {}", txt.trim()); + finish(BatchOutcome::PaymentRequired, 0, 0); + return; + } warn!(target: "hoverfly::upload", "lane {push_url} rejected batch ({code}): {}", txt.trim()); - finish(BatchOutcome::Failed(format!("http {code}")), 0); + finish(BatchOutcome::Failed(format!("http {code}")), 0, 0); return; } let mut stream = resp.bytes_stream(); let mut buf = Vec::new(); - let handle = |line: &[u8], acked: &mut usize| { + let handle = |line: &[u8], acked: &mut usize, dedup_bytes: &mut u64| { let Ok(v) = serde_json::from_slice::(line) else { return; }; @@ -3092,6 +3512,14 @@ async fn post_batch_streaming( let po = v.get("po").and_then(|p| p.as_u64()).unwrap_or(0) as u8; let best_po = v.get("bpo").and_then(|p| p.as_u64()).unwrap_or(0) as u8; let shallow = v.get("shallow").and_then(|s| s.as_bool()).unwrap_or(false); + // The relay served this from its recent-ack cache and billed zero + // for it (§8.2). Subtract the same bytes here or our total drifts + // above the relay's and the next cheque is refused. + if v.get("dedup").and_then(|d| d.as_bool()).unwrap_or(false) + && let Some(c) = batch.iter().find(|c| c.addr == addr) + { + *dedup_bytes += (crate::pushframe::HEADER_LEN + c.wire.len()) as u64; + } let _ = tx.send(LaneEv::Ack { lane, addr, @@ -3108,20 +3536,20 @@ async fn post_batch_streaming( buf.extend_from_slice(&bytes); while let Some(nl) = buf.iter().position(|&b| b == b'\n') { let line: Vec = buf.drain(..=nl).collect(); - handle(&line[..line.len() - 1], &mut acked); + handle(&line[..line.len() - 1], &mut acked, &mut dedup_bytes); } } Err(e) => { warn!(target: "hoverfly::upload", "lane {push_url} stream error: {e}"); - finish(BatchOutcome::Failed(e.to_string()), acked); + finish(BatchOutcome::Failed(e.to_string()), acked, dedup_bytes); return; } } } if !buf.is_empty() { - handle(&buf, &mut acked); + handle(&buf, &mut acked, &mut dedup_bytes); } - finish(BatchOutcome::Answered, acked); + finish(BatchOutcome::Answered, acked, dedup_bytes); } /// Read a lane's `/v1/status` advertisement. @@ -3130,7 +3558,11 @@ async fn post_batch_streaming( /// yields `LaneInfo::default()` and gets scheduled on priors rather than /// being excluded. #[cfg(not(target_arch = "wasm32"))] -async fn fetch_lane_info(http: &reqwest::Client, base_url: &str) -> crate::pushsched::LaneInfo { +async fn fetch_lane_info( + http: &reqwest::Client, + base_url: &str, + price_ceiling: u128, +) -> crate::pushsched::LaneInfo { use crate::pushsched::LaneInfo; let url = format!("{}/v1/status", base_url.trim_end_matches('/')); // Generous: free-tier instances cold-start on the first request. @@ -3151,8 +3583,39 @@ async fn fetch_lane_info(http: &reqwest::Client, base_url: &str) -> crate::pushs .and_then(|s| s.as_str()) .and_then(|s| hex::decode(s.trim_start_matches("0x")).ok()) .and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok()); + // The signed price quote (§7.3). Verified, never merely parsed: an + // unsigned or unverifiable price is repudiable in both directions, so a + // quote that does not check out leaves the lane looking unmetered rather + // than looking free. + let (price_plur_per_kib, hard_enforcement, quote) = match v.get("payment") { + Some(pay) if !pay.is_null() => { + // The overlay cross-check is skipped here: it needs the + // network id, which this driver does not carry, and it is the + // *optional* half of verification. The signature and (where a + // caller supplies one) the pin are what establish identity; + // deriving the overlay only turns the lane's own `overlay` + // field from an assertion into a check. A caller that pins + // does the full version. + match crate::payer::PaymentQuote::verify(pay, None, 0, None, price_ceiling) { + Ok(q) => ( + Some(q.params.price_plur_per_kib), + q.hard_enforcement, + Some(q), + ), + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {base_url}: payment quote rejected ({e}); treating as unmetered"); + (None, false, None) + } + } + } + _ => (None, false, None), + }; LaneInfo { overlay, + price_plur_per_kib, + hard_enforcement, + quote, batch_max: v .get("batch_max") .and_then(|x| x.as_u64()) diff --git a/src/inbound_limit.rs b/src/inbound_limit.rs new file mode 100644 index 0000000..fd5c837 --- /dev/null +++ b/src/inbound_limit.rs @@ -0,0 +1,205 @@ +//! Inbound per-key rate limiting — `docs/pusher-incentives.md` §11.6. +//! +//! **`src/ratelimit.rs` cannot be reused for this.** That one is a per-peer +//! *outbound libp2p dial* GCRA pacer that **parks** the caller until its +//! slot comes up; it has no inbound, per-account or HTTP concept. Parking +//! is exactly wrong here — an attacker that can make the relay hold a task +//! per request has turned a rate limiter into a memory amplifier. This one +//! refuses immediately. +//! +//! Used on `/v1/challenge` (per IP — no account exists yet), `/v1/pay` (per +//! account) and `/v1/push` (per account). +//! +//! ## Eviction is fail-closed, and that is the whole design +//! +//! The bucket map is keyed by something the caller influences, so it has to +//! be bounded. But naive eviction *is* the bypass: if an attacker can push +//! its own throttled bucket out of the map, the next request re-creates it +//! with a full budget and the limit never binds. +//! +//! So only buckets that have nothing to lose are evictable — ones refilled +//! to full, which is indistinguishable from never having existed. When the +//! map is at capacity and every bucket is *still throttled*, new keys are +//! **refused** rather than admitted. Under attack the relay gets stricter, +//! not more permissive. + +use std::collections::HashMap; +use std::time::Instant; + +pub struct InboundLimiter { + buckets: HashMap, Bucket>, + cap: usize, + /// Sustained requests per second. + rate: f64, + /// Maximum burst, in requests. + burst: f64, +} + +#[derive(Clone, Copy)] +struct Bucket { + tokens: f64, + last: Instant, +} + +impl Bucket { + fn refill(&mut self, now: Instant, rate: f64, burst: f64) { + let dt = now.saturating_duration_since(self.last).as_secs_f64(); + self.tokens = (self.tokens + dt * rate).min(burst); + self.last = now; + } + + /// Nothing to lose by dropping it: a fresh bucket is identical. + fn is_full(&self, burst: f64) -> bool { + self.tokens >= burst + } +} + +impl InboundLimiter { + pub fn new(rate_per_sec: f64, burst: f64, cap: usize) -> Self { + Self { + buckets: HashMap::new(), + cap: cap.max(1), + rate: rate_per_sec.max(f64::MIN_POSITIVE), + burst: burst.max(1.0), + } + } + + pub fn allow(&mut self, key: &[u8]) -> bool { + self.allow_at(key, Instant::now()) + } + + pub fn allow_at(&mut self, key: &[u8], now: Instant) -> bool { + let (rate, burst) = (self.rate, self.burst); + if let Some(b) = self.buckets.get_mut(key) { + b.refill(now, rate, burst); + if b.tokens < 1.0 { + return false; + } + b.tokens -= 1.0; + return true; + } + if self.buckets.len() >= self.cap { + self.sweep(now); + if self.buckets.len() >= self.cap { + // Every bucket is still throttled and we are full. Admitting + // this one would mean an attacker with enough distinct keys + // can mint budget on demand; refusing costs a stranger one + // request during an active flood. + return false; + } + } + self.buckets.insert( + key.to_vec(), + Bucket { + tokens: burst - 1.0, + last: now, + }, + ); + true + } + + /// Drop only buckets that have refilled to full. + fn sweep(&mut self, now: Instant) { + let (rate, burst) = (self.rate, self.burst); + self.buckets.retain(|_, b| { + b.refill(now, rate, burst); + !b.is_full(burst) + }); + } + + pub fn tracked(&self) -> usize { + self.buckets.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn a_burst_is_allowed_then_the_rate_binds() { + let mut l = InboundLimiter::new(1.0, 5.0, 128); + let t0 = Instant::now(); + for i in 0..5 { + assert!(l.allow_at(b"a", t0), "burst request {i} must pass"); + } + assert!( + !l.allow_at(b"a", t0), + "the 6th in the same instant must not" + ); + } + + #[test] + fn tokens_refill_over_time() { + let mut l = InboundLimiter::new(2.0, 2.0, 128); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(l.allow_at(b"a", t0)); + assert!(!l.allow_at(b"a", t0)); + assert!( + l.allow_at(b"a", t0 + Duration::from_millis(500)), + "half a second at 2/s is one token" + ); + } + + #[test] + fn keys_are_independent() { + let mut l = InboundLimiter::new(1.0, 1.0, 128); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(!l.allow_at(b"a", t0)); + assert!( + l.allow_at(b"b", t0), + "one key's flood must not throttle another" + ); + } + + /// The bypass this design exists to close: an attacker cycling keys must + /// not be able to evict its own throttled bucket and come back fresh. + #[test] + fn a_throttled_bucket_cannot_be_evicted_by_flooding_new_keys() { + let mut l = InboundLimiter::new(0.001, 1.0, 4); + let t0 = Instant::now(); + assert!(l.allow_at(b"victim", t0)); + assert!(!l.allow_at(b"victim", t0), "victim is now throttled"); + // Flood distinct keys to try to push `victim` out. + for i in 0..200u32 { + l.allow_at(format!("k{i}").as_bytes(), t0); + } + assert!( + !l.allow_at(b"victim", t0), + "the throttled bucket must have survived the flood" + ); + assert!(l.tracked() <= 4, "map stays bounded, got {}", l.tracked()); + } + + /// Fail-closed: when the map is full of throttled buckets, a new key is + /// refused rather than admitted. + #[test] + fn a_full_map_of_throttled_buckets_refuses_new_keys() { + let mut l = InboundLimiter::new(0.001, 1.0, 2); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(l.allow_at(b"b", t0)); + assert!( + !l.allow_at(b"c", t0), + "must refuse rather than evict a live limit" + ); + } + + /// …but a bucket that has refilled to full is free to drop, so the map + /// recovers once the flood stops. + #[test] + fn full_buckets_are_reclaimed_so_the_map_recovers() { + let mut l = InboundLimiter::new(10.0, 1.0, 2); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(l.allow_at(b"b", t0)); + let later = t0 + Duration::from_secs(60); + assert!( + l.allow_at(b"c", later), + "once a and b are refilled they carry no state worth keeping" + ); + } +} diff --git a/src/ledger.rs b/src/ledger.rs new file mode 100644 index 0000000..0ea209e --- /dev/null +++ b/src/ledger.rs @@ -0,0 +1,722 @@ +//! Relay-side payment ledger — `docs/pusher-incentives.md` §10.2, §11.4. +//! +//! Four coupled per-account quantities, written by N spawned push tasks and +//! read by admission, so they live under one lock: +//! +//! - `owed_plur` — billed and unpaid. +//! - `reserved_plur` — admitted but not yet completed. +//! - `last_cumulative[chequebook]` — the monotonic high-water mark that +//! makes a re-presented cheque worth zero. +//! - the `chequebook → account` binding. +//! +//! **Three of those are persisted and one is not, and the asymmetry is +//! load-bearing.** A reservation belongs to an in-flight POST, and no +//! in-flight POST survives a restart — there is no task left to release it. +//! Restoring `reserved` from disk therefore leaks credit permanently and can +//! brick an account into 402 with no way out, which is exactly the no-exit +//! failure §10.1's invariant exists to prevent. So: +//! +//! > Persist `owed`, `last_cumulative`, the binding and `relay_secret` +//! > atomically. Reconstruct `reserved` as zero at boot. +//! +//! The exposure from zeroing is one body's worth of over-admission right +//! after a restart — cents of egress, against an accounting corruption that +//! never self-heals. +//! +//! Losing `last_cumulative` alone is worse than losing everything: a client +//! re-presents its most recent cheque and is credited the *full cumulative* +//! instead of the delta, repeatably, for free (§11.4). Hence one atomic +//! write covering all of it, never a field at a time. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Refuse a cumulative past this. Total BZZ supply is 10^8 BZZ = 10^24 +/// PLUR, so anything above 10^30 is not a payment — it is an attempt to +/// find an overflow. Bounding it here keeps every downstream figure in +/// `u128` (max ≈ 3.4×10^38) with room to spare. +pub const MAX_CUMULATIVE_PLUR: u128 = 1_000_000_000_000_000_000_000_000_000_000; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum LedgerError { + #[error("cheque cumulative {got} is not greater than the {have} already accepted")] + NotIncreasing { got: u128, have: u128 }, + #[error("chequebook 0x{chequebook} is already bound to a different account")] + ChequebookBound { chequebook: String }, + #[error("cumulative {0} is implausibly large")] + Absurd(u128), + #[error("cheque credits {got} but only {owed} is owed")] + Overpayment { got: u128, owed: u128 }, +} + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("io: {0}")] + Io(String), + #[error("json: {0}")] + Json(#[from] serde_json::Error), +} + +/// Outcome of admitting a request against a credit line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Admission { + pub reserved_plur: u128, + /// `owed + reserved` after this reservation. + pub outstanding_plur: u128, + pub cap_plur: u128, + /// True when this request pushed the account past its line. Soft mode + /// records it; hard mode releases the reservation and answers 402. + pub over_cap: bool, +} + +#[derive(Debug, Default, Clone)] +struct Account { + owed_plur: u128, + /// Deliberately absent from the on-disk form. See the module docs. + reserved_plur: u128, + last_cumulative: HashMap<[u8; 20], HeldCheque>, +} + +/// The latest cheque accepted from one chequebook. +/// +/// **The signature has to be kept.** Cheques are cumulative, so only the +/// newest one is ever worth presenting (§7.2) — but without its signature +/// the relay holds a number it can prove nothing about and can never cash. +/// The first cut stored only the cumulative, which made the whole ledger a +/// record of money that could not be collected. +#[derive(Debug, Clone, Copy)] +pub struct HeldCheque { + pub cumulative_plur: u128, + pub signature: [u8; 65], +} + +impl Account { + fn outstanding(&self) -> u128 { + self.owed_plur.saturating_add(self.reserved_plur) + } +} + +pub struct Ledger { + accounts: HashMap<[u8; 20], Account>, + /// A chequebook belongs to the first account that paid with it and + /// cannot move. Without this, two accounts could share a cumulative and + /// one would ride the other's payments. + binding: HashMap<[u8; 20], [u8; 20]>, + secret: [u8; 32], + path: Option, +} + +// ── On-disk form ───────────────────────────────────────────────────────── +// Hex-keyed and decimal-stringed: JSON has no u128, and PLUR amounts +// comfortably exceed what an f64 can hold exactly. + +#[derive(Serialize, Deserialize)] +struct OnDisk { + version: u32, + secret_hex: String, + accounts: Vec, + binding: Vec<(String, String)>, +} + +#[derive(Serialize, Deserialize)] +struct OnDiskAccount { + account: String, + owed_plur: String, + /// `(chequebook, cumulative, signature_hex)`. Version 1 files carry no + /// signature; they load, but nothing in them can be cashed. + last_cumulative: Vec<(String, String, String)>, +} + +impl Ledger { + /// Fresh ledger with a random secret. Used when no `--state-dir` is + /// configured, which metered mode forbids (§5) but open mode allows. + pub fn ephemeral() -> Self { + let mut secret = [0u8; 32]; + // A failure here would mean no entropy source at all; a zero secret + // would make every nonce forgeable, so refuse to run instead. + getrandom::fill(&mut secret).expect("system entropy for the relay secret"); + Self { + accounts: HashMap::new(), + binding: HashMap::new(), + secret, + path: None, + } + } + + pub fn secret(&self) -> &[u8; 32] { + &self.secret + } + + /// Load from disk, or create and immediately persist a new ledger. + /// + /// `relay_secret` is part of the same file: regenerating it at boot + /// invalidates every outstanding challenge, which on a host that sleeps + /// and cold-starts turns each restart into a 403 storm for clients + /// mid-upload (§7.2). + pub fn load_or_create>(path: P) -> Result { + let path = path.as_ref().to_path_buf(); + let text = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let mut fresh = Self::ephemeral(); + fresh.path = Some(path); + fresh.persist()?; + return Ok(fresh); + } + Err(e) => return Err(StoreError::Io(e.to_string())), + }; + let disk: OnDisk = serde_json::from_str(&text)?; + let mut secret = [0u8; 32]; + let raw = hex::decode(disk.secret_hex.trim_start_matches("0x")) + .map_err(|e| StoreError::Io(format!("secret hex: {e}")))?; + if raw.len() != 32 { + return Err(StoreError::Io(format!( + "relay secret must be 32 bytes, got {}", + raw.len() + ))); + } + secret.copy_from_slice(&raw); + + let mut accounts = HashMap::new(); + for a in disk.accounts { + let key = parse_addr(&a.account)?; + let mut last_cumulative = HashMap::new(); + for (cb, v, sig_hex) in a.last_cumulative { + let raw = hex::decode(sig_hex.trim_start_matches("0x")) + .map_err(|e| StoreError::Io(format!("cheque signature hex: {e}")))?; + // A v1 entry has no signature. Keep the cumulative — losing + // it would let the client replay (§11.4) — but leave the + // signature zeroed so cashout skips it rather than + // submitting something the contract will reject. + let mut signature = [0u8; 65]; + if raw.len() == 65 { + signature.copy_from_slice(&raw); + } + last_cumulative.insert( + parse_addr(&cb)?, + HeldCheque { + cumulative_plur: parse_u128(&v)?, + signature, + }, + ); + } + accounts.insert( + key, + Account { + owed_plur: parse_u128(&a.owed_plur)?, + // Never restored. See the module docs. + reserved_plur: 0, + last_cumulative, + }, + ); + } + let mut binding = HashMap::new(); + for (cb, acct) in disk.binding { + binding.insert(parse_addr(&cb)?, parse_addr(&acct)?); + } + Ok(Self { + accounts, + binding, + secret, + path: Some(path), + }) + } + + /// Write the durable half atomically: a temp file in the same directory + /// followed by a rename, so a crash mid-write leaves the previous state + /// rather than a truncated one. Losing `last_cumulative` while keeping + /// `owed` is the specific corruption this prevents. + pub fn persist(&self) -> Result<(), StoreError> { + let Some(path) = &self.path else { + return Ok(()); + }; + let mut accounts: Vec = self + .accounts + .iter() + .filter(|(_, a)| a.owed_plur > 0 || !a.last_cumulative.is_empty()) + .map(|(k, a)| { + let mut last: Vec<(String, String, String)> = a + .last_cumulative + .iter() + .map(|(cb, c)| { + ( + hex::encode(cb), + c.cumulative_plur.to_string(), + hex::encode(c.signature), + ) + }) + .collect(); + last.sort(); + OnDiskAccount { + account: hex::encode(k), + owed_plur: a.owed_plur.to_string(), + last_cumulative: last, + } + }) + .collect(); + accounts.sort_by(|a, b| a.account.cmp(&b.account)); + let mut binding: Vec<(String, String)> = self + .binding + .iter() + .map(|(cb, a)| (hex::encode(cb), hex::encode(a))) + .collect(); + binding.sort(); + + let disk = OnDisk { + version: 2, + secret_hex: hex::encode(self.secret), + accounts, + binding, + }; + let body = serde_json::to_vec_pretty(&disk)?; + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, &body).map_err(|e| StoreError::Io(e.to_string()))?; + std::fs::rename(&tmp, path).map_err(|e| StoreError::Io(e.to_string()))?; + Ok(()) + } + + pub fn owed(&self, account: &[u8; 20]) -> u128 { + self.accounts.get(account).map_or(0, |a| a.owed_plur) + } + + pub fn reserved(&self, account: &[u8; 20]) -> u128 { + self.accounts.get(account).map_or(0, |a| a.reserved_plur) + } + + pub fn outstanding(&self, account: &[u8; 20]) -> u128 { + self.accounts.get(account).map_or(0, Account::outstanding) + } + + pub fn last_cumulative(&self, account: &[u8; 20], chequebook: &[u8; 20]) -> u128 { + self.accounts + .get(account) + .and_then(|a| a.last_cumulative.get(chequebook)) + .map(|c| c.cumulative_plur) + .unwrap_or(0) + } + + /// Every cheque the relay holds, newest per chequebook. This is what + /// `hoverfly cashout` presents on-chain. + pub fn held_cheques(&self) -> Vec<([u8; 20], [u8; 20], HeldCheque)> { + let mut out = Vec::new(); + for (account, a) in &self.accounts { + for (cb, held) in &a.last_cumulative { + out.push((*account, *cb, *held)); + } + } + out.sort_by_key(|(_, cb, _)| *cb); + out + } + + /// Number of accounts holding a live reservation — the cardinality + /// §7.2 says to bound, since the map is attacker-influenced. + pub fn live_reservations(&self) -> usize { + self.accounts + .values() + .filter(|a| a.reserved_plur > 0) + .count() + } + + /// Reserve against a credit line, atomically with respect to every + /// other in-flight request for this account. + /// + /// Always reserves, and *reports* whether it went over rather than + /// deciding: soft mode records the overshoot and serves anyway, hard + /// mode releases and answers 402 (§7.1). Doing the arithmetic in one + /// place means the two modes cannot disagree about what "over" means. + pub fn reserve(&mut self, account: [u8; 20], amount: u128, cap: u128) -> Admission { + let a = self.accounts.entry(account).or_default(); + a.reserved_plur = a.reserved_plur.saturating_add(amount); + let outstanding = a.outstanding(); + Admission { + reserved_plur: amount, + outstanding_plur: outstanding, + cap_plur: cap, + over_cap: outstanding > cap, + } + } + + /// Give back an unused reservation, e.g. after a hard-mode 402. + pub fn release(&mut self, account: [u8; 20], amount: u128) { + if let Some(a) = self.accounts.get_mut(&account) { + a.reserved_plur = a.reserved_plur.saturating_sub(amount); + } + } + + /// Turn a reservation into debt for what was actually admitted, and + /// release the remainder. + pub fn commit(&mut self, account: [u8; 20], reserved: u128, billed: u128) { + let a = self.accounts.entry(account).or_default(); + a.reserved_plur = a.reserved_plur.saturating_sub(reserved); + a.owed_plur = a.owed_plur.saturating_add(billed); + } + + /// Accept a cheque and return the amount it newly credits. + /// + /// Monotonicity is what makes a cheque replay-proof *within a live + /// relay* — a re-presented cheque credits zero — and it is why losing + /// this map across a restart is an unbounded free-service loop (§11.4). + pub fn credit( + &mut self, + account: [u8; 20], + chequebook: [u8; 20], + cumulative_plur: u128, + signature: [u8; 65], + ) -> Result { + if cumulative_plur > MAX_CUMULATIVE_PLUR { + return Err(LedgerError::Absurd(cumulative_plur)); + } + match self.binding.get(&chequebook) { + Some(bound) if *bound != account => { + return Err(LedgerError::ChequebookBound { + chequebook: hex::encode(chequebook), + }); + } + _ => {} + } + let a = self.accounts.entry(account).or_default(); + let have = a + .last_cumulative + .get(&chequebook) + .map(|c| c.cumulative_plur) + .unwrap_or(0); + if cumulative_plur <= have { + return Err(LedgerError::NotIncreasing { + got: cumulative_plur, + have, + }); + } + let delta = cumulative_plur - have; + // Refuse to bank more than is owed. Postpaid means a client should + // never be ahead (§10), and accepting an overpayment would turn the + // relay into a place to park value it cannot return. + if delta > a.owed_plur { + return Err(LedgerError::Overpayment { + got: delta, + owed: a.owed_plur, + }); + } + a.last_cumulative.insert( + chequebook, + HeldCheque { + cumulative_plur, + signature, + }, + ); + a.owed_plur -= delta; + self.binding.insert(chequebook, account); + Ok(delta) + } +} + +fn parse_addr(s: &str) -> Result<[u8; 20], StoreError> { + let raw = hex::decode(s.trim_start_matches("0x")) + .map_err(|e| StoreError::Io(format!("address hex: {e}")))?; + if raw.len() != 20 { + return Err(StoreError::Io(format!("address must be 20 bytes: {s}"))); + } + let mut out = [0u8; 20]; + out.copy_from_slice(&raw); + Ok(out) +} + +fn parse_u128(s: &str) -> Result { + s.parse() + .map_err(|e| StoreError::Io(format!("u128 parse {s}: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: [u8; 20] = [1u8; 20]; + const B: [u8; 20] = [2u8; 20]; + const CB: [u8; 20] = [9u8; 20]; + + fn tmpdir() -> PathBuf { + let base = std::env::var("CARGO_TARGET_TMPDIR").unwrap_or_else(|_| "/tmp".into()); + let p = PathBuf::from(base).join(format!("ledger-test-{}", std::process::id())); + std::fs::create_dir_all(&p).expect("mkdir"); + p + } + + #[test] + fn a_reservation_shows_up_as_outstanding_and_releases_cleanly() { + let mut l = Ledger::ephemeral(); + let adm = l.reserve(A, 500, 1000); + assert_eq!(adm.outstanding_plur, 500); + assert!(!adm.over_cap); + assert_eq!(l.reserved(&A), 500); + l.release(A, 500); + assert_eq!(l.outstanding(&A), 0); + } + + /// The concurrency case §10.2 is about: N requests each reserve before + /// any of them completes, so the cap must see their sum. + #[test] + fn concurrent_reservations_accumulate_against_one_cap() { + let mut l = Ledger::ephemeral(); + for _ in 0..7 { + assert!(!l.reserve(A, 100, 1000).over_cap); + } + let adm = l.reserve(A, 100, 1000); + assert_eq!(adm.outstanding_plur, 800); + assert!(!adm.over_cap); + let adm = l.reserve(A, 300, 1000); + assert!(adm.over_cap, "1100 > 1000 must report over cap"); + } + + #[test] + fn commit_turns_a_reservation_into_debt_and_frees_the_rest() { + let mut l = Ledger::ephemeral(); + l.reserve(A, 1000, 10_000); + l.commit(A, 1000, 240); + assert_eq!(l.reserved(&A), 0, "the whole reservation is released"); + assert_eq!(l.owed(&A), 240, "only what was admitted is billed"); + } + + #[test] + fn a_cheque_credits_only_the_delta() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + assert_eq!(l.credit_test(A, CB, 400).expect("first"), 400); + assert_eq!(l.owed(&A), 600); + assert_eq!(l.credit_test(A, CB, 900).expect("second"), 500); + assert_eq!(l.owed(&A), 100); + } + + /// Replay within a live relay must be worth exactly zero. + #[test] + fn a_re_presented_cheque_credits_nothing() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + l.credit_test(A, CB, 400).expect("first"); + assert_eq!( + l.credit_test(A, CB, 400), + Err(LedgerError::NotIncreasing { + got: 400, + have: 400 + }) + ); + assert_eq!(l.owed(&A), 600, "owed must not move"); + } + + #[test] + fn a_chequebook_cannot_move_between_accounts() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + l.credit_test(A, CB, 100).expect("bind to A"); + l.commit(B, 0, 1000); + assert!(matches!( + l.credit_test(B, CB, 500), + Err(LedgerError::ChequebookBound { .. }) + )); + } + + #[test] + fn absurd_and_overpaying_cumulatives_are_refused() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 100); + assert!(matches!( + l.credit_test(A, CB, MAX_CUMULATIVE_PLUR + 1), + Err(LedgerError::Absurd(_)) + )); + assert!(matches!( + l.credit_test(A, CB, 101), + Err(LedgerError::Overpayment { + got: 101, + owed: 100 + }) + )); + l.credit_test(A, CB, 100) + .expect("paying exactly what is owed is fine"); + assert_eq!(l.owed(&A), 0); + } + + /// The asymmetry the module exists to enforce: debt and the cumulative + /// high-water mark survive a restart; a reservation does not. + #[test] + fn a_restart_keeps_debt_and_cumulative_but_drops_reservations() { + let dir = tmpdir(); + let path = dir.join("restart.json"); + let _ = std::fs::remove_file(&path); + + let secret = { + let mut l = Ledger::load_or_create(&path).expect("create"); + l.commit(A, 0, 5000); + l.credit_test(A, CB, 1200).expect("pay"); + l.reserve(A, 900, 100_000); + l.persist().expect("persist"); + *l.secret() + }; + + let l = Ledger::load_or_create(&path).expect("reload"); + assert_eq!(l.owed(&A), 3800, "debt survives"); + assert_eq!(l.last_cumulative(&A, &CB), 1200, "cumulative survives"); + assert_eq!( + l.reserved(&A), + 0, + "a reservation must NOT survive — no task remains to release it" + ); + assert_eq!( + l.secret(), + &secret, + "a regenerated secret would 403 every live client" + ); + let _ = std::fs::remove_file(&path); + } + + /// §11.4's attack: pay once, consume, wait for a restart, re-present the + /// same cheque. It must still credit zero. + #[test] + fn a_cheque_cannot_be_replayed_across_a_restart() { + let dir = tmpdir(); + let path = dir.join("replay.json"); + let _ = std::fs::remove_file(&path); + { + let mut l = Ledger::load_or_create(&path).expect("create"); + l.commit(A, 0, 5000); + l.credit_test(A, CB, 1200).expect("pay"); + l.persist().expect("persist"); + } + let mut l = Ledger::load_or_create(&path).expect("reload"); + assert!( + matches!( + l.credit_test(A, CB, 1200), + Err(LedgerError::NotIncreasing { .. }) + ), + "re-presenting the same cheque after a restart must credit nothing" + ); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn the_binding_survives_a_restart_too() { + let dir = tmpdir(); + let path = dir.join("binding.json"); + let _ = std::fs::remove_file(&path); + { + let mut l = Ledger::load_or_create(&path).expect("create"); + l.commit(A, 0, 500); + l.credit_test(A, CB, 100).expect("bind"); + l.persist().expect("persist"); + } + let mut l = Ledger::load_or_create(&path).expect("reload"); + l.commit(B, 0, 500); + assert!( + matches!( + l.credit_test(B, CB, 200), + Err(LedgerError::ChequebookBound { .. }) + ), + "the chequebook binding must survive a restart" + ); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn live_reservations_are_countable_for_shedding() { + let mut l = Ledger::ephemeral(); + assert_eq!(l.live_reservations(), 0); + l.reserve(A, 10, 1000); + l.reserve(B, 10, 1000); + assert_eq!(l.live_reservations(), 2); + l.release(A, 10); + assert_eq!(l.live_reservations(), 1); + } +} + +#[cfg(test)] +mod leak_tests { + //! A reservation that is never committed must never survive. + //! + //! Found by review: four early-return paths in `push_response` (oversize + //! body, read timeout, frame-decode failure, empty batch) dropped the + //! admission without releasing, and nothing else lowers `reserved_plur` + //! — `credit` only touches `owed`. These pin the ledger-level invariants + //! the RAII guard in `pusher.rs` relies on. + use super::*; + + const A: [u8; 20] = [1u8; 20]; + const CB: [u8; 20] = [9u8; 20]; + + /// Paying does **not** clear a reservation. This is the property that + /// turns a leak into a permanent one, so it is worth stating outright. + #[test] + fn paying_a_cheque_does_not_release_a_reservation() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + l.reserve(A, 500, 100_000); + l.credit_test(A, CB, 1000).expect("pay off the debt"); + assert_eq!(l.owed(&A), 0, "the debt is cleared"); + assert_eq!( + l.reserved(&A), + 500, + "but the reservation is untouched — only commit or release move it" + ); + } + + /// The ratchet: leaked reservations accumulate until `outstanding` + /// exceeds the cap, and no cheque can bring it back down. + #[test] + fn leaked_reservations_ratchet_an_account_past_its_cap_with_no_way_back() { + let mut l = Ledger::ephemeral(); + let cap = 10_000u128; + for _ in 0..20 { + l.reserve(A, 1000, cap); // admitted, then dropped without commit + } + assert!(l.outstanding(&A) > cap, "the account is now over its cap"); + // There is no debt to pay, so no cheque exists that could help. + assert_eq!(l.owed(&A), 0); + assert!( + matches!( + l.credit_test(A, CB, 1), + Err(LedgerError::Overpayment { .. }) + ), + "with nothing owed, a cheque cannot clear the overshoot" + ); + // Only releasing does. + for _ in 0..20 { + l.release(A, 1000); + } + assert_eq!(l.outstanding(&A), 0); + } + + /// Releasing an unused reservation must leave no residue at all — this + /// is what every early-return path now does via `Drop`. + #[test] + fn releasing_an_unused_reservation_leaves_nothing_behind() { + let mut l = Ledger::ephemeral(); + let adm = l.reserve(A, 4096, 100_000); + l.release(A, adm.reserved_plur); + assert_eq!(l.outstanding(&A), 0); + assert_eq!(l.live_reservations(), 0, "and frees its shed-cap slot"); + } + + /// Committing zero bytes is equivalent to releasing: a POST that + /// admitted nothing owes nothing and holds nothing. + #[test] + fn committing_nothing_is_equivalent_to_releasing() { + let mut l = Ledger::ephemeral(); + let adm = l.reserve(A, 4096, 100_000); + l.commit(A, adm.reserved_plur, 0); + assert_eq!(l.outstanding(&A), 0); + assert_eq!(l.live_reservations(), 0); + } +} + +#[cfg(test)] +impl Ledger { + /// Test shim: cheques in unit tests carry a dummy signature, since the + /// ledger never inspects it — only `hoverfly cashout` does. + fn credit_test( + &mut self, + account: [u8; 20], + chequebook: [u8; 20], + cumulative_plur: u128, + ) -> Result { + let mut sig = [0u8; 65]; + sig[64] = 27; + self.credit(account, chequebook, cumulative_plur, sig) + } +} diff --git a/src/lib.rs b/src/lib.rs index 1362921..a5d98e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,6 +68,32 @@ pub mod daemon; #[cfg(not(target_arch = "wasm32"))] pub mod inbound; +// The incentive layer has two halves, and only one of them is the relay. +// `pusher` is the relay feature — it exists to pull in hyper. A *client* +// paying a metered relay needs the challenge wire format, the pricing +// arithmetic and the payer, but no server: gating those behind `pusher` +// made `--no-default-features --features cli` fail to build, because +// `client.rs` reaches for them unconditionally on every relay push. +#[cfg(not(target_arch = "wasm32"))] +pub mod challenge; + +#[cfg(not(target_arch = "wasm32"))] +pub mod meter; + +#[cfg(not(target_arch = "wasm32"))] +pub mod payer; + +// Relay-side only, and genuinely so: these hold the ledger the relay bills +// against and the state machine that defends it. +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod inbound_limit; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod ledger; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod metered; + #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub mod pusher; diff --git a/src/meter.rs b/src/meter.rs new file mode 100644 index 0000000..ad96ccd --- /dev/null +++ b/src/meter.rs @@ -0,0 +1,873 @@ +//! Stage 0 shadow metering — `docs/pusher-incentives.md` §14. +//! +//! Measures what a *metered* relay would have billed, while changing +//! nothing on the wire. No new client-visible behaviour, no refusals, no +//! payment: the relay simply counts, and an operator reads the counts off +//! `/v1/meter`. +//! +//! It answers the two questions Stage 1 is gated on: +//! +//! 1. **Is anyone consuming enough to justify metering?** Per-account bytes +//! admitted, what they would owe at §9.2's candidate price, and how many +//! accounts would ever cross the cashout threshold — not a profitability +//! line (§9.3's gas is negligible), but the volume at which cashing out +//! is worth an operator's round trip at all. +//! 2. **Is `credit_ratio = 1000` right?** For every batch actually seen, the +//! credit line §10.3 would have granted it, against the size of a full +//! POST. A batch whose line is under one POST has to split its uploads, +//! which is fine; a population where most batches are in that state means +//! the ratio is wrong. +//! +//! It also reports §9.1's egress multiplier, which the doc *estimates* at +//! ×3 racing × 1.15 shallow ≈ 3.45 attempts per chunk — the number that +//! sets the whole cost basis. **The reported figure is not yet a valid +//! measurement of it.** It divides completed push outcomes by frames +//! admitted, and the racing dispatcher cancels losing racers before they +//! complete, so their egress is spent but uncounted (see `stream_attempts` +//! in `src/pusher.rs`). Treat it as a floor, not an observation. +//! +//! **Hot path cost is one lock per POST.** A request accumulates into a +//! [`PostTally`] on its own stack and merges once at completion, so N +//! concurrent POSTs contend N times rather than N × `PUSH_BATCH_MAX`. +//! +//! **Known limitation:** state is in-memory, so a restart resets it. That +//! biases "do accounts return?" downward on hosts that sleep, which is +//! exactly the free tier §5 says must not run metered anyway. Read the +//! window length (`window_secs`) before drawing conclusions from repeat +//! rates. + +use serde_json::json; +use std::collections::{HashMap, VecDeque}; +use std::time::Instant; + +// ── Candidate parameters (docs/pusher-incentives.md §9.2, §10.1) ───────── +// +// Constants rather than flags on purpose: Stage 0 exists to decide whether +// these numbers are right, and every figure in the report is derived from +// the same raw observations, so changing one re-derives the report without +// re-collecting anything. + +/// $0.02/GiB at $0.40/BZZ → 5e14 PLUR/GiB ÷ 1 048 576 KiB. +pub const PRICE_PLUR_PER_KIB: u128 = 480_000_000; +/// ~32 MiB — one settlement window. +pub const SETTLE_EVERY_PLUR: u128 = 15_600_000_000_000; +/// ~127 MiB — the global ceiling on a credit line. +pub const MAX_OUTSTANDING_PLUR: u128 = 62_200_000_000_000; +/// 0.25 BZZ ≈ 5 GiB. Not a break-even: §9.3's measured cashout gas is ~1e-10 +/// xDAI, so any non-zero cheque repays it. This is a batching convenience — +/// how much value to let accumulate before spending an RPC round trip and a +/// pending transaction on it — and should be derived from `eth_gasPrice` +/// rather than pinned, since Gnosis gas will not stay this cheap. +pub const CASHOUT_THRESHOLD_PLUR: u128 = 2_500_000_000_000_000; +/// Credit line = batch remaining value ÷ this (§10.3). +pub const CREDIT_RATIO: u128 = 1_000; + +const PLUR_PER_BZZ: f64 = 1e16; +const USD_PER_BZZ: f64 = 0.40; + +/// Distinct `(owner, batch)` pairs held before FIFO eviction. Bounded for +/// the same reason the owner cache is (§16.2): the key is attacker-chosen, +/// so an unbounded map is a memory DoS. Evicted rows keep contributing to +/// the totals, only their per-row detail is lost. +const METER_ROW_CAP: usize = 4096; + +/// A shadow account is the batch-owner EOA (§6), and credit is keyed one +/// level finer, on the batch — so the ledger is keyed on both. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +pub struct AccountBatch { + pub owner: [u8; 20], + pub batch: [u8; 32], +} + +#[derive(Clone)] +struct Row { + /// Body bytes attributable to admitted frames, in KiB (§8). + kib_admitted: u64, + /// Of which served from the recent-ack cache, and so billed at zero + /// (§8.2). + kib_dedup: u64, + frames: u64, + dedup_frames: u64, + /// `remainingBalance × 2^depth` observed at resolution (§6). Zero when + /// the batch was never resolved with a value, which cannot currently + /// happen but is not worth panicking over. + remaining_value_plur: u128, + first_seen: Instant, + last_seen: Instant, +} + +impl Row { + fn billable_kib(&self) -> u64 { + self.kib_admitted.saturating_sub(self.kib_dedup) + } + + /// What §10.3 would have granted this batch. + fn credit_plur(&self) -> u128 { + (self.remaining_value_plur / CREDIT_RATIO).min(MAX_OUTSTANDING_PLUR) + } + + fn credit_kib(&self) -> u64 { + (self.credit_plur() / PRICE_PLUR_PER_KIB).min(u64::MAX as u128) as u64 + } +} + +/// One owner's rollup across every batch it pushed under. The account is +/// what settlement and cashout are keyed on (§6, §9.3), while credit is +/// keyed per batch — so the report needs both views of the same rows. +#[derive(Default, Clone, Copy)] +struct OwnerTotals { + kib: u64, + kib_dedup: u64, + frames: u64, + batches: u64, +} + +impl OwnerTotals { + fn owed_plur(&self) -> u128 { + self.kib.saturating_sub(self.kib_dedup) as u128 * PRICE_PLUR_PER_KIB + } +} + +/// Per-request staging buffer. A POST touches at most +/// `PUSH_MAX_BATCH_LOOKUPS` distinct batches, so a linear scan beats a +/// `HashMap` and allocates nothing in the common single-batch case. +#[derive(Default)] +pub struct PostTally { + rows: Vec, +} + +struct TallyRow { + key: AccountBatch, + remaining_value_plur: u128, + bytes_admitted: u64, + bytes_dedup: u64, + frames: u64, + dedup_frames: u64, +} + +impl PostTally { + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + fn slot(&mut self, key: AccountBatch, remaining_value_plur: u128) -> &mut TallyRow { + if let Some(i) = self.rows.iter().position(|r| r.key == key) { + // A later frame may carry a value the first one lacked; never + // let a zero overwrite a real reading. + if self.rows[i].remaining_value_plur == 0 { + self.rows[i].remaining_value_plur = remaining_value_plur; + } + return &mut self.rows[i]; + } + self.rows.push(TallyRow { + key, + remaining_value_plur, + bytes_admitted: 0, + bytes_dedup: 0, + frames: 0, + dedup_frames: 0, + }); + self.rows.last_mut().expect("just pushed") + } + + /// A frame that passed stamp validation and owner resolution. `bytes` is + /// the body it occupied — header plus wire — which is what §8 bills. + pub fn admit(&mut self, key: AccountBatch, remaining_value_plur: u128, bytes: u64) { + let row = self.slot(key, remaining_value_plur); + row.bytes_admitted += bytes; + row.frames += 1; + } + + /// An admitted frame that the recent-ack cache answered. Counted in + /// `admit` as well — this records the portion billed at zero (§8.2). + pub fn dedup(&mut self, key: AccountBatch, remaining_value_plur: u128, bytes: u64) { + let row = self.slot(key, remaining_value_plur); + row.bytes_dedup += bytes; + row.dedup_frames += 1; + } +} + +/// Bounded shadow ledger over `(owner, batch)`. +pub struct Meter { + rows: HashMap, + order: VecDeque, + cap: usize, + started: Instant, + /// Totals that survive eviction, so the headline figures stay honest + /// even once detail rows are dropped. + evicted_rows: u64, + evicted_kib: u64, + evicted_kib_dedup: u64, + evicted_frames: u64, +} + +impl Default for Meter { + fn default() -> Self { + Self::new(METER_ROW_CAP) + } +} + +impl Meter { + pub fn new(cap: usize) -> Self { + Self { + rows: HashMap::new(), + order: VecDeque::new(), + cap: cap.max(1), + started: Instant::now(), + evicted_rows: 0, + evicted_kib: 0, + evicted_kib_dedup: 0, + evicted_frames: 0, + } + } + + /// Fold one request's tally in. Bytes become KiB here, once per + /// `(POST, batch)` — which is how a metered relay would round, since it + /// bills a whole body against one `Content-Length` (§8). + pub fn merge(&mut self, tally: PostTally) { + let now = Instant::now(); + for t in tally.rows { + let kib = t.bytes_admitted.div_ceil(1024); + let kib_dedup = t.bytes_dedup.div_ceil(1024).min(kib); + match self.rows.get_mut(&t.key) { + Some(row) => { + row.kib_admitted += kib; + row.kib_dedup += kib_dedup; + row.frames += t.frames; + row.dedup_frames += t.dedup_frames; + if t.remaining_value_plur > 0 { + // Latest reading wins: the line decays as the batch + // is spent down, and the decay is the point (§10.3). + row.remaining_value_plur = t.remaining_value_plur; + } + row.last_seen = now; + } + None => { + self.rows.insert( + t.key, + Row { + kib_admitted: kib, + kib_dedup, + frames: t.frames, + dedup_frames: t.dedup_frames, + remaining_value_plur: t.remaining_value_plur, + first_seen: now, + last_seen: now, + }, + ); + self.order.push_back(t.key); + } + } + } + while self.order.len() > self.cap { + let Some(old) = self.order.pop_front() else { + break; + }; + if let Some(row) = self.rows.remove(&old) { + self.evicted_rows += 1; + self.evicted_kib += row.kib_admitted; + self.evicted_kib_dedup += row.kib_dedup; + self.evicted_frames += row.frames; + } + } + } + + fn window_secs(&self) -> u64 { + self.started.elapsed().as_secs() + } + + fn by_owner(&self) -> HashMap<[u8; 20], OwnerTotals> { + let mut out: HashMap<[u8; 20], OwnerTotals> = HashMap::new(); + for (k, r) in &self.rows { + let e = out.entry(k.owner).or_default(); + e.kib += r.kib_admitted; + e.kib_dedup += r.kib_dedup; + e.frames += r.frames; + e.batches += 1; + } + out + } + + /// Headline figures. Names no account, so it is safe on the public + /// `/v1/status` — total volume is already published there as + /// `bytes_pushed`. + /// + /// `full_post_kib` is the body of a maximal POST, and `attempts` the sum + /// of the per-stream push outcome counters (§9.1). + pub fn summary(&self, full_post_kib: u64, attempts: u64) -> serde_json::Value { + let live_kib: u64 = self.rows.values().map(|r| r.kib_admitted).sum(); + let live_dedup: u64 = self.rows.values().map(|r| r.kib_dedup).sum(); + let live_frames: u64 = self.rows.values().map(|r| r.frames).sum(); + let kib = live_kib + self.evicted_kib; + let dedup = live_kib.min(live_dedup) + self.evicted_kib_dedup; + let frames = live_frames + self.evicted_frames; + let billable = kib.saturating_sub(dedup); + let owed = billable as u128 * PRICE_PLUR_PER_KIB; + + let owners = self.by_owner(); + let would_settle = owners + .values() + .filter(|t| t.owed_plur() >= SETTLE_EVERY_PLUR) + .count(); + let would_cash = owners + .values() + .filter(|t| t.owed_plur() >= CASHOUT_THRESHOLD_PLUR) + .count(); + + json!({ + "window_secs": self.window_secs(), + "price_plur_per_kib": PRICE_PLUR_PER_KIB.to_string(), + "accounts": owners.len(), + "batches": self.rows.len(), + "evicted_batches": self.evicted_rows, + "kib_admitted": kib, + "kib_dedup": dedup, + "frames_admitted": frames, + "owed_plur": owed.to_string(), + "owed_usd": plur_to_usd(owed), + // The two questions §9.3 turns on: how many accounts reach one + // settlement at all, and how many ever repay their cashout gas. + "accounts_reaching_settlement": would_settle, + "accounts_reaching_cashout": would_cash, + "egress": egress(frames, billable, attempts), + "credit": self.credit_summary(full_post_kib), + }) + } + + /// §10.3's calibration: what credit line every observed batch would get. + fn credit_summary(&self, full_post_kib: u64) -> serde_json::Value { + let mut lines: Vec = self + .rows + .values() + .filter(|r| r.remaining_value_plur > 0) + .map(Row::credit_kib) + .collect(); + if lines.is_empty() { + return json!({"batches_priced": 0}); + } + lines.sort_unstable(); + let below_post = lines.iter().filter(|&&c| c < full_post_kib).count(); + let capped = self + .rows + .values() + .filter(|r| r.remaining_value_plur > 0) + .filter(|r| r.remaining_value_plur / CREDIT_RATIO < MAX_OUTSTANDING_PLUR) + .count(); + json!({ + "batches_priced": lines.len(), + "credit_ratio": CREDIT_RATIO, + "full_post_kib": full_post_kib, + // A batch here must split its uploads across smaller POSTs + // (§7.2). Fine individually; a high fraction means the ratio is + // mis-set. + "batches_below_one_full_post": below_post, + // Below the global ceiling, i.e. the per-batch line binds. + "batches_capped_below_ceiling": capped, + "credit_kib_p10": pct(&lines, 10), + "credit_kib_p50": pct(&lines, 50), + "credit_kib_p90": pct(&lines, 90), + }) + } + + /// Per-account and per-batch detail. **Operator-only** — this is a + /// volume oracle over on-chain-enumerable batch owners, which is exactly + /// why §7 authenticates `/v1/account`. + pub fn detail(&self, full_post_kib: u64, attempts: u64, top: usize) -> serde_json::Value { + let mut owners: Vec<([u8; 20], OwnerTotals)> = self.by_owner().into_iter().collect(); + owners.sort_by(|a, b| b.1.kib.cmp(&a.1.kib).then(a.0.cmp(&b.0))); + let accounts: Vec = owners + .iter() + .take(top) + .map(|(owner, t)| { + let owed = t.owed_plur(); + json!({ + "account": format!("0x{}", hex::encode(owner)), + "batches": t.batches, + "kib_admitted": t.kib, + "kib_dedup": t.kib_dedup, + "frames": t.frames, + "owed_plur": owed.to_string(), + "owed_usd": plur_to_usd(owed), + "settlements": (owed / SETTLE_EVERY_PLUR) as u64, + "reaches_cashout": owed >= CASHOUT_THRESHOLD_PLUR, + }) + }) + .collect(); + + let mut rows: Vec<(&AccountBatch, &Row)> = self.rows.iter().collect(); + rows.sort_by(|a, b| b.1.kib_admitted.cmp(&a.1.kib_admitted).then(a.0.cmp(b.0))); + let batches: Vec = rows + .iter() + .take(top) + .map(|(k, r)| { + let credit_kib = r.credit_kib(); + json!({ + "account": format!("0x{}", hex::encode(k.owner)), + "batch": format!("0x{}", hex::encode(k.batch)), + "kib_admitted": r.kib_admitted, + "kib_dedup": r.kib_dedup, + "frames": r.frames, + "dedup_frames": r.dedup_frames, + "remaining_value_plur": r.remaining_value_plur.to_string(), + "credit_plur": r.credit_plur().to_string(), + "credit_kib": credit_kib, + "fits_full_post": credit_kib >= full_post_kib, + "billable_kib": r.billable_kib(), + "age_secs": r.first_seen.elapsed().as_secs(), + "idle_secs": r.last_seen.elapsed().as_secs(), + }) + }) + .collect(); + + json!({ + "summary": self.summary(full_post_kib, attempts), + "accounts": accounts, + "batches": batches, + "truncated_to": top, + }) + } +} + +/// §9.1's cost basis. The doc's model is ×3 peer race × 1.15 shallow +/// retries ≈ 3.45 stream attempts per chunk. +/// +/// `attempts` counts *completed* outcomes, so this is a lower bound rather +/// than the observation §9.2 needs — cancelled racers are missing. Do not +/// reprice on it until the counter moves to dispatch. +fn egress(frames: u64, billable_kib: u64, attempts: u64) -> serde_json::Value { + if frames == 0 { + return json!({"frames": 0, "stream_attempts": attempts}); + } + let per = attempts as f64 / frames as f64; + let mut out = json!({ + "frames": frames, + "stream_attempts": attempts, + // The measured number. Everything below is derived from it. + "attempts_per_frame": round3(per), + "modelled_attempts_per_frame": 3.45, + }); + if billable_kib > 0 { + // Each Delivery is ~4.4 KiB on the wire for a 4 KiB chunk (§9.1: + // addr + stamp + span/data, plus ~5 % protobuf/yamux/noise/TCP). + // Denominated in *billable* payload, since a dedup hit generates no + // attempts and would otherwise dilute the ratio. The doc models 3.7. + let ratio = attempts as f64 * 4.4 / billable_kib as f64; + out["egress_ratio_estimate"] = json!(round3(ratio)); + out["modelled_egress_ratio"] = json!(3.7); + } + out +} + +fn pct(sorted: &[u64], p: usize) -> u64 { + if sorted.is_empty() { + return 0; + } + let i = (sorted.len() * p / 100).min(sorted.len() - 1); + sorted[i] +} + +fn plur_to_usd(plur: u128) -> f64 { + round6(plur as f64 / PLUR_PER_BZZ * USD_PER_BZZ) +} + +fn round3(v: f64) -> f64 { + (v * 1000.0).round() / 1000.0 +} + +fn round6(v: f64) -> f64 { + (v * 1e6).round() / 1e6 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(owner: u8, batch: u8) -> AccountBatch { + AccountBatch { + owner: [owner; 20], + batch: [batch; 32], + } + } + + /// One full frame: 147 B header + 4104 B wire (`src/pushframe.rs`). + const FRAME: u64 = 4251; + + #[test] + fn bytes_round_to_kib_once_per_post_not_once_per_frame() { + // Per-frame rounding would charge 5 KiB for a 4.15 KiB frame — a + // 20 % over-count that would have made every figure in the report + // wrong in the relay's favour. + let mut m = Meter::default(); + let mut t = PostTally::default(); + for _ in 0..100 { + t.admit(key(1, 1), 0, FRAME); + } + m.merge(t); + let want = (FRAME * 100).div_ceil(1024); + assert_eq!(want, 416, "100 frames is 415.14 KiB, ceil 416"); + let s = m.summary(2126, 0); + assert_eq!(s["kib_admitted"], want); + } + + #[test] + fn dedup_is_recorded_but_not_double_counted_as_volume() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + // Ten frames admitted, three of them cache hits: dedup is a subset + // of admitted, never an addition to it (§8.2). + for _ in 0..10 { + t.admit(key(1, 1), 0, FRAME); + } + for _ in 0..3 { + t.dedup(key(1, 1), 0, FRAME); + } + m.merge(t); + let s = m.summary(2126, 0); + assert_eq!(s["kib_admitted"], (FRAME * 10).div_ceil(1024)); + assert_eq!(s["kib_dedup"], (FRAME * 3).div_ceil(1024)); + let billable = + s["kib_admitted"].as_u64().expect("u64") - s["kib_dedup"].as_u64().expect("u64"); + assert_eq!( + s["owed_plur"].as_str().expect("string"), + (billable as u128 * PRICE_PLUR_PER_KIB).to_string() + ); + } + + #[test] + fn credit_line_follows_batch_value_and_caps_at_the_ceiling() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + // 0.01 BZZ = 1e14 PLUR → 1e11 credit → ~208 KiB (§10.3's worked + // example). Must not saturate the global ceiling. + t.admit(key(1, 1), 100_000_000_000_000, FRAME); + // 100 BZZ is far past the ceiling, so the line clamps. + t.admit(key(2, 2), 1_000_000_000_000_000_000, FRAME); + m.merge(t); + let d = m.detail(2126, 0, 10); + let by_batch: HashMap = d["batches"] + .as_array() + .expect("array") + .iter() + .map(|b| (b["account"].as_str().expect("acct").to_string(), b.clone())) + .collect(); + let dust = &by_batch[&format!("0x{}", hex::encode([1u8; 20]))]; + assert_eq!(dust["credit_kib"], 208, "0.01 BZZ batch earns ~208 KiB"); + assert_eq!(dust["fits_full_post"], false, "dust must split its POSTs"); + let rich = &by_batch[&format!("0x{}", hex::encode([2u8; 20]))]; + assert_eq!( + rich["credit_plur"].as_str().expect("string"), + MAX_OUTSTANDING_PLUR.to_string(), + "a rich batch clamps to the global ceiling" + ); + assert_eq!(rich["fits_full_post"], true); + } + + #[test] + fn eviction_is_bounded_and_totals_survive_it() { + let mut m = Meter::new(4); + for i in 0..64u8 { + let mut t = PostTally::default(); + t.admit(key(i, i), 0, 1024); + m.merge(t); + } + assert_eq!(m.rows.len(), 4, "rows stay at cap"); + assert_eq!(m.order.len(), 4, "order stays at cap"); + // The whole point of carrying evicted totals: the headline volume + // must not silently shrink as detail rows are dropped. + let s = m.summary(2126, 0); + assert_eq!(s["kib_admitted"], 64, "all 64 KiB still counted"); + assert_eq!(s["evicted_batches"], 60); + } + + #[test] + fn a_later_reading_updates_the_batch_value_but_zero_never_clobbers_it() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + t.admit(key(1, 1), 0, FRAME); // resolved without a value + t.admit(key(1, 1), 100_000_000_000_000, FRAME); // then with one + m.merge(t); + let mut t2 = PostTally::default(); + t2.admit(key(1, 1), 0, FRAME); // a valueless frame must not erase it + m.merge(t2); + let d = m.detail(2126, 0, 10); + assert_eq!( + d["batches"][0]["remaining_value_plur"] + .as_str() + .expect("string"), + "100000000000000" + ); + } + + #[test] + fn egress_multiplier_is_attempts_over_frames() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + for _ in 0..100 { + t.admit(key(1, 1), 0, FRAME); + } + m.merge(t); + // 345 stream attempts for 100 chunks is the doc's modelled 3.45. + let s = m.summary(2126, 345); + assert_eq!(s["egress"]["attempts_per_frame"], 3.45); + // …which lands within a whisker of §9.1's modelled 3.7 GiB of real + // egress per GiB of payload. If a live relay reports something far + // from this, §9.2's price is wrong. + let ratio = s["egress"]["egress_ratio_estimate"].as_f64().expect("f64"); + assert!( + (3.6..3.8).contains(&ratio), + "modelled attempts should reproduce the modelled ratio, got {ratio}" + ); + } + + #[test] + fn cashout_and_settlement_thresholds_classify_accounts() { + let mut m = Meter::default(); + // 5 GiB is the cashout threshold (§9.3); 32 MiB is one settlement. + let five_gib_kib = 5 * 1024 * 1024; + let mut t = PostTally::default(); + t.admit(key(9, 9), 0, five_gib_kib * 1024); + t.admit(key(3, 3), 0, 40 * 1024 * 1024); // ~40 MiB, one settlement + t.admit(key(4, 4), 0, 1024 * 1024); // 1 MiB, neither + m.merge(t); + let s = m.summary(2126, 0); + assert_eq!(s["accounts_reaching_cashout"], 1); + assert_eq!(s["accounts_reaching_settlement"], 2); + } +} + +/// ~8 MiB — the dust floor. Exists only to bound RPC cost per unit of +/// value (§11.6 lists up to 4 `eth_call`s per cheque), *not* to cover +/// cashout gas, which cumulative cheques amortize separately (§8.3). +pub const MIN_CHEQUE_PLUR: u128 = 3_900_000_000_000; + +/// Metered-mode parameters, quoted in `/v1/status` and enforced at +/// admission. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Params { + pub price_plur_per_kib: u128, + pub min_cheque_plur: u128, + pub settle_every_plur: u128, + pub max_outstanding_plur: u128, + pub credit_ratio: u128, +} + +impl Default for Params { + fn default() -> Self { + Self { + price_plur_per_kib: PRICE_PLUR_PER_KIB, + min_cheque_plur: MIN_CHEQUE_PLUR, + settle_every_plur: SETTLE_EVERY_PLUR, + max_outstanding_plur: MAX_OUTSTANDING_PLUR, + credit_ratio: CREDIT_RATIO, + } + } +} + +impl Params { + /// §10.1's invariant, checked at startup so a misconfigured relay + /// refuses to boot rather than bricking every account it serves. + /// + /// Violating it is not a tuning mistake, it is a deadlock: an account + /// accrues, crosses `settle_every`, signs a cheque, has it **rejected as + /// dust**, keeps accruing, hits its cap, and the only cheque that would + /// clear the 402 is larger than what it owes — which the no-prepayment + /// rule forbids. There is no exit. + pub fn validate(&self) -> Result<(), String> { + if self.price_plur_per_kib == 0 { + return Err("price_plur_per_kib must be non-zero".into()); + } + if self.credit_ratio == 0 { + return Err("credit_ratio must be non-zero".into()); + } + if self.min_cheque_plur > self.settle_every_plur { + return Err(format!( + "min_cheque_plur ({}) exceeds settle_every_plur ({}): every account \ + would accrue past settlement and have its cheque refused as dust, \ + with no cheque able to clear the resulting 402 (§10.1)", + self.min_cheque_plur, self.settle_every_plur + )); + } + if self.settle_every_plur >= self.max_outstanding_plur { + return Err(format!( + "settle_every_plur ({}) must be below max_outstanding_plur ({}): \ + otherwise an account hits its cap before it is ever asked to pay (§10.1)", + self.settle_every_plur, self.max_outstanding_plur + )); + } + Ok(()) + } + + /// §10.3: scale the credit line to the batch's on-chain value rather + /// than asserting a constant, so the Sybil margin is `credit_ratio` by + /// construction and independent of batch size. + pub fn credit_line(&self, remaining_value_plur: u128) -> u128 { + (remaining_value_plur / self.credit_ratio).min(self.max_outstanding_plur) + } + + /// What a body of `bytes` costs. Rounds up to KiB **once**, per §8 — + /// rounding per frame would over-count a 4 251-byte frame by 20 %. + pub fn price_bytes(&self, bytes: u64) -> u128 { + u128::from(bytes.div_ceil(1024)) * self.price_plur_per_kib + } + + /// The settlement thresholds that actually apply to an account whose + /// credit line is `cap`. + /// + /// `validate` checks §10.1's invariant against `max_outstanding_plur`, + /// but that is only the *ceiling* on a credit line — the line that + /// binds is per batch, `min(remaining_value / credit_ratio, ceiling)` + /// (§10.3). For any batch smaller than + /// `min_cheque_plur * credit_ratio` the configured floor sits *above* + /// everything that account can ever owe, and the invariant quietly + /// stops holding: it accrues to its cap, is refused, and cannot write + /// a cheque large enough to be accepted. Service stops permanently, + /// for a batch that is paid up and behaving. + /// + /// Observed live with the shipped defaults: a credit line of + /// 679,783,122,862 against a 3,900,000,000,000 floor — 5.7× short — + /// which is §10.3's small batch, the case the scaled line exists to + /// keep serving. + /// + /// Both sides derive this from `(params, cap)` and `cap` is already in + /// the challenge, so the two agree without exchanging anything new. + /// Settling at half the line leaves the other half as the working + /// headroom a POST is dispatched into. + pub fn effective(&self, cap: u128) -> EffectiveParams { + let settle_every = self.settle_every_plur.min(cap / 2); + EffectiveParams { + // Preserves `min_cheque <= settle_every` — the half of §10.1 + // that makes a 402 clearable — at any credit line. + min_cheque_plur: self.min_cheque_plur.min(settle_every), + settle_every_plur: settle_every, + } + } +} + +/// §10.1's thresholds resolved against a particular credit line. See +/// [`Params::effective`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EffectiveParams { + pub min_cheque_plur: u128, + pub settle_every_plur: u128, +} + +#[cfg(test)] +mod param_tests { + use super::*; + + /// §10.1's invariant is checked against `max_outstanding_plur`, but the + /// line that binds is per batch. A small batch gets a line far under + /// the configured floor, and then no cheque it can write is acceptable + /// — it accrues to its cap and is refused for good. Seen live at a + /// 679,783,122,862 line against a 3,900,000,000,000 floor. + #[test] + fn a_credit_line_below_the_dust_floor_can_still_settle() { + let p = Params::default(); + let line = 679_783_122_862u128; + assert!( + line < p.min_cheque_plur, + "this is the case: the whole line is under the configured floor" + ); + + let e = p.effective(line); + assert!( + e.min_cheque_plur <= e.settle_every_plur, + "a 402 must be clearable by the cheque the client is told to write" + ); + assert!( + e.settle_every_plur < line, + "and settlement must trigger before the line is full, or the \ + account is refused before it is ever asked to pay" + ); + assert!(e.min_cheque_plur > 0, "some cheque must be acceptable"); + } + + /// A line with room to spare must keep the configured thresholds — + /// scaling down is for the batches that need it, not a general discount. + #[test] + fn a_generous_credit_line_keeps_the_configured_thresholds() { + let p = Params::default(); + let e = p.effective(p.max_outstanding_plur); + assert_eq!(e.min_cheque_plur, p.min_cheque_plur); + assert_eq!(e.settle_every_plur, p.settle_every_plur); + } + + #[test] + fn the_shipped_defaults_satisfy_the_invariant() { + Params::default() + .validate() + .expect("defaults must be valid"); + } + + /// The exact misconfiguration an early draft published: a dust floor 87× + /// larger than the settlement window. + #[test] + fn a_dust_floor_above_the_settlement_window_is_refused_at_startup() { + let p = Params { + min_cheque_plur: SETTLE_EVERY_PLUR * 87, + ..Params::default() + }; + let e = p.validate().expect_err("must refuse to boot"); + assert!(e.contains("no exit") || e.contains("dust"), "got: {e}"); + } + + #[test] + fn a_cap_at_or_below_the_settlement_window_is_refused() { + let p = Params { + max_outstanding_plur: SETTLE_EVERY_PLUR, + ..Params::default() + }; + p.validate() + .expect_err("cap must exceed the settlement window"); + } + + #[test] + fn zero_price_or_ratio_is_refused() { + Params { + price_plur_per_kib: 0, + ..Params::default() + } + .validate() + .expect_err("zero price"); + Params { + credit_ratio: 0, + ..Params::default() + } + .validate() + .expect_err("zero ratio would divide by zero"); + } + + /// §10.3's worked example, and the §7.2 consequence: a dust batch gets a + /// usable line that is nonetheless too small for one full POST. + #[test] + fn a_dust_batch_gets_a_small_but_usable_line() { + let p = Params::default(); + let line = p.credit_line(100_000_000_000_000); // 0.01 BZZ + assert_eq!(line / p.price_plur_per_kib, 208, "~208 KiB of credit"); + assert!(line < p.max_outstanding_plur, "must not reach the ceiling"); + } + + #[test] + fn a_rich_batch_clamps_to_the_global_ceiling() { + let p = Params::default(); + assert_eq!( + p.credit_line(u128::MAX / 2), + p.max_outstanding_plur, + "the ceiling binds however rich the batch" + ); + } + + #[test] + fn a_body_is_priced_by_rounding_up_once() { + let p = Params::default(); + assert_eq!( + p.price_bytes(1), + p.price_plur_per_kib, + "a partial KiB is a KiB" + ); + assert_eq!(p.price_bytes(1024), p.price_plur_per_kib); + assert_eq!(p.price_bytes(1025), 2 * p.price_plur_per_kib); + // One full frame, priced once rather than per-frame. + assert_eq!(p.price_bytes(4251), 5 * p.price_plur_per_kib); + } +} diff --git a/src/metered.rs b/src/metered.rs new file mode 100644 index 0000000..2560309 --- /dev/null +++ b/src/metered.rs @@ -0,0 +1,776 @@ +//! Metered relay mode — `docs/pusher-incentives.md` Stage 1. +//! +//! Holds the state the metered endpoints share and implements their logic, +//! so `src/pusher.rs` stays a router. Everything here defends the **relay** +//! against the **client** (§2); nothing here tries to prove to a client that +//! the relay did its work. +//! +//! Stage 1 runs in **soft mode**: the relay meters, reports, and accepts +//! cheques, but never refuses a push. Hard mode (402 enforcement) is +//! Stage 2 and flips one flag — the arithmetic that decides "over cap" is +//! already computed here so the two modes cannot disagree about it. + +use crate::challenge::{ + ChallengeError, ChallengeFields, IssuedChallenge, MAX_CHALLENGE_HEADER, PresentedChallenge, +}; +use crate::inbound_limit::InboundLimiter; +use crate::ledger::{Ledger, LedgerError}; +use crate::meter::Params; +use alloy_primitives::Address; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// Accounts allowed to hold a live reservation at once (§7.2). The map is +/// attacker-influenced — one entry per batch in standing — so it is capped +/// and sheds beyond. +const MAX_LIVE_RESERVATIONS: usize = 4096; + +/// Chequebook-deployment answers cached per address. `true` never changes, +/// and `false` is exactly what an attacker replays, so both are cached +/// (§11.6). +const DEPLOYED_CACHE_CAP: usize = 4096; +const DEPLOYED_OK_TTL: Duration = Duration::from_secs(86_400); +const DEPLOYED_BAD_TTL: Duration = Duration::from_secs(600); + +/// How long a chequebook's balance reads are reused. +/// +/// **This does not weaken anything, because there was no guarantee to +/// weaken.** §11.2 is explicit that the funding check is true *at +/// acceptance time, not at cashout time*: the issuer can `withdraw()` the +/// instant after we accept, and bee has the identical exposure. So a fresh +/// read per cheque only narrows the window in which an attacker must act +/// from "any time after acceptance" to "any time after acceptance, or up to +/// `STATE_TTL` before it" — against an exposure already bounded by +/// `max_outstanding` either way. +/// +/// What it buys is real: four sequential `eth_call`s per `/v1/pay` became +/// one batched request, and with this most cheques cost none at all. At +/// §10.1's 32 MiB settlement window that is ~2-3 cheques per 71 MB upload, +/// so the chain reads stop being on the critical path entirely. +const STATE_TTL: Duration = Duration::from_secs(30); +const STATE_CACHE_CAP: usize = 2048; + +#[derive(Debug, Clone)] +pub struct MeterConfig { + /// Configured hostnames. **Never derived from a request header** — see + /// `challenge::verify`. + pub origins: Vec, + pub beneficiary: [u8; 20], + pub chain_id: u64, + pub factory: Address, + pub params: Params, + /// Stage 2. False = soft mode: record the overshoot, serve anyway. + pub hard_mode: bool, +} + +pub struct Metered { + pub cfg: MeterConfig, + pub ledger: Mutex, + /// Per-IP: `/v1/challenge` runs before any account exists. + challenge_limit: Mutex, + /// Per-account: `/v1/pay` and `/v1/push`. + account_limit: Mutex, + deployed: Mutex, + /// `chequebook -> (state, read_at)`. `issuer` inside it is immutable, + /// so a hit is always authoritative for that field. + cb_state: Mutex, +} + +impl Metered { + pub fn new(cfg: MeterConfig, ledger: Ledger) -> Self { + Self { + cfg, + ledger: Mutex::new(ledger), + // A client needs one challenge per POST batch, so a handful per + // second sustained is generous; the burst covers a pipelined + // upload opening several lanes at once. + challenge_limit: Mutex::new(InboundLimiter::new(5.0, 40.0, 8192)), + account_limit: Mutex::new(InboundLimiter::new(20.0, 120.0, 8192)), + deployed: Mutex::new(DeployedCache::new(DEPLOYED_CACHE_CAP)), + cb_state: Mutex::new(StateCache::new(STATE_CACHE_CAP)), + } + } + + /// Chequebook state, from cache when it is fresh enough. + pub async fn chequebook_state( + &self, + rpc_url: &str, + chequebook: [u8; 20], + ) -> Result { + if let Some(hit) = self + .cb_state + .lock() + .expect("state cache poisoned") + .get(&chequebook) + { + return Ok(hit); + } + let st = crate::batch::read_chequebook_state( + rpc_url, + Address::from(chequebook), + Address::from(self.cfg.beneficiary), + ) + .await + .map_err(|e| format!("chequebook: {e}"))?; + self.cb_state + .lock() + .expect("state cache poisoned") + .insert(chequebook, st); + Ok(st) + } + + /// Drop a cached read after we act on it, so the next cheque from this + /// chequebook sees the balance it actually left behind rather than the + /// one from before we credited. + pub fn invalidate_chequebook(&self, chequebook: &[u8; 20]) { + self.cb_state + .lock() + .expect("state cache poisoned") + .remove(chequebook); + } + + pub fn allow_challenge(&self, ip: &str) -> bool { + self.challenge_limit + .lock() + .expect("challenge limiter poisoned") + .allow(ip.as_bytes()) + } + + pub fn allow_account(&self, account: &[u8; 20]) -> bool { + self.account_limit + .lock() + .expect("account limiter poisoned") + .allow(account) + } + + /// Issue a capability for a batch in good standing. + /// + /// Standing is resolved by the caller (it owns the RPC path and its + /// cache); this turns it into a credit line and a MAC. The result is + /// what makes `/v1/push` admission chain-free. + pub fn issue( + &self, + account: [u8; 20], + batch: [u8; 32], + remaining_value_plur: u128, + origin: &str, + now: u64, + ) -> Result { + let cap = self.cfg.params.credit_line(remaining_value_plur); + let fields = ChallengeFields { + account, + batch, + origin: origin.to_string(), + expiry_unix: now + crate::challenge::CHALLENGE_TTL_SECS, + cap_plur: cap, + }; + let secret = *self.ledger.lock().expect("ledger poisoned").secret(); + let nonce = crate::challenge::nonce(&secret, &fields)?; + Ok(IssuedChallenge { fields, nonce }) + } + + /// Verify a presented challenge header end to end. + /// + /// Two independent proofs, and both are required: + /// 1. our MAC over the nonce — this relay issued this capability; + /// 2. the client's EIP-712 signature — the caller holds the account key + /// *and* signed for this origin (§11.1). + /// + /// A valid MAC alone would let anyone who observed a challenge use it, + /// and a valid signature alone would let a challenge issued by another + /// relay be presented here. + pub fn verify_header(&self, raw: &str, now: u64) -> Result { + if raw.len() > MAX_CHALLENGE_HEADER { + return Err(format!("challenge header too large: {} bytes", raw.len())); + } + let presented = PresentedChallenge::decode(raw)?; + let secret = *self.ledger.lock().expect("ledger poisoned").secret(); + crate::challenge::verify( + &secret, + &presented.fields, + &presented.nonce, + now, + &self.cfg.origins, + ) + .map_err(|e| e.to_string())?; + + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(presented.nonce), + origin: presented.fields.origin.clone(), + account: Address::from(presented.fields.account), + batchId: alloy_primitives::B256::from(presented.fields.batch), + expiry: alloy_primitives::U256::from(presented.fields.expiry_unix), + }; + let signer = crate::signer::recover_push_challenge(&sol, self.cfg.chain_id, &presented.sig) + .map_err(|e| format!("challenge signature: {e}"))?; + if signer != presented.fields.account { + return Err(format!( + "challenge signed by 0x{} but claims account 0x{}", + hex::encode(signer), + hex::encode(presented.fields.account) + )); + } + Ok(VerifiedChallenge { + account: presented.fields.account, + batch: presented.fields.batch, + cap_plur: presented.fields.cap_plur, + }) + } + + /// Is this address a chequebook our canonical factory deployed? + /// Cached both ways — an uncached miss is a one-RPC-per-request + /// amplifier on an endpoint an attacker reaches for free (§11.6). + pub async fn is_deployed(&self, rpc_url: &str, chequebook: [u8; 20]) -> Result { + if let Some(hit) = self + .deployed + .lock() + .expect("deployed cache poisoned") + .get(&chequebook) + { + return Ok(hit); + } + let ok = crate::batch::is_deployed_chequebook( + rpc_url, + self.cfg.factory, + Address::from(chequebook), + ) + .await + .map_err(|e| format!("factory lookup: {e}"))?; + self.deployed + .lock() + .expect("deployed cache poisoned") + .insert(chequebook, ok); + Ok(ok) + } + + /// Reserve for a request whose body is `content_length` bytes. + /// + /// The reservation and the eventual bill are the *same* quantity + /// computed the same way (§7.2), so there is no estimate to be wrong + /// about and no flat over-reserve to lock small batches out. + pub fn reserve_for_body( + &self, + account: [u8; 20], + content_length: u64, + cap: u128, + ) -> crate::ledger::Admission { + let amount = self.cfg.params.price_bytes(content_length); + let mut l = self.ledger.lock().expect("ledger poisoned"); + l.reserve(account, amount, cap) + } + + pub fn shed_reservations(&self) -> bool { + self.ledger + .lock() + .expect("ledger poisoned") + .live_reservations() + >= MAX_LIVE_RESERVATIONS + } + + /// Apply a cheque. Every free check runs before this is called; this is + /// the ledger half only. + pub fn credit( + &self, + account: [u8; 20], + chequebook: [u8; 20], + cumulative: u128, + signature: [u8; 65], + ) -> Result { + let mut l = self.ledger.lock().expect("ledger poisoned"); + let accepted = l.credit(account, chequebook, cumulative, signature)?; + // Persist immediately: the window between accepting a cheque and + // durably recording its cumulative is exactly §11.4's replay hole. + if let Err(e) = l.persist() { + tracing::error!("ledger persist after credit failed: {e}"); + } + Ok(accepted) + } +} + +#[derive(Debug, Clone)] +pub struct VerifiedChallenge { + pub account: [u8; 20], + pub batch: [u8; 32], + pub cap_plur: u128, +} + +struct StateCache { + map: HashMap<[u8; 20], (crate::batch::ChequebookState, Instant)>, + order: std::collections::VecDeque<[u8; 20]>, + cap: usize, +} + +impl StateCache { + fn new(cap: usize) -> Self { + Self { + map: HashMap::new(), + order: std::collections::VecDeque::new(), + cap: cap.max(1), + } + } + + fn get(&self, k: &[u8; 20]) -> Option { + let (v, at) = self.map.get(k)?; + (at.elapsed() < STATE_TTL).then_some(*v) + } + + fn insert(&mut self, k: [u8; 20], v: crate::batch::ChequebookState) { + if self.map.insert(k, (v, Instant::now())).is_none() { + self.order.push_back(k); + } + while self.order.len() > self.cap { + if let Some(old) = self.order.pop_front() { + self.map.remove(&old); + } + } + } + + fn remove(&mut self, k: &[u8; 20]) { + self.map.remove(k); + } +} + +struct DeployedCache { + map: HashMap<[u8; 20], (bool, Instant)>, + order: std::collections::VecDeque<[u8; 20]>, + cap: usize, +} + +impl DeployedCache { + fn new(cap: usize) -> Self { + Self { + map: HashMap::new(), + order: std::collections::VecDeque::new(), + cap: cap.max(1), + } + } + + fn get(&self, k: &[u8; 20]) -> Option { + let (v, at) = self.map.get(k)?; + let ttl = if *v { + DEPLOYED_OK_TTL + } else { + DEPLOYED_BAD_TTL + }; + (at.elapsed() < ttl).then_some(*v) + } + + fn insert(&mut self, k: [u8; 20], v: bool) { + if self.map.insert(k, (v, Instant::now())).is_none() { + self.order.push_back(k); + } + while self.order.len() > self.cap { + if let Some(old) = self.order.pop_front() { + self.map.remove(&old); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::challenge::encode_challenge_header; + use crate::signer::SwarmSigner; + + const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + + fn signer() -> SwarmSigner { + SwarmSigner::from_hex_with_nonce(KEY, &format!("0x{}", hex::encode([0u8; 32])), 1) + .expect("key") + } + + fn metered() -> Metered { + Metered::new( + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: Address::ZERO, + params: Params::default(), + hard_mode: false, + }, + Ledger::ephemeral(), + ) + } + + /// Issue → sign → present → verify, the whole admission path. + fn round_trip(m: &Metered, now: u64) -> Result { + let s = signer(); + let account = *s.eth_address(); + let issued = m + .issue( + account, + [5u8; 32], + 6_200_000_000_000_000_000, + "relay-a.example", + now, + ) + .expect("issue"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(account), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + m.verify_header(&header, now) + } + + #[test] + fn a_challenge_we_issued_and_the_client_signed_is_admitted() { + let m = metered(); + let v = round_trip(&m, 1000).expect("must admit"); + assert_eq!(v.account, *signer().eth_address()); + assert_eq!(v.batch, [5u8; 32]); + assert_eq!( + v.cap_plur, + Params::default().max_outstanding_plur, + "a rich batch clamps to the ceiling" + ); + } + + /// A capability alone is not enough — the caller must prove it holds the + /// account key, or a harvested header would be usable by anyone. + #[test] + fn a_challenge_signed_by_the_wrong_key_is_refused() { + let m = metered(); + let victim = *signer().eth_address(); + let issued = m + .issue( + victim, + [5u8; 32], + 1_000_000_000_000_000, + "relay-a.example", + 1000, + ) + .expect("issue"); + let attacker = SwarmSigner::from_hex_with_nonce( + "0x1111111111111111111111111111111111111111111111111111111111111111", + &format!("0x{}", hex::encode([0u8; 32])), + 1, + ) + .expect("key"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(victim), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = attacker.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = m.verify_header(&header, 1000).expect_err("must refuse"); + assert!(e.contains("claims account"), "got: {e}"); + } + + /// §11.1: a signature gathered at relay A must be useless at relay B. + #[test] + fn a_challenge_for_another_relay_is_refused() { + let a = metered(); + let mut b_cfg = a.cfg.clone(); + b_cfg.origins = vec!["relay-b.example".into()]; + let b = Metered::new(b_cfg, Ledger::ephemeral()); + + let s = signer(); + let issued = a + .issue( + *s.eth_address(), + [5u8; 32], + 1_000_000_000_000_000, + "relay-a.example", + 1000, + ) + .expect("issue"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = b.verify_header(&header, 1000).expect_err("must refuse"); + assert!(e.contains("origin"), "got: {e}"); + } + + #[test] + fn an_expired_challenge_is_refused() { + let m = metered(); + let e = round_trip(&m, 1000).map(|_| ()).and(Ok(())); + assert!(e.is_ok()); + // Same issue instant, far-future presentation. + let s = signer(); + let issued = m + .issue( + *s.eth_address(), + [5u8; 32], + 1_000_000_000_000_000, + "relay-a.example", + 1000, + ) + .expect("issue"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = m + .verify_header(&header, 1000 + crate::challenge::CHALLENGE_TTL_SECS + 1) + .expect_err("must refuse"); + assert!(e.contains("expired"), "got: {e}"); + } + + /// The cap is inside the MAC, so a client cannot present a dust batch's + /// id alongside a rich batch's credit line. + #[test] + fn an_inflated_cap_is_refused() { + let m = metered(); + let s = signer(); + let mut issued = m + .issue( + *s.eth_address(), + [5u8; 32], + 100_000_000_000_000, + "relay-a.example", + 1000, + ) + .expect("issue"); + let honest_cap = issued.fields.cap_plur; + issued.fields.cap_plur = honest_cap * 1_000_000; + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = m.verify_header(&header, 1000).expect_err("must refuse"); + assert!(e.contains("not ours"), "got: {e}"); + } + + #[test] + fn malformed_headers_are_refused_not_panicked_on() { + let m = metered(); + for raw in ["", "!!!!", "eyJ9", "e30="] { + m.verify_header(raw, 1000).expect_err("must refuse"); + } + m.verify_header(&"A".repeat(MAX_CHALLENGE_HEADER + 1), 1000) + .expect_err("oversized"); + } + + #[test] + fn the_reservation_is_the_price_of_the_declared_body() { + let m = metered(); + let p = Params::default(); + let adm = m.reserve_for_body([1u8; 20], 4251, p.max_outstanding_plur); + assert_eq!(adm.reserved_plur, p.price_bytes(4251)); + assert!(!adm.over_cap); + } + + /// §7.2's small-batch case: a one-frame POST from a dust batch fits, + /// where an earlier design's flat `PUSH_BATCH_MAX × price` reserve would + /// have 402'd it. + #[test] + fn a_dust_batch_can_afford_a_small_post() { + let m = metered(); + let cap = Params::default().credit_line(100_000_000_000_000); + let adm = m.reserve_for_body([1u8; 20], 4251, cap); + assert!(!adm.over_cap, "a single frame must fit a dust batch's line"); + let m2 = metered(); + let big = m2.reserve_for_body([1u8; 20], 512 * 4251, cap); + assert!(big.over_cap, "but a full 512-frame POST does not"); + } +} + +#[cfg(test)] +mod lifecycle_tests { + //! The full soft-mode money path over the real ledger: admit → bill → + //! settle → replay. These exercise `Metered` + `Ledger` together, which + //! is where the accounting can silently drift. + use super::*; + use crate::meter::Params; + + fn m() -> Metered { + Metered::new( + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: Address::ZERO, + params: Params::default(), + hard_mode: false, + }, + Ledger::ephemeral(), + ) + } + + const ACCT: [u8; 20] = [1u8; 20]; + const CB: [u8; 20] = [9u8; 20]; + + /// One POST: reserve on the declared body, bill what was admitted, + /// release the rest. The reservation must not survive as phantom debt. + #[test] + fn a_request_reserves_then_commits_only_what_it_admitted() { + let m = m(); + let p = Params::default(); + let cap = p.max_outstanding_plur; + let adm = m.reserve_for_body(ACCT, 512 * 4251, cap); + assert_eq!(m.ledger.lock().unwrap().reserved(&ACCT), adm.reserved_plur); + + // Only 100 frames actually got admitted. + let billed = p.price_bytes(100 * 4251); + m.ledger + .lock() + .unwrap() + .commit(ACCT, adm.reserved_plur, billed); + let l = m.ledger.lock().unwrap(); + assert_eq!(l.reserved(&ACCT), 0, "the reservation is fully released"); + assert_eq!(l.owed(&ACCT), billed, "only admitted bytes are billed"); + assert!( + billed < adm.reserved_plur, + "and it is less than was reserved" + ); + } + + /// A POST that admits nothing must still hand its reservation back — + /// leaking it ratchets the account toward a 402 it can never clear. + #[test] + fn a_request_that_admits_nothing_leaks_no_credit() { + let m = m(); + let adm = m.reserve_for_body(ACCT, 512 * 4251, Params::default().max_outstanding_plur); + m.ledger.lock().unwrap().commit(ACCT, adm.reserved_plur, 0); + let l = m.ledger.lock().unwrap(); + assert_eq!(l.outstanding(&ACCT), 0, "nothing owed, nothing reserved"); + } + + /// Settle, then push again: the cumulative carries forward, so the + /// second cheque credits only the new debt. + #[test] + fn settlement_across_two_uploads_uses_one_growing_cumulative() { + let m = m(); + let p = Params::default(); + let first = p.price_bytes(40 * 1024 * 1024); + m.ledger.lock().unwrap().commit(ACCT, 0, first); + + let accepted = m.credit(ACCT, CB, first, [27u8; 65]).expect("first cheque"); + assert_eq!(accepted, first); + assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); + + let second = p.price_bytes(32 * 1024 * 1024); + m.ledger.lock().unwrap().commit(ACCT, 0, second); + // A cumulative cheque: the *total*, not the delta. + let accepted = m + .credit(ACCT, CB, first + second, [27u8; 65]) + .expect("second cheque"); + assert_eq!(accepted, second, "only the new debt is credited"); + assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); + } + + /// Soft mode reports the overshoot but still admits — that is the whole + /// point of Stage 1 shipping soft. + #[test] + fn soft_mode_reports_an_overshoot_without_refusing() { + let m = m(); + let cap = Params::default().credit_line(100_000_000_000_000); // dust + let adm = m.reserve_for_body(ACCT, 512 * 4251, cap); + assert!(adm.over_cap, "a full POST exceeds a dust batch's line"); + assert!(!m.cfg.hard_mode, "Stage 1 ships soft"); + assert!( + m.ledger.lock().unwrap().reserved(&ACCT) > 0, + "soft mode still reserves, so the measurement is real" + ); + } + + /// The same request under hard mode must leave no trace once refused. + #[test] + fn hard_mode_releases_the_reservation_it_refuses() { + let mut cfg = m().cfg.clone(); + cfg.hard_mode = true; + let m = Metered::new(cfg, Ledger::ephemeral()); + let cap = Params::default().credit_line(100_000_000_000_000); + let adm = m.reserve_for_body(ACCT, 512 * 4251, cap); + assert!(adm.over_cap); + m.ledger.lock().unwrap().release(ACCT, adm.reserved_plur); + assert_eq!( + m.ledger.lock().unwrap().outstanding(&ACCT), + 0, + "a refused request must not leave phantom debt behind" + ); + } + + /// §10.1's invariant in motion: a client 402'd at the cap can always + /// clear it with a cheque for exactly what it owes. + #[test] + fn an_account_at_its_cap_can_always_pay_its_way_out() { + let m = m(); + let p = Params::default(); + let cap = p.max_outstanding_plur; + // Accrue right up to the ceiling. + m.ledger.lock().unwrap().commit(ACCT, 0, cap); + let owed = m.ledger.lock().unwrap().owed(&ACCT); + assert!( + owed >= p.min_cheque_plur, + "what is owed must clear the dust floor, or there is no exit" + ); + m.credit(ACCT, CB, owed, [27u8; 65]) + .expect("a cheque for exactly what is owed"); + assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); + assert!( + !m.reserve_for_body(ACCT, 4251, cap).over_cap, + "and the account can push again" + ); + } +} + +#[cfg(test)] +mod soft_mode_tests { + //! §7.1's rollout property: soft mode must serve clients that predate + //! the payment protocol. Requiring a challenge unconditionally would + //! 401 the whole existing fleet the moment `--meter` was enabled, which + //! is the opposite of a staged rollout. + use super::*; + use crate::meter::Params; + + fn cfg(hard: bool) -> MeterConfig { + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: Address::ZERO, + params: Params::default(), + hard_mode: hard, + } + } + + /// An absent header is not an invalid one — it is a client that does not + /// speak the protocol yet. + #[test] + fn an_empty_header_is_distinguishable_from_a_malformed_one() { + let m = Metered::new(cfg(false), Ledger::ephemeral()); + // Malformed is always refused, in either mode: claiming a capability + // you do not hold must not become valid by corrupting a byte. + m.verify_header("not-base64!!", 1000) + .expect_err("a malformed header is always refused"); + m.verify_header("", 1000) + .expect_err("verify_header itself has no opinion about absence"); + } + + /// Hard mode is where the challenge becomes mandatory, because by then + /// there is a 402 to enforce. + #[test] + fn hard_mode_is_what_makes_the_challenge_mandatory() { + assert!(!cfg(false).hard_mode, "soft is the shipped default"); + assert!(cfg(true).hard_mode); + } +} diff --git a/src/payer.rs b/src/payer.rs new file mode 100644 index 0000000..50e4288 --- /dev/null +++ b/src/payer.rs @@ -0,0 +1,1578 @@ +//! Client-side payment for metered lanes — `docs/pusher-incentives.md` +//! Stage 1, client half. +//! +//! Four jobs, in the order a client meets them: +//! +//! 1. **Pin the lane.** Parse and *verify* the signed quote from +//! `/v1/status`, checking it against the identity in config rather than +//! trusting what the lane says about itself (§7.3). +//! 2. **Get admitted.** Fetch a challenge, sign it, and carry the header on +//! every `/v1/push` and `/v1/pay`. +//! 3. **Size the POST.** The challenge returns the credit line; a body that +//! would exceed it is split rather than sent and refused (§7.2). +//! 4. **Settle.** Track what is owed by the same arithmetic the relay uses, +//! and issue a cumulative cheque when it crosses `settle_every`. +//! +//! The client computes its bill from **bytes it sent**, not from anything +//! the relay reports. That is the property §8 is built on, and it is why +//! there is nothing here that verifies the relay's work: a disagreement is +//! arithmetic, visible immediately, and settled by not paying. + +use crate::meter::Params; + +/// A lane's signed `payment` block, after verification. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PaymentQuote { + pub beneficiary: [u8; 20], + /// Recovered from `sig`. **This is what a client pins**, not the + /// overlay — see [`PaymentQuote::verify`]. + pub node_eth_address: [u8; 20], + pub overlay_nonce: [u8; 32], + pub origin: String, + pub chain_id: u64, + pub params: Params, + /// True when the relay enforces 402. Soft-mode lanes bill but serve. + pub hard_enforcement: bool, +} + +/// What a client pins in config for a lane it is willing to pay. +/// +/// `PUSHER_URLS` is already a hardcoded list, so carrying two more fields +/// per entry costs nothing — and reading the beneficiary from `/v1/status` +/// at runtime instead would mean paying whoever answers the URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LanePin { + pub node_eth_address: [u8; 20], + pub beneficiary: [u8; 20], +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum QuoteError { + #[error("quote field {0} missing or malformed")] + Field(&'static str), + #[error("quote signature does not verify: {0}")] + Signature(String), + #[error("quote signed by 0x{got} but this lane is pinned to 0x{want}")] + WrongSigner { got: String, want: String }, + #[error("quote beneficiary 0x{got} is not the pinned 0x{want}")] + WrongBeneficiary { got: String, want: String }, + #[error("advertised overlay does not derive from the signed identity")] + OverlayMismatch, + #[error("quote parameters are unusable: {0}")] + BadParams(String), + #[error("lane price {got} exceeds the client's ceiling {ceiling}")] + TooExpensive { got: u128, ceiling: u128 }, +} + +impl PaymentQuote { + /// Parse and verify a `/v1/status` `payment` block. + /// + /// `advertised_overlay` is the lane's own `overlay` field. Checking that + /// it derives from the *signed* identity is what makes the overlay + /// trustworthy at all: an overlay is + /// `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`, so a signature alone + /// yields the eth address while the nonce is neither transmitted nor + /// derivable — which is why "pin `(url, overlay)`" was never + /// implementable and the pin is on the address. + pub fn verify( + payment: &serde_json::Value, + advertised_overlay: Option<&[u8; 32]>, + network_id: u64, + pin: Option<&LanePin>, + price_ceiling_plur_per_kib: u128, + ) -> Result { + let sig_hex = payment + .get("sig") + .and_then(|s| s.as_str()) + .ok_or(QuoteError::Field("sig"))?; + let sig = hex::decode(sig_hex.trim_start_matches("0x")) + .map_err(|e| QuoteError::Signature(e.to_string()))?; + + // The relay signs the block *without* `sig`, and `serde_json`'s map + // is a `BTreeMap`, so re-serializing after removing that one field + // reproduces the signed bytes exactly. + let mut unsigned = payment.clone(); + unsigned + .as_object_mut() + .ok_or(QuoteError::Field("payment"))? + .remove("sig"); + let payload = unsigned.to_string(); + let node_eth_address = + crate::signer::recover_eth_address_from_eip191(payload.as_bytes(), &sig) + .map_err(|e| QuoteError::Signature(e.to_string()))?; + + let addr = |k: &'static str| -> Result<[u8; 20], QuoteError> { + let s = payment + .get(k) + .and_then(|x| x.as_str()) + .ok_or(QuoteError::Field(k))?; + let raw = hex::decode(s.trim_start_matches("0x")).map_err(|_| QuoteError::Field(k))?; + <[u8; 20]>::try_from(raw.as_slice()).map_err(|_| QuoteError::Field(k)) + }; + let plur = |k: &'static str| -> Result { + payment + .get(k) + .and_then(|x| x.as_str()) + .and_then(|s| s.parse().ok()) + .ok_or(QuoteError::Field(k)) + }; + + let beneficiary = addr("beneficiary")?; + let claimed_node = addr("node_eth_address")?; + if claimed_node != node_eth_address { + return Err(QuoteError::WrongSigner { + got: hex::encode(node_eth_address), + want: hex::encode(claimed_node), + }); + } + let overlay_nonce = { + let s = payment + .get("overlay_nonce") + .and_then(|x| x.as_str()) + .ok_or(QuoteError::Field("overlay_nonce"))?; + let raw = hex::decode(s.trim_start_matches("0x")) + .map_err(|_| QuoteError::Field("overlay_nonce"))?; + <[u8; 32]>::try_from(raw.as_slice()).map_err(|_| QuoteError::Field("overlay_nonce"))? + }; + + // Pinning is the actual root of trust (§2): the lane URL over HTTPS + // plus an identity the client already knew. + if let Some(pin) = pin { + if pin.node_eth_address != node_eth_address { + return Err(QuoteError::WrongSigner { + got: hex::encode(node_eth_address), + want: hex::encode(pin.node_eth_address), + }); + } + if pin.beneficiary != beneficiary { + return Err(QuoteError::WrongBeneficiary { + got: hex::encode(beneficiary), + want: hex::encode(pin.beneficiary), + }); + } + } + + if let Some(overlay) = advertised_overlay { + let derived = + crate::signer::derive_overlay(&node_eth_address, network_id, &overlay_nonce); + if &derived != overlay { + return Err(QuoteError::OverlayMismatch); + } + } + + let params = Params { + price_plur_per_kib: plur("price_plur_per_kib")?, + min_cheque_plur: plur("min_cheque_plur")?, + settle_every_plur: plur("settle_every_plur")?, + max_outstanding_plur: plur("max_outstanding_plur")?, + credit_ratio: payment + .get("credit_ratio") + .and_then(|x| x.as_u64()) + .map(u128::from) + .ok_or(QuoteError::Field("credit_ratio"))?, + }; + // A lane whose parameters violate §10.1's invariant would brick this + // client, so refuse it here rather than discovering it at the first + // 402 with no cheque able to clear it. + params.validate().map_err(QuoteError::BadParams)?; + if params.price_plur_per_kib > price_ceiling_plur_per_kib { + return Err(QuoteError::TooExpensive { + got: params.price_plur_per_kib, + ceiling: price_ceiling_plur_per_kib, + }); + } + + Ok(Self { + beneficiary, + node_eth_address, + overlay_nonce, + origin: payment + .get("origin") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(), + chain_id: payment + .get("chain_id") + .and_then(|x| x.as_u64()) + .ok_or(QuoteError::Field("chain_id"))?, + params, + hard_enforcement: payment.get("enforcement").and_then(|x| x.as_str()) == Some("hard"), + }) + } +} + +/// A challenge the relay issued, ready to sign. +#[derive(Debug, Clone)] +pub struct OfferedChallenge { + pub nonce: [u8; 32], + pub account: [u8; 20], + pub batch: [u8; 32], + pub origin: String, + pub expiry_unix: u64, + pub cap_plur: u128, +} + +impl OfferedChallenge { + pub fn parse(v: &serde_json::Value) -> Result { + let fixed = |k: &str, n: usize| -> Result, String> { + let s = v + .get(k) + .and_then(|x| x.as_str()) + .ok_or_else(|| format!("challenge: missing {k}"))?; + let raw = hex::decode(s.trim_start_matches("0x")) + .map_err(|e| format!("challenge {k}: {e}"))?; + if raw.len() != n { + return Err(format!("challenge {k}: want {n} bytes, got {}", raw.len())); + } + Ok(raw) + }; + let mut nonce = [0u8; 32]; + nonce.copy_from_slice(&fixed("nonce", 32)?); + let mut account = [0u8; 20]; + account.copy_from_slice(&fixed("account", 20)?); + let mut batch = [0u8; 32]; + batch.copy_from_slice(&fixed("batch", 32)?); + Ok(Self { + nonce, + account, + batch, + origin: v + .get("origin") + .and_then(|x| x.as_str()) + .ok_or("challenge: missing origin")? + .to_string(), + expiry_unix: v + .get("expiry") + .and_then(|x| x.as_u64()) + .ok_or("challenge: missing expiry")?, + cap_plur: v + .get("max_outstanding_plur") + .and_then(|x| x.as_str()) + .and_then(|s| s.parse().ok()) + .ok_or("challenge: missing max_outstanding_plur")?, + }) + } + + /// Sign it and produce the header value. + /// + /// Signing binds `origin`, which is what makes this header useless at + /// any other relay even if it is observed in flight (§11.1). + pub fn sign( + &self, + signer: &crate::signer::SwarmSigner, + chain_id: u64, + ) -> Result { + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(self.nonce), + origin: self.origin.clone(), + account: alloy_primitives::Address::from(self.account), + batchId: alloy_primitives::B256::from(self.batch), + expiry: alloy_primitives::U256::from(self.expiry_unix), + }; + let sig = signer + .sign_push_challenge(&sol, chain_id) + .map_err(|e| e.to_string())?; + let issued = crate::challenge::IssuedChallenge { + fields: crate::challenge::ChallengeFields { + account: self.account, + batch: self.batch, + origin: self.origin.clone(), + expiry_unix: self.expiry_unix, + cap_plur: self.cap_plur, + }, + nonce: self.nonce, + }; + Ok(crate::challenge::encode_challenge_header(&issued, &sig)) + } + + /// Re-fetch before this, rather than racing the expiry with a POST in + /// flight. A challenge is cheap; a mid-upload 401 is not. + pub fn stale_after(&self) -> u64 { + self.expiry_unix.saturating_sub(30) + } +} + +/// Per-lane running total, tracked by the client from bytes it sent. +#[derive(Debug, Clone)] +pub struct LaneAccount { + pub params: Params, + pub beneficiary: [u8; 20], + /// Billed and not yet covered by a cheque. + owed_plur: u128, + /// Dispatched but not yet answered. Mirrors the relay's `reserved`: + /// bytes on the wire are not yet a debt, because the relay bills what + /// it *admits*, and a POST it refuses (402) or drops costs nothing. + /// Counting these as owed made the client sign cheques for several + /// times what the relay had booked. + pending_plur: u128, + /// Total already promised to this beneficiary. Cheques are cumulative, + /// so this only grows. + cumulative_plur: u128, + /// This account's credit line, once a challenge has reported one. + /// §10.1's thresholds are resolved against it — see + /// [`Params::effective`] — because the configured floor can sit above + /// everything a small batch is able to owe. + cap_plur: u128, +} + +impl LaneAccount { + pub fn new(params: Params, beneficiary: [u8; 20]) -> Self { + Self { + params, + beneficiary, + owed_plur: 0, + pending_plur: 0, + cumulative_plur: 0, + cap_plur: 0, + } + } + + /// Restore the cumulative from the on-disk store, so a second CLI run + /// does not issue a cheque the relay rejects as non-increasing. + pub fn with_cumulative(mut self, cumulative_plur: u128) -> Self { + self.cumulative_plur = cumulative_plur; + self + } + + pub fn owed(&self) -> u128 { + self.owed_plur + } + + /// Bytes dispatched but not yet answered, priced. Not debt yet — see + /// [`Self::adopt_relay_debt`] for why the distinction matters. + pub fn pending(&self) -> u128 { + self.pending_plur + } + + /// Owed plus in-flight — the client's mirror of the relay's + /// `owed + reserved`, and what the credit line actually binds on. + pub fn outstanding(&self) -> u128 { + self.owed_plur.saturating_add(self.pending_plur) + } + + pub fn cumulative(&self) -> u128 { + self.cumulative_plur + } + + /// A POST is on the wire. Held as *pending*, not owed — see + /// [`Self::pending_plur`]. + pub fn record_sent(&mut self, body_bytes: u64) { + self.pending_plur = self + .pending_plur + .saturating_add(self.params.price_bytes(body_bytes)); + } + + /// A POST came back. `reached_relay` is false only when the relay + /// refused it at admission (402) — before reading a byte, so neither + /// side bills it. Any other outcome, *including a broken stream*, means + /// the body arrived and the relay billed it, so we must too. + /// + /// This mirrors the relay's reserve→commit exactly, which is what keeps + /// the two sides' arithmetic identical without them exchanging totals. + pub fn record_answered(&mut self, body_bytes: u64, reached_relay: bool) { + let price = self.params.price_bytes(body_bytes); + self.pending_plur = self.pending_plur.saturating_sub(price); + if reached_relay { + self.owed_plur = self.owed_plur.saturating_add(price); + } + } + + /// A dedup hit costs nothing, so give it back when the ack says so + /// (§8.2). The relay's claim only ever lowers the bill, so believing it + /// is safe. + pub fn refund_dedup(&mut self, body_bytes: u64) { + self.owed_plur = self + .owed_plur + .saturating_sub(self.params.price_bytes(body_bytes)); + } + + /// The credit line the relay reported. Setting it is what lets the + /// thresholds below scale down to a small batch. + pub fn set_cap(&mut self, cap: u128) { + self.cap_plur = cap; + } + + /// §10.1's thresholds as they apply to this account. Falls back to the + /// configured values before any challenge has reported a line. + fn thresholds(&self) -> crate::meter::EffectiveParams { + if self.cap_plur == 0 { + return crate::meter::EffectiveParams { + min_cheque_plur: self.params.min_cheque_plur, + settle_every_plur: self.params.settle_every_plur, + }; + } + self.params.effective(self.cap_plur) + } + + pub fn should_settle(&self) -> bool { + self.owed_plur >= self.thresholds().settle_every_plur + } + + /// The cumulative for the next cheque, or `None` when what is owed is + /// still under the lane's dust floor and would be refused. + pub fn next_cumulative(&self) -> Option { + if self.owed_plur < self.thresholds().min_cheque_plur { + return None; + } + Some(self.cumulative_plur.saturating_add(self.owed_plur)) + } + + /// Drop debt the relay will not accept. See the caller in + /// [`LanePayer::settle`] — this is a divergence artifact, not a + /// discount, and it can only ever move in the client's favour by + /// removing an obligation the counterparty has already disclaimed. + pub fn forgive_phantom_debt(&mut self) { + self.owed_plur = 0; + } + + /// Take the relay's figure as ours outright, in whichever direction. + /// + /// Only correct after it has **rejected** a cheque: nothing was issued, + /// so there is no cumulative to be inconsistent with, and the relay has + /// just told us what it is prepared to accept. + /// + /// The divergence this repairs is §7.3's ack-tail. A POST whose + /// response stream breaks was still read, so the client bills it — but + /// if the relay's task is cancelled before it commits, its `Admitted` + /// guard releases the reservation and it never books those bytes. The + /// client cannot tell the two apart from its side, so it over-counts, + /// and every subsequent cheque is refused as an overpayment until + /// somebody yields. The relay is the party deciding what to accept, so + /// it yields to the relay. + pub fn sync_relay_debt(&mut self, relay_owed: u128) { + self.owed_plur = relay_owed; + } + + /// Adopt a larger debt figure the relay reports for us. + /// + /// The mirror of [`Self::forgive_phantom_debt`], and the half that is + /// load-bearing across *sessions*. The relay's ledger is durable and + /// ours is not: every run ends leaving the sub-dust residual unpaid + /// (see the final settle in `drive_pushers`), and the relay keeps + /// counting it against the credit line while a fresh client process + /// starts believing it owes nothing. Enough runs of that and the + /// account is over its cap with a client that cannot compute a cheque + /// to clear it — refused forever, having genuinely incurred the debt. + /// + /// `relay_owed` must be the relay's *owed*, never the `outstanding` in + /// a 402 body: that one is quoted with the just-refused reservation + /// already added, so adopting it over-pays by exactly the body that was + /// turned away and the next cheque is rejected as an overpayment. + /// Reservations are excluded for the same reason on our side — bytes + /// still in flight are already counted in `pending_plur`, and would be + /// billed twice when those POSTs land. + /// + /// Only ever raises, and only by what is not already accounted for in + /// flight. An under-count is safe and self-correcting (the next settle + /// picks up the rest); an over-count is a cheque the relay refuses. + pub fn adopt_relay_debt(&mut self, relay_owed: u128) -> bool { + // Deduct what is still on the wire. A POST the relay has finished + // reading is already in the figure it just reported, while locally + // it is still `pending` until its response closes — so adopting the + // raw number and then letting `record_answered` move those same + // bytes into `owed` counts them twice. That surfaces as a cheque + // the relay rejects for overpayment, which jams settlement for the + // rest of the run. Under-adopting is safe: the remainder is still + // owed, and the next settle collects it. + let adopt = relay_owed.saturating_sub(self.pending_plur); + if adopt <= self.owed_plur { + return false; + } + self.owed_plur = adopt; + true + } + + /// Call once a cheque for `cumulative` has been accepted. + pub fn settled(&mut self, cumulative: u128) { + let credited = cumulative.saturating_sub(self.cumulative_plur); + self.cumulative_plur = cumulative; + self.owed_plur = self.owed_plur.saturating_sub(credited); + } + + /// Largest body this lane will admit right now, in bytes. + /// + /// The client sizes its POST to fit rather than discovering the ceiling + /// as a 402 — which matters most for exactly the small batches §10.3 + /// exists to keep, whose whole credit line is under one full POST. + /// + /// Measured against `outstanding`, not `owed`: the relay reserves each + /// body at admission, so concurrent POSTs hold credit that is not yet + /// debt. Sizing against `owed` alone hands every in-flight POST the + /// whole line as if it were the only one, and with several on the wire + /// their reservations sum past the cap — the relay 402s a client that + /// believes it has headroom, and nothing is owed that paying could + /// clear. `has_headroom` and `would_exceed` already bind on + /// `outstanding`; this is the same quantity. + pub fn max_body_bytes(&self, cap_plur: u128) -> u64 { + let headroom = cap_plur.saturating_sub(self.outstanding()); + let kib = headroom / self.params.price_plur_per_kib.max(1); + (kib.saturating_mul(1024)).min(u64::MAX as u128) as u64 + } +} + +/// Aggregate exposure across every beneficiary drawn on one chequebook. +/// +/// Cumulative payouts are per `(chequebook, beneficiary)`, so N lanes are N +/// independent claims on **one** balance. Without this a cheque to the +/// second lane silently exceeds it and bounces — and §11.3's Sybil case is +/// exactly one operator presenting several beneficiaries. +#[derive(Debug, Default, Clone)] +pub struct TotalIssued { + per_beneficiary: std::collections::BTreeMap<[u8; 20], u128>, +} + +impl TotalIssued { + pub fn total(&self) -> u128 { + self.per_beneficiary.values().copied().sum() + } + + pub fn issued_to(&self, beneficiary: &[u8; 20]) -> u128 { + self.per_beneficiary.get(beneficiary).copied().unwrap_or(0) + } + + /// Would raising this beneficiary's cumulative to `cumulative` push the + /// total past the chequebook's balance? + pub fn would_exceed(&self, beneficiary: &[u8; 20], cumulative: u128, balance: u128) -> bool { + let others = self.total() - self.issued_to(beneficiary); + others.saturating_add(cumulative) > balance + } + + pub fn record(&mut self, beneficiary: [u8; 20], cumulative: u128) { + let e = self.per_beneficiary.entry(beneficiary).or_insert(0); + // Cumulatives only grow; a lower value is a stale report, not a + // refund. + if cumulative > *e { + *e = cumulative; + } + } +} + +// ────────────────────────────────────────────────────────────────────── +// The payment loop (docs/pusher-incentives.md §12) +// ────────────────────────────────────────────────────────────────────── + +/// Everything the client needs to pay a metered lane. +/// +/// The account key is the **batch owner's** signer — the same key that +/// stamps chunks (§6) — so a metered upload needs no extra credential and, +/// in a browser, no wallet prompt. +#[cfg(not(target_arch = "wasm32"))] +pub struct PaymentConfig { + pub signer: crate::signer::SwarmSigner, + pub batch: [u8; 32], + pub chequebook: [u8; 20], + pub chain_id: u64, + /// Shared across lanes: N beneficiaries are N claims on **one** balance + /// (§8.3), so the cumulative store has to be common. + pub cheques: std::sync::Arc>, + /// On-chain liquid balance of the chequebook, read once at startup. A + /// cheque that would push total issuance past this is not signed — it + /// would be accepted and then fail at cashout, which looks like the + /// relay's fault and costs the lane's trust rather than ours. + pub balance_plur: u128, +} + +/// Per-lane payment state: the verified quote, a cached capability, and the +/// running total. +#[cfg(not(target_arch = "wasm32"))] +pub struct LanePayer { + pub base_url: String, + pub quote: PaymentQuote, + pub account: LaneAccount, + header: Option, + header_stale_after: u64, + cap_plur: u128, +} + +#[cfg(not(target_arch = "wasm32"))] +impl LanePayer { + pub fn new(base_url: String, quote: PaymentQuote, cumulative: u128) -> Self { + let account = LaneAccount::new(quote.params, quote.beneficiary).with_cumulative(cumulative); + Self { + base_url, + quote, + account, + header: None, + header_stale_after: 0, + cap_plur: 0, + } + } + + /// The credit line the relay last told us about, or 0 before the first + /// challenge. Used to size POSTs (§7.2). + pub fn cap_plur(&self) -> u128 { + self.cap_plur + } + + /// The credit line normally arrives with a challenge, which needs a + /// relay; sizing is pure arithmetic and worth testing without one. + #[cfg(test)] + pub(crate) fn set_cap_for_test(&mut self, cap: u128) { + self.cap_plur = cap; + } + + /// A valid challenge header, fetching and signing one if needed. + /// + /// Re-fetched 30 s before expiry rather than on failure: racing the + /// expiry with a POST already in flight turns a cheap GET into a + /// mid-upload 401. + pub async fn header( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result<&str, String> { + let now = crate::challenge::now_unix(); + if self.header.is_none() || now >= self.header_stale_after { + let account = *cfg.signer.eth_address(); + let url = format!( + "{}/v1/challenge?account=0x{}&batch=0x{}", + self.base_url.trim_end_matches('/'), + hex::encode(account), + hex::encode(cfg.batch), + ); + let resp = http + .get(&url) + .timeout(std::time::Duration::from_secs(60)) + .send() + .await + .map_err(|e| format!("challenge fetch: {e}"))?; + if !resp.status().is_success() { + let code = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(format!("challenge {code}: {}", body.trim())); + } + let v: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("challenge json: {e}"))?; + let offered = OfferedChallenge::parse(&v)?; + self.cap_plur = offered.cap_plur; + // §10.1's thresholds scale with the line, so the account needs + // it too — see `Params::effective`. + self.account.set_cap(offered.cap_plur); + self.header_stale_after = offered.stale_after(); + self.header = Some(offered.sign(&cfg.signer, self.quote.chain_id)?); + } + Ok(self.header.as_deref().unwrap_or_default()) + } + + /// Largest POST body this lane will currently admit. + pub fn max_body_bytes(&self) -> u64 { + if self.cap_plur == 0 { + return u64::MAX; + } + self.account.max_body_bytes(self.cap_plur) + } + + /// Frames per POST this lane can ever afford, ignoring current debt. + /// + /// This is what the scheduler needs: a body larger than the *whole* + /// credit line can never be admitted no matter how promptly we settle, + /// so it must never be built. Computed against a full frame and then + /// walked down until it genuinely fits, because the relay bills a + /// KiB-rounded body and an off-by-one here is an unfixable 402 loop. + pub fn max_frames(&self) -> usize { + // A zero cap is "no credit line known", i.e. an open lane — not a + // lane that can afford nothing. + if self.cap_plur == 0 { + return usize::MAX; + } + self.frames_within(self.cap_plur) + } + + /// Frames per POST affordable *right now*, with current debt and + /// in-flight bytes deducted. + /// + /// This is what each dispatch must be sized by. Gating instead on + /// whether a *full* POST would fit stalls the lane outright whenever + /// the leftover debt cannot be settled: a residual under + /// `min_cheque_plur` is unpayable by construction (§10.2), so if a + /// full-size POST needs the whole line, no cheque can ever restore the + /// headroom the guard is waiting for and the upload fails with chunks + /// still pending. Observed exactly that way against a batch whose + /// credit line had decayed to roughly one POST. + /// + /// Returns 0 when not even one frame fits, which is the caller's cue to + /// settle or wait rather than to dispatch. + pub fn affordable_frames(&self) -> usize { + if self.cap_plur == 0 { + return usize::MAX; + } + let headroom = self.cap_plur.saturating_sub(self.account.outstanding()); + let frame = crate::pushframe::MAX_FRAME_LEN as u64; + if self.quote.params.price_bytes(frame) > headroom { + return 0; + } + self.frames_within(headroom) + } + + /// Largest frame count whose KiB-rounded body prices at or under + /// `budget`. Estimated, then walked down until it genuinely fits: the + /// relay bills a rounded body, and an off-by-one here is an unfixable + /// 402 loop. + fn frames_within(&self, budget: u128) -> usize { + if budget == 0 { + return 0; + } + let frame = crate::pushframe::MAX_FRAME_LEN as u128; + let mut n = (budget / self.quote.params.price_plur_per_kib) + .saturating_mul(1024) + .checked_div(frame) + .unwrap_or(0) + .min(usize::MAX as u128) as usize; + while n > 1 && self.quote.params.price_bytes(n as u64 * frame as u64) > budget { + n -= 1; + } + n.max(1) + } + + /// Is there room for a POST of any useful size right now? + /// + /// Checked *before* asking the scheduler for work: taking an assignment + /// and handing it back costs the chunks a retry attempt each time, so a + /// tight credit line would exhaust their budget and fail the upload + /// rather than merely pausing it. + /// One frame is the smallest thing worth dispatching, so that is what + /// this asks about. It is *not* a licence to then send a full batch: + /// the body is sized separately by [`Self::affordable_frames`], and the + /// two must be read together. Asking here about a full POST instead + /// looks safer and is not — see `affordable_frames` for the stall it + /// causes when the leftover debt is under the dust floor. + pub fn has_headroom(&self) -> bool { + if self.cap_plur == 0 { + return true; + } + let one_frame = self + .quote + .params + .price_bytes(crate::pushframe::MAX_FRAME_LEN as u64); + self.account.outstanding().saturating_add(one_frame) <= self.cap_plur + } + + /// Would dispatching `body_bytes` right now exceed the credit line? + /// The caller settles first if so, which is what keeps an upload from + /// ever reaching its cap rather than recovering from it. + pub fn would_exceed(&self, body_bytes: u64) -> bool { + self.cap_plur > 0 + && self + .account + .outstanding() + .saturating_add(self.quote.params.price_bytes(body_bytes)) + > self.cap_plur + } + + /// What the relay's ledger says this account owes, from `/v1/account`. + /// + /// The relay is authoritative here — it is the party deciding what to + /// accept — so this is the number both the upward reconcile and a + /// rejected cheque correct themselves against. + async fn relay_owed( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result { + let header = self.header(http, cfg).await?.to_string(); + let resp = http + .get(format!( + "{}/v1/account", + self.base_url.trim_end_matches('/') + )) + .header(crate::challenge::CHALLENGE_HEADER, header) + .timeout(std::time::Duration::from_secs(60)) + .send() + .await + .map_err(|e| format!("account fetch: {e}"))?; + if !resp.status().is_success() { + return Err(format!("account: http {}", resp.status())); + } + let v: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("account decode: {e}"))?; + v.get("owed_plur") + .and_then(|x| x.as_str()) + .ok_or("account: missing owed_plur")? + .parse() + .map_err(|e| format!("account: bad owed_plur: {e}")) + } + + /// Ask the relay what it thinks we owe, and adopt the figure if it is + /// larger than ours. + /// + /// Called only when a 402 arrives that our own books say we cannot pay + /// — the deadlock in [`LaneAccount::adopt_relay_debt`]. `/v1/account` + /// is used rather than the number in the 402 body because the body + /// quotes `owed + reserved` *including the refused request*, which is + /// not a payable amount. + /// + /// Returns whether the debt moved. + pub async fn reconcile( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result { + let owed = self.relay_owed(http, cfg).await?; + self.check_reported_debt(owed, cfg.balance_plur)?; + Ok(self.account.adopt_relay_debt(owed)) + } + + /// Is a debt figure the relay reports one it could legitimately hold, + /// and one we could pay? + /// + /// Split out of [`Self::reconcile`] because it is the only part of the + /// reconcile that *decides* anything, and it should be testable without + /// standing up a server. + /// + /// Two ceilings, and the tighter one binds. + /// + /// The first is the quote's global `max_outstanding_plur`. The relay + /// refuses admission whenever `outstanding + reserve > cap`, and §10.3 + /// makes every per-batch cap `min(value / ratio, ceiling)` — never above + /// the ceiling. So no debt can *legitimately* stand above it, whatever + /// our own books say. + /// + /// Deliberately not the per-batch credit line, which §17.1 rejected for + /// a reason that still holds: the line shrinks as the batch is spent + /// down, so it can fall below debt properly incurred when the batch was + /// worth more, and refusing on that basis preserves the very deadlock + /// this reconcile exists to break. The ceiling is a constant of the + /// signed quote and does not move. + /// + /// Why bound at all, when the relay is the one keeping the ledger: a + /// relay is not a curated identity. It is an HTTP service the client + /// chose and pinned (§7.3) — anyone can run one, and there is no + /// registry to be admitted to. Pinning buys the right to trust its + /// arithmetic *within the credit it granted*. Trusting it past that puts + /// the entire chequebook behind one JSON field. + fn check_reported_debt(&self, owed: u128, balance_plur: u128) -> Result<(), String> { + let ceiling = self.quote.params.max_outstanding_plur; + if owed > ceiling { + return Err(format!( + "relay claims {owed} owed, above the {ceiling} ceiling it signed \ + — it cannot have admitted that much" + )); + } + // Then what we can actually pay. `settle` enforces this exactly + // across all lanes (§8.3); stating it here turns a confusing failure + // at signing time into a legible one at reconcile time. + if owed > balance_plur { + return Err(format!( + "relay claims {owed} owed, more than the chequebook's {balance_plur} balance" + )); + } + Ok(()) + } + + /// Settle if there is enough owed to be worth a cheque. + /// + /// Returns the amount accepted, or `None` when nothing was owed above + /// the lane's dust floor. Errors are the caller's cue to stop using the + /// lane, not to retry blindly — a rejected cheque usually means the two + /// sides disagree about the cumulative, which retrying cannot fix. + pub async fn settle( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result, String> { + let Some(cumulative) = self.account.next_cumulative() else { + return Ok(None); + }; + self.pay(http, cfg, cumulative, true).await + } + + /// Sign and present one cheque. + /// + /// `correct_once` allows a single re-present against the relay's own + /// figure when it rejects ours as an overpayment; the retry passes + /// `false` so a relay that keeps refusing cannot loop us. + async fn pay( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + cumulative: u128, + correct_once: bool, + ) -> Result, String> { + // Aggregate exposure across every beneficiary drawn on this one + // chequebook (§8.3): the second lane's cheque is what silently + // bounces without this. + let key = crate::cheques::relay_key(&self.quote.beneficiary); + { + let store = cfg.cheques.lock().expect("cheque store poisoned"); + if store.would_exceed_balance(&key, cumulative, cfg.balance_plur) { + return Err(format!( + "cheque for {cumulative} would push total issuance past the chequebook's \ + {} balance across all lanes", + cfg.balance_plur + )); + } + } + let sig = cfg + .signer + .sign_cheque( + &cfg.chequebook, + &self.quote.beneficiary, + alloy_primitives::U256::from(cumulative), + self.quote.chain_id, + ) + .map_err(|e| format!("sign cheque: {e}"))?; + let body = crate::protocols::swap::encode_signed_cheque_json_pub( + &cfg.chequebook, + &self.quote.beneficiary, + alloy_primitives::U256::from(cumulative), + &sig, + ); + let header = self.header(http, cfg).await?.to_string(); + let resp = http + .post(format!("{}/v1/pay", self.base_url.trim_end_matches('/'))) + .header(crate::challenge::CHALLENGE_HEADER, header) + .header("content-type", "application/json") + .body(body) + .timeout(std::time::Duration::from_secs(120)) + .send() + .await + .map_err(|e| format!("pay: {e}"))?; + if !resp.status().is_success() { + let code = resp.status(); + let text = resp.text().await.unwrap_or_default(); + // The relay's ledger is authoritative for what it will accept. + // If it says nothing is owed, our extra is an artifact — bytes + // we charged ourselves for a POST whose completion we never + // saw, and which the relay therefore never billed. Carrying it + // forever would only eat our own headroom, since a cheque for + // it is refused every time. + if text.contains("nothing owed") { + self.account.forgive_phantom_debt(); + return Ok(None); + } + // Same divergence, partial rather than total: the relay booked + // *less* than we billed ourselves, so our cumulative overshoots + // and it refuses. Nothing was issued, so we can simply take its + // figure and re-present. Without this the overshoot is + // permanent — every later cheque carries it and is refused for + // the same reason, and the lane never settles again. + if correct_once && text.contains("is owed") { + let relay = self.relay_owed(http, cfg).await?; + self.account.sync_relay_debt(relay); + let Some(corrected) = self.account.next_cumulative() else { + return Ok(None); + }; + return Box::pin(self.pay(http, cfg, corrected, false)).await; + } + return Err(format!("pay {code}: {}", text.trim())); + } + // Record the cumulative *before* trusting the reply: we have + // certainly issued it, and under-recording is what causes the next + // cheque to be rejected as non-increasing. + { + let mut store = cfg.cheques.lock().expect("cheque store poisoned"); + store + .set_cumulative(&key, cumulative) + .map_err(|e| format!("cheque store: {e}"))?; + let _ = store.save(); + } + self.account.settled(cumulative); + Ok(Some(cumulative)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::signer::SwarmSigner; + + const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + const NONCE: [u8; 32] = [0u8; 32]; + + fn node() -> SwarmSigner { + SwarmSigner::from_hex_with_nonce(KEY, &format!("0x{}", hex::encode(NONCE)), 1).expect("key") + } + + /// Build a quote exactly the way the relay does, so the test exercises + /// the real signed bytes rather than a hand-rolled approximation. + fn quote_json(beneficiary: [u8; 20]) -> serde_json::Value { + let n = node(); + let p = Params::default(); + let mut body = serde_json::json!({ + "mode": "metered", + "enforcement": "soft", + "beneficiary": format!("0x{}", hex::encode(beneficiary)), + "node_eth_address": format!("0x{}", hex::encode(n.eth_address())), + "overlay_nonce": format!("0x{}", hex::encode(NONCE)), + "origin": "relay-a.example", + "chain_id": 100, + "factory": format!("0x{}", hex::encode([0u8; 20])), + "price_plur_per_kib": p.price_plur_per_kib.to_string(), + "min_cheque_plur": p.min_cheque_plur.to_string(), + "settle_every_plur": p.settle_every_plur.to_string(), + "max_outstanding_plur": p.max_outstanding_plur.to_string(), + "credit_ratio": p.credit_ratio as u64, + "challenge_ttl_secs": 300, + }); + let sig = n.sign_eip191(body.to_string().as_bytes()).expect("sign"); + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + body + } + + fn ceiling() -> u128 { + Params::default().price_plur_per_kib * 4 + } + + #[test] + fn a_signed_quote_verifies_and_yields_the_signing_identity() { + let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()) + .expect("must verify"); + assert_eq!(q.node_eth_address, *node().eth_address()); + assert_eq!(q.beneficiary, [3u8; 20]); + assert_eq!(q.params, Params::default()); + assert!(!q.hard_enforcement); + } + + /// The reason the pin is on the address and the nonce is published: + /// the client can now check the lane's overlay claim instead of taking + /// it on faith. + #[test] + fn the_advertised_overlay_must_derive_from_the_signed_identity() { + let good = crate::signer::derive_overlay(node().eth_address(), 1, &NONCE); + PaymentQuote::verify("e_json([3u8; 20]), Some(&good), 1, None, ceiling()) + .expect("a derivable overlay verifies"); + assert_eq!( + PaymentQuote::verify("e_json([3u8; 20]), Some(&[9u8; 32]), 1, None, ceiling()), + Err(QuoteError::OverlayMismatch) + ); + } + + /// Tampering with any signed field must break the signature — otherwise + /// a lane could serve one price and bill another. + #[test] + fn tampering_with_the_quote_breaks_it() { + for field in ["price_plur_per_kib", "beneficiary", "origin", "chain_id"] { + let mut q = quote_json([3u8; 20]); + q[field] = match field { + "price_plur_per_kib" => serde_json::json!("1"), + "beneficiary" => serde_json::json!(format!("0x{}", hex::encode([0xAAu8; 20]))), + "origin" => serde_json::json!("evil.example"), + _ => serde_json::json!(1u64), + }; + assert!( + PaymentQuote::verify(&q, None, 1, None, ceiling()).is_err(), + "tampering with {field} must be caught" + ); + } + } + + /// The root of trust: an identity the client already knew, not one the + /// lane asserts about itself. + #[test] + fn a_quote_from_an_unpinned_identity_is_refused() { + let pin = LanePin { + node_eth_address: [0xEE; 20], + beneficiary: [3u8; 20], + }; + let e = PaymentQuote::verify("e_json([3u8; 20]), None, 1, Some(&pin), ceiling()) + .expect_err("must refuse"); + assert!(matches!(e, QuoteError::WrongSigner { .. }), "got {e:?}"); + } + + /// §11.3: a correctly-signed relay advertising someone else's + /// beneficiary must not be paid. + #[test] + fn a_quote_with_an_unpinned_beneficiary_is_refused() { + let pin = LanePin { + node_eth_address: *node().eth_address(), + beneficiary: [3u8; 20], + }; + let e = PaymentQuote::verify("e_json([0xBB; 20]), None, 1, Some(&pin), ceiling()) + .expect_err("must refuse"); + assert!( + matches!(e, QuoteError::WrongBeneficiary { .. }), + "got {e:?}" + ); + } + + #[test] + fn an_overpriced_or_bricking_lane_is_refused() { + let e = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, 1) + .expect_err("price ceiling"); + assert!(matches!(e, QuoteError::TooExpensive { .. }), "got {e:?}"); + + // A lane whose dust floor exceeds its settlement window would brick + // this client with no cheque able to clear the 402. + let n = node(); + let mut body = quote_json([3u8; 20]); + body["min_cheque_plur"] = + serde_json::json!((Params::default().settle_every_plur * 87).to_string()); + let mut unsigned = body.clone(); + unsigned.as_object_mut().unwrap().remove("sig"); + let sig = n + .sign_eip191(unsigned.to_string().as_bytes()) + .expect("sign"); + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + let e = PaymentQuote::verify(&body, None, 1, None, ceiling()).expect_err("bricking lane"); + assert!(matches!(e, QuoteError::BadParams(_)), "got {e:?}"); + } + + // The only test here that needs the relay half, to prove the two ends + // of the header agree. Everything else about paying is testable without + // a server, which is the point of building the client half separately. + #[test] + #[cfg(feature = "pusher")] + fn a_challenge_round_trips_into_a_header_the_relay_accepts() { + use crate::ledger::Ledger; + use crate::metered::{MeterConfig, Metered}; + let acct_signer = node(); + let account = *acct_signer.eth_address(); + let m = Metered::new( + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: alloy_primitives::Address::ZERO, + params: Params::default(), + hard_mode: false, + }, + Ledger::ephemeral(), + ); + let issued = m + .issue( + account, + [7u8; 32], + 6_200_000_000_000_000_000, + "relay-a.example", + 1000, + ) + .expect("issue"); + // Straight through the wire form the relay actually serves. + let offered = OfferedChallenge::parse(&issued.to_json()).expect("parse"); + let header = offered.sign(&acct_signer, 100).expect("sign"); + let v = m.verify_header(&header, 1000).expect("relay must accept"); + assert_eq!(v.account, account); + assert_eq!(v.batch, [7u8; 32]); + } + + #[test] + fn owed_tracks_bytes_sent_and_a_cheque_clears_it() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let body = 32 * 1024 * 1024; + a.record_sent(body); + a.record_answered(body, true); + assert_eq!(a.owed(), p.price_bytes(body)); + assert!(a.should_settle(), "32 MiB crosses the settlement window"); + let c = a.next_cumulative().expect("above the dust floor"); + a.settled(c); + assert_eq!(a.owed(), 0); + assert_eq!(a.cumulative(), c); + } + + /// Cheques are cumulative, so a second upload adds to the same running + /// total rather than starting over — which is what a relay's + /// monotonicity check requires. + #[test] + fn a_second_upload_grows_the_same_cumulative() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(40 * 1024 * 1024); + a.record_answered(40 * 1024 * 1024, true); + let first = a.next_cumulative().expect("cheque"); + a.settled(first); + a.record_sent(40 * 1024 * 1024); + a.record_answered(40 * 1024 * 1024, true); + let second = a.next_cumulative().expect("cheque"); + assert!( + second > first, + "cumulative must increase: {second} > {first}" + ); + assert_eq!(second - first, p.price_bytes(40 * 1024 * 1024)); + } + + #[test] + fn a_cumulative_restored_from_disk_keeps_increasing() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]).with_cumulative(5_000_000_000_000_000); + a.record_sent(40 * 1024 * 1024); + a.record_answered(40 * 1024 * 1024, true); + let c = a.next_cumulative().expect("cheque"); + assert!( + c > 5_000_000_000_000_000, + "a fresh run must not re-issue below what a previous run already sent" + ); + } + + #[test] + fn dust_owings_do_not_produce_a_cheque_the_relay_would_refuse() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(1024); + assert!(!a.should_settle()); + assert_eq!(a.next_cumulative(), None, "below the lane's dust floor"); + } + + /// The divergence a live run found: recording debt at dispatch made the + /// client sign cheques for several times what the relay had booked, + /// because a 402'd POST is never billed on the relay side. + /// A POST whose response broke still cost the relay the bytes it read, + /// so it must still be billed — otherwise the client silently + /// under-pays for every interrupted stream (§7.3). + #[test] + fn an_interrupted_post_is_still_billed() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let body = 64 * 1024; + a.record_sent(body); + a.record_answered(body, true); // stream broke, but the body arrived + assert_eq!(a.owed(), p.price_bytes(body)); + } + + #[test] + fn a_refused_post_never_becomes_debt() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(100 * 4251); + assert_eq!(a.owed(), 0, "in flight is not yet owed"); + assert!(a.outstanding() > 0, "but it does count against the cap"); + a.record_answered(100 * 4251, false); // 402 + assert_eq!(a.owed(), 0, "a refused POST is never billed"); + assert_eq!(a.outstanding(), 0, "and stops holding headroom"); + } + + #[test] + fn an_accepted_post_becomes_debt_exactly_once() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let body = 100 * 4251; + a.record_sent(body); + a.record_answered(body, true); + assert_eq!(a.owed(), p.price_bytes(body)); + assert_eq!(a.outstanding(), a.owed(), "nothing left in flight"); + } + + /// Several POSTs dispatch before any completes; the cap must see their + /// sum, or the client blows through it and 402s on the tail. + #[test] + fn concurrent_posts_all_count_against_the_cap() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + for _ in 0..8 { + a.record_sent(64 * 1024); + } + assert_eq!(a.outstanding(), p.price_bytes(64 * 1024) * 8); + assert_eq!(a.owed(), 0); + } + + /// A client can charge itself for a POST whose completion it never + /// saw; the relay never billed it, so it refuses the cheque. Carrying + /// that debt forever would slowly eat the client's own credit line. + #[test] + fn phantom_debt_can_be_dropped() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(4251); + a.record_answered(4251, true); + assert!(a.owed() > 0); + a.forgive_phantom_debt(); + assert_eq!(a.owed(), 0); + assert_eq!(a.outstanding(), 0, "and it stops holding headroom"); + } + + #[test] + fn dedup_hits_are_refunded() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(100 * 4251); + a.record_answered(100 * 4251, true); + let before = a.owed(); + a.refund_dedup(10 * 4251); + assert!(a.owed() < before); + } + + /// §7.2: size the POST to the credit line instead of discovering the + /// ceiling as a 402. A dust batch gets a small but usable body. + #[test] + fn post_size_is_bounded_by_the_credit_line() { + let p = Params::default(); + let a = LaneAccount::new(p, [3u8; 20]); + let dust_cap = p.credit_line(100_000_000_000_000); + let max = a.max_body_bytes(dust_cap); + assert_eq!(max, 208 * 1024, "~208 KiB, matching the credit line"); + assert!(max >= 4251, "and still enough for at least one frame"); + // A rich batch is bounded by the global ceiling instead. + let rich = a.max_body_bytes(p.max_outstanding_plur); + assert!(rich > 512 * 4251, "a full POST fits comfortably"); + } + + /// Concurrent POSTs hold reservations on the relay before they are + /// debt. Sizing the next body against `owed` alone gave each in-flight + /// POST the whole line, so their reservations summed past the cap and + /// the relay 402'd a client with nothing to pay — an unpayable refusal + /// that only clears by waiting. + #[test] + fn post_size_accounts_for_bytes_already_in_flight() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let cap = p.credit_line(100_000_000_000_000); + let whole_line = a.max_body_bytes(cap); + // Put half the line on the wire, unanswered: still pending, not owed. + let in_flight = whole_line / 2; + a.record_sent(in_flight); + assert_eq!(a.owed(), 0, "in-flight bytes are not debt yet"); + let next = a.max_body_bytes(cap); + assert!( + next <= whole_line - in_flight, + "sized {next} with {in_flight} already on the wire against a {whole_line} line" + ); + // And the two together must fit, which is the property the relay + // actually enforces. + assert!(a.outstanding().saturating_add(p.price_bytes(next)) <= cap); + } + + #[test] + fn headroom_shrinks_as_debt_accrues() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let cap = p.max_outstanding_plur; + let before = a.max_body_bytes(cap); + a.record_sent(64 * 1024 * 1024); + a.record_answered(64 * 1024 * 1024, true); + assert!(a.max_body_bytes(cap) < before, "unpaid debt eats the line"); + } + + /// A lane whose credit line is about one POST wide must keep making + /// progress, in smaller POSTs. + /// + /// Regression: gating dispatch on whether a *full* POST fits stalled + /// the upload outright here. The leftover debt is under + /// `min_cheque_plur`, so it cannot be settled (§10.2) and the headroom + /// the guard waited for could never come back — 60 of 76 chunks were + /// left unacked against a live relay. + /// A relay's reported debt is trusted only up to the ceiling it signed. + /// + /// Reconcile adopts the relay's own figure (§17.1), which is sound + /// *within the credit it granted*: admission refuses above the cap, and + /// every per-batch cap is `min(value / ratio, ceiling)`. Above the + /// ceiling there is no story in which the debt was legitimately + /// incurred, so the figure is refused rather than signed for. + /// + /// This matters because a relay is not a curated identity — it is an + /// HTTP service the client pinned. Bounding only by the chequebook + /// balance, as an earlier version did, let any lane a client pointed at + /// name the whole balance and be paid it. + #[test] + fn a_relay_cannot_claim_more_debt_than_the_ceiling_it_signed() { + let q = + PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let cap = q.params.max_outstanding_plur; + let payer = LanePayer::new("http://lane".into(), q, 0); + // A chequebook far richer than the credit line, which is the normal + // case and exactly what made the old bound useless. + let balance = cap * 1000; + + payer + .check_reported_debt(cap, balance) + .expect("debt exactly at the ceiling is legitimate"); + payer + .check_reported_debt(cap - 1, balance) + .expect("and anything under it"); + + let e = payer + .check_reported_debt(cap + 1, balance) + .expect_err("one PLUR above the ceiling cannot have been admitted"); + assert!(e.contains("ceiling"), "got {e}"); + + let e = payer + .check_reported_debt(balance, balance) + .expect_err("and the whole chequebook certainly cannot"); + assert!(e.contains("ceiling"), "got {e}"); + } + + /// The balance still bounds, for a chequebook too thin to cover a + /// legitimate debt — caught here rather than as a confusing failure at + /// signing time. + #[test] + fn debt_within_the_ceiling_but_over_the_balance_is_refused() { + let q = + PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let cap = q.params.max_outstanding_plur; + let payer = LanePayer::new("http://lane".into(), q, 0); + + let e = payer + .check_reported_debt(cap, cap / 2) + .expect_err("we cannot sign for more than the chequebook holds"); + assert!(e.contains("balance"), "got {e}"); + } + + #[test] + fn a_line_barely_wider_than_one_post_still_makes_progress() { + let q = + PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let p = q.params; + let mut payer = LanePayer::new("http://lane".into(), q, 0); + + // A line that fits one full POST and very little more — the shape a + // batch decays into as its remaining value is spent down. + let full_post = p.price_bytes((512 * crate::pushframe::MAX_FRAME_LEN) as u64); + payer.set_cap_for_test(full_post + full_post / 20); + + let first = payer.affordable_frames(); + assert!(first > 0, "a fresh lane can dispatch"); + + // Send one small POST and leave the debt unsettleable. + let sent = 16 * crate::pushframe::MAX_FRAME_LEN as u64; + payer.account.record_sent(sent); + payer.account.record_answered(sent, true); + assert!(payer.account.owed() > 0); + assert!( + payer.account.next_cumulative().is_none(), + "this residual is below the dust floor, so no cheque can clear it" + ); + + let next = payer.affordable_frames(); + assert!( + next > 0, + "must still dispatch a smaller POST; a lane that can never settle \ + and never send has stalled the upload" + ); + // And whatever it sizes must actually fit, or the relay 402s it. + let body = p.price_bytes((next * crate::pushframe::MAX_FRAME_LEN) as u64); + assert!( + payer.account.outstanding().saturating_add(body) <= payer.cap_plur(), + "sized {next} frames but that does not fit the remaining line" + ); + } + + /// The relay's ledger outlives the client's. Every run ends leaving the + /// sub-dust residual unpaid, and a fresh process starts believing it + /// owes nothing — so the relay refuses service for debt the client + /// cannot compute a cheque for. Adopting the relay's figure is the only + /// way out, and it is what the 402 recovery path does. + #[test] + fn carried_debt_from_a_previous_run_is_adoptable() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + // Fresh process: no debt, and nothing to pay with. + assert_eq!(a.owed(), 0); + assert_eq!( + a.next_cumulative(), + None, + "cannot pay what it does not know" + ); + + let carried = p.min_cheque_plur * 3; + assert!(a.adopt_relay_debt(carried), "relay knows more than we do"); + assert_eq!(a.owed(), carried); + assert_eq!( + a.next_cumulative(), + Some(carried), + "now a cheque clears the refusal" + ); + } + + /// §7.3's ack-tail leaves the two sides disagreeing in the *other* + /// direction, and the client must yield. + /// + /// A POST whose response stream breaks was still read, so the client + /// bills it — but if the relay's task is cancelled before it commits, + /// the `Admitted` guard releases the reservation and it books nothing. + /// The overshoot then rides on every later cheque, each refused for the + /// same reason, and the lane never settles again. Seen over a real + /// proxy: `credits 212640000000 but only 148800000000 is owed`. + #[test] + fn a_relay_that_booked_less_than_we_billed_is_taken_at_its_word() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + // Large enough that both figures clear the dust floor, so the + // assertion is about the correction and not about §10.2. + let sent = 16 * 1024 * 1024u64; + a.record_sent(sent); + a.record_answered(sent, true); + let ours = a.owed(); + assert!(ours > p.min_cheque_plur); + + // The relay booked less than we billed ourselves. + let theirs = ours - p.price_bytes(133 * 1024); + assert!(theirs > p.min_cheque_plur); + a.sync_relay_debt(theirs); + assert_eq!(a.owed(), theirs, "the relay decides what it will accept"); + assert_eq!( + a.next_cumulative(), + Some(theirs), + "and the corrected cheque is exactly its figure" + ); + } + + /// Reconciling mid-flight must not bill the same POST twice. + /// + /// Regression: the relay finishes reading a body and books it, so its + /// reported `owed` already covers a POST the client still has in + /// `pending`. Adopting that figure raw and then letting + /// `record_answered` move the same bytes into `owed` over-counted by + /// one POST, and the run ended with `cheque credits 535680000000 but + /// only 510240000000 is owed` — settlement jammed for the rest of it. + #[test] + fn adopting_relay_debt_does_not_double_count_bytes_in_flight() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + + let body = 64 * 1024u64; + let priced = p.price_bytes(body); + a.record_sent(body); + assert_eq!(a.pending(), priced, "on the wire, not yet debt"); + + // The relay has read that body and reports it as owed, while our + // response has not closed yet. + a.adopt_relay_debt(priced); + // Now it closes. + a.record_answered(body, true); + + assert_eq!( + a.owed(), + priced, + "billed once, not twice: adopted {priced} then answered the same body" + ); + } + + /// Only ever upward. The downward direction is `forgive_phantom_debt`, + /// which is reached from a rejected cheque — a relay reporting *less* + /// than we think must not silently shrink a debt we are still liable + /// for, and an over-count is a cheque the relay refuses outright. + #[test] + fn adopting_relay_debt_never_lowers_our_own() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(1024 * 1024); + a.record_answered(1024 * 1024, true); + let owed = a.owed(); + assert!(owed > 0); + + assert_eq!(a.pending(), 0, "nothing in flight, so nothing to deduct"); + assert!(!a.adopt_relay_debt(owed - 1), "a smaller figure is ignored"); + assert_eq!(a.owed(), owed); + assert!(!a.adopt_relay_debt(0), "and zero especially so"); + assert_eq!(a.owed(), owed); + assert!(a.adopt_relay_debt(owed + 1), "larger still wins"); + assert_eq!(a.owed(), owed + 1); + } + + /// N lanes are N claims on one balance. The client must see the sum, or + /// the second cheque bounces. + #[test] + fn total_issued_aggregates_across_beneficiaries() { + let mut t = TotalIssued::default(); + t.record([1u8; 20], 600); + t.record([2u8; 20], 300); + assert_eq!(t.total(), 900); + assert!(!t.would_exceed(&[1u8; 20], 700, 1000), "700 + 300 fits"); + assert!(t.would_exceed(&[1u8; 20], 800, 1000), "800 + 300 does not"); + assert!( + !t.would_exceed(&[3u8; 20], 100, 1000), + "a new beneficiary is checked against the others' total" + ); + } + + #[test] + fn a_stale_cumulative_report_never_lowers_the_total() { + let mut t = TotalIssued::default(); + t.record([1u8; 20], 600); + t.record([1u8; 20], 100); + assert_eq!(t.total(), 600, "cumulatives only grow"); + } +} diff --git a/src/protocols/pushsync.rs b/src/protocols/pushsync.rs index 9936b3e..303021b 100644 --- a/src/protocols/pushsync.rs +++ b/src/protocols/pushsync.rs @@ -121,6 +121,31 @@ where if !receipt.err.is_empty() { return Err(PushsyncError::Peer(receipt.err)); } + // The receipt must be *for the chunk we pushed*. Two things go wrong + // without this check, and both are reachable by any peer in the pool + // (membership comes from hive gossip and the seed list, so it is not a + // trusted set): + // + // 1. `address` is copied straight off the wire, and callers build a + // `[u8; 32]` from it with `copy_from_slice` (`src/client.rs:5380`, + // `:5398`), which panics on any other length. A 0- or 33-byte + // address is a remote panic that unwinds the push task. + // 2. Nothing else compares it to what we sent. A peer could accept the + // Delivery, store nothing, and sign a receipt for some *other* + // address deep inside its own neighborhood: `is_shallow` and the + // `po` computation both read `r.address`, so the forged receipt + // looks like a perfect deep delivery and the chunk is acked `ok` + // having never been stored. + // + // Checking it here means every `PushsyncReceipt` in the codebase + // carries exactly the 32-byte address that was pushed. + if receipt.address != address[..] { + return Err(PushsyncError::Peer(format!( + "receipt address mismatch: pushed {}, receipt {}", + hex::encode(address), + hex::encode(&receipt.address), + ))); + } Ok(PushsyncReceipt { address: receipt.address, signature: receipt.signature, @@ -128,3 +153,111 @@ where storage_radius: receipt.storage_radius, }) } + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message; + use std::pin::Pin; + use std::task::{Context, Poll}; + + /// Stream whose read side replays a canned peer response and whose + /// write side is discarded — enough to drive `push` to the receipt. + struct Canned { + read: Vec, + pos: usize, + } + + impl futures::AsyncRead for Canned { + fn poll_read( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let n = (self.read.len() - self.pos).min(buf.len()); + buf[..n].copy_from_slice(&self.read[self.pos..self.pos + n]); + self.pos += n; + Poll::Ready(Ok(n)) + } + } + + impl futures::AsyncWrite for Canned { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Poll::Ready(Ok(buf.len())) + } + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + /// Response headers followed by `receipt`, both length-delimited. + fn peer_saying(receipt: pb::Receipt) -> Canned { + let mut read = Vec::new(); + hdr::Headers { headers: vec![] } + .encode_length_delimited(&mut read) + .expect("encode headers"); + receipt + .encode_length_delimited(&mut read) + .expect("encode receipt"); + Canned { read, pos: 0 } + } + + fn receipt_for(address: Vec) -> pb::Receipt { + pb::Receipt { + address, + signature: vec![7u8; 65], + nonce: vec![9u8; 32], + err: String::new(), + storage_radius: 8, + } + } + + fn push_against(receipt: pb::Receipt) -> Result { + let pushed = [1u8; 32]; + let mut stream = peer_saying(receipt); + tokio_test::block_on(push(&mut stream, &pushed, &[0u8; 16], &[0u8; 113])) + } + + #[test] + fn receipt_for_the_pushed_address_is_accepted() { + let r = push_against(receipt_for(vec![1u8; 32])).expect("should accept"); + assert_eq!(r.address, vec![1u8; 32]); + assert_eq!(r.storage_radius, 8); + } + + /// A peer can otherwise sign a receipt for a *different* address deep in + /// its own neighborhood, having stored nothing: `is_shallow` and the `po` + /// computation both read the receipt's address, so it would look like a + /// perfect delivery. + #[test] + fn receipt_for_a_different_address_is_rejected() { + let err = push_against(receipt_for(vec![2u8; 32])).expect_err("should reject"); + assert!( + matches!(&err, PushsyncError::Peer(m) if m.contains("address mismatch")), + "unexpected error: {err}" + ); + } + + /// Callers build `[u8; 32]` from this with `copy_from_slice`, which + /// panics on any other length — so a malformed address must be rejected + /// here rather than unwinding the push task. + #[test] + fn missized_addresses_are_rejected_not_panicked_on() { + for bad in [vec![], vec![1u8; 31], vec![1u8; 33], vec![1u8; 64]] { + let n = bad.len(); + let err = + push_against(receipt_for(bad)).expect_err("missized address must be rejected"); + assert!( + matches!(&err, PushsyncError::Peer(m) if m.contains("address mismatch")), + "unexpected error for {n}-byte address: {err}" + ); + } + } +} diff --git a/src/protocols/swap.rs b/src/protocols/swap.rs index 905df88..702f4e4 100644 --- a/src/protocols/swap.rs +++ b/src/protocols/swap.rs @@ -105,6 +105,18 @@ fn encode_signed_cheque_json( .into_bytes() } +/// Public wrapper: the client needs to produce this body for a metered +/// relay's `POST /v1/pay`, and it must be byte-identical to what bee's +/// `json.Marshal` emits (see the encoder's own note on `*big.Int`). +pub fn encode_signed_cheque_json_pub( + chequebook: &[u8; 20], + beneficiary: &[u8; 20], + cumulative_payout: U256, + signature: &[u8; 65], +) -> Vec { + encode_signed_cheque_json(chequebook, beneficiary, cumulative_payout, signature) +} + /// Outbound `Handshake { Beneficiary }`. Called once per session by /// the connection-setup path. Caller exchanges empty headers first. pub async fn send_handshake(stream: &mut S, beneficiary: &[u8; 20]) -> Result<(), SwapError> @@ -179,3 +191,235 @@ where write_message(stream, &msg).await?; Ok(()) } + +// ────────────────────────────────────────────────────────────────────── +// Inbound cheques (metered relay — docs/pusher-incentives.md Stage 1) +// ────────────────────────────────────────────────────────────────────── + +/// A cheque as it arrives at `POST /v1/pay`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SignedCheque { + pub chequebook: [u8; 20], + pub beneficiary: [u8; 20], + pub cumulative_payout: U256, + pub signature: [u8; 65], +} + +/// Largest body we will even look at. A cheque is ~250 bytes; anything +/// bigger is not one, and scanning it is work an unauthenticated caller +/// should not be able to buy (§11.6). +pub const MAX_CHEQUE_JSON: usize = 4096; + +/// Decode the JSON `encode_signed_cheque_json` produces — and that bee's +/// `json.Marshal(SignedCheque)` produces, which is the same bytes. +/// +/// `CumulativePayout` is extracted from the raw input rather than through +/// `serde_json`, for the same reason the encoder is hand-written: Go emits +/// `*big.Int` as a **bare JSON number**, and `serde_json` without +/// `arbitrary_precision` silently widens anything past `u64` into `f64`. +/// A cheque for 10^20 PLUR would round to a different number and still +/// parse, so the relay would credit an amount the signature does not cover +/// and the recovered issuer would be garbage. Losing precision here is not +/// a rounding bug, it is a money bug. +pub fn decode_signed_cheque_json(bytes: &[u8]) -> Result { + if bytes.len() > MAX_CHEQUE_JSON { + return Err(SwapError::Json(format!( + "cheque body too large: {} bytes (max {MAX_CHEQUE_JSON})", + bytes.len() + ))); + } + let v: serde_json::Value = + serde_json::from_slice(bytes).map_err(|e| SwapError::Json(e.to_string()))?; + + let chequebook = json_address(&v, "Chequebook")?; + let beneficiary = json_address(&v, "Beneficiary")?; + let cumulative_payout = extract_cumulative(bytes)?; + + let sig_b64 = v + .get("Signature") + .and_then(|s| s.as_str()) + .ok_or_else(|| SwapError::Json("missing Signature".into()))?; + let sig_bytes = { + use base64::Engine; + base64::engine::general_purpose::STANDARD + .decode(sig_b64) + .map_err(|e| SwapError::Json(format!("Signature base64: {e}")))? + }; + // Canonical form is checked *here*, at the only place every cheque + // passes through, rather than left to each caller. A high-`s` or + // `v ∈ {0,1}` cheque recovers fine off-chain and reverts at cashout + // (§11.6), so accepting one means giving away service for a signature + // that can never be redeemed. + crate::signer::check_canonical_signature(&sig_bytes) + .map_err(|e| SwapError::Json(e.to_string()))?; + let mut signature = [0u8; 65]; + signature.copy_from_slice(&sig_bytes); + + Ok(SignedCheque { + chequebook, + beneficiary, + cumulative_payout, + signature, + }) +} + +fn json_address(v: &serde_json::Value, field: &str) -> Result<[u8; 20], SwapError> { + let s = v + .get(field) + .and_then(|x| x.as_str()) + .ok_or_else(|| SwapError::Json(format!("missing {field}")))?; + let hex_str = s.trim_start_matches("0x").trim_start_matches("0X"); + let raw = hex::decode(hex_str).map_err(|e| SwapError::Json(format!("{field} not hex: {e}")))?; + if raw.len() != 20 { + return Err(SwapError::Json(format!( + "{field} must be 20 bytes, got {}", + raw.len() + ))); + } + let mut out = [0u8; 20]; + out.copy_from_slice(&raw); + Ok(out) +} + +/// Pull `CumulativePayout`'s digits straight out of the input. +/// +/// Rejects a repeated key rather than picking one: duplicate keys are legal +/// JSON and `serde_json` keeps the last, so if the scan and the parser +/// disagreed about which one counts, an attacker could show the relay one +/// amount and the signature check another. +fn extract_cumulative(bytes: &[u8]) -> Result { + const KEY: &[u8] = b"\"CumulativePayout\""; + let mut hits = bytes + .windows(KEY.len()) + .enumerate() + .filter(|(_, w)| *w == KEY) + .map(|(i, _)| i); + let at = hits + .next() + .ok_or_else(|| SwapError::Json("missing CumulativePayout".into()))?; + if hits.next().is_some() { + return Err(SwapError::Json("duplicate CumulativePayout key".into())); + } + let mut i = at + KEY.len(); + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if i >= bytes.len() || bytes[i] != b':' { + return Err(SwapError::Json("CumulativePayout is not a field".into())); + } + i += 1; + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + let start = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if start == i { + return Err(SwapError::Json( + "CumulativePayout must be a bare JSON integer, as Go emits it".into(), + )); + } + let digits = std::str::from_utf8(&bytes[start..i]) + .map_err(|e| SwapError::Json(format!("CumulativePayout utf8: {e}")))?; + U256::from_str_radix(digits, 10) + .map_err(|e| SwapError::Json(format!("CumulativePayout overflows u256: {e}"))) +} + +#[cfg(test)] +mod cheque_decode_tests { + use super::*; + + fn sample(cumulative: U256) -> Vec { + encode_signed_cheque_json(&[0x11; 20], &[0x22; 20], cumulative, &canonical_sig()) + } + + /// `v = 27` and a low `s`, so it survives the canonical check. + fn canonical_sig() -> [u8; 65] { + let mut s = [0x01u8; 65]; + s[64] = 27; + s + } + + #[test] + fn round_trips_our_own_encoder() { + let amount = U256::from(1_234_567u64); + let got = decode_signed_cheque_json(&sample(amount)).expect("decode"); + assert_eq!(got.chequebook, [0x11; 20]); + assert_eq!(got.beneficiary, [0x22; 20]); + assert_eq!(got.cumulative_payout, amount); + assert_eq!(got.signature, canonical_sig()); + } + + /// The reason the decoder is hand-written. `serde_json` without + /// `arbitrary_precision` widens this to `f64` and hands back a + /// *different number* — which would have the relay credit an amount the + /// signature never covered. + #[test] + fn a_payout_past_u64_survives_exactly() { + // 2^80 + 1: needs u256, and is not representable in f64. + let amount = U256::from(1u64) << 80 | U256::from(1u64); + let body = sample(amount); + let got = decode_signed_cheque_json(&body).expect("decode"); + assert_eq!(got.cumulative_payout, amount); + + let lossy = serde_json::from_slice::(&body) + .expect("parses") + .get("CumulativePayout") + .and_then(|n| n.as_u64()); + assert!( + lossy.is_none(), + "if serde ever parses this exactly, the hand-rolled scan can go" + ); + } + + #[test] + fn a_payout_at_the_u256_ceiling_decodes() { + let amount = U256::MAX; + let got = decode_signed_cheque_json(&sample(amount)).expect("decode"); + assert_eq!(got.cumulative_payout, amount); + } + + #[test] + fn a_payout_past_the_u256_ceiling_is_rejected() { + let mut body = String::from_utf8(sample(U256::from(1u64))).expect("utf8"); + body = body.replace( + "\"CumulativePayout\":1,", + &format!("\"CumulativePayout\":{},", "9".repeat(78)), + ); + decode_signed_cheque_json(body.as_bytes()).expect_err("must not wrap around"); + } + + /// Duplicate keys are legal JSON and serde keeps the last. If the scan + /// picked the first, the relay would credit one amount while verifying + /// a signature over another. + #[test] + fn a_duplicated_payout_key_is_rejected_not_guessed() { + let body = br#"{"Chequebook":"0x1111111111111111111111111111111111111111","Beneficiary":"0x2222222222222222222222222222222222222222","CumulativePayout":1,"CumulativePayout":999999,"Signature":"AQ=="}"#; + let e = decode_signed_cheque_json(body).expect_err("ambiguous"); + assert!(format!("{e}").contains("duplicate"), "got: {e}"); + } + + #[test] + fn non_canonical_signatures_are_rejected_at_the_boundary() { + let mut bad = canonical_sig(); + bad[64] = 1; + let body = encode_signed_cheque_json(&[0x11; 20], &[0x22; 20], U256::from(5u64), &bad); + let e = decode_signed_cheque_json(&body).expect_err("uncashable cheque"); + assert!(format!("{e}").contains("non-canonical"), "got: {e}"); + } + + #[test] + fn malformed_bodies_are_rejected_not_panicked_on() { + for body in [ + &b"{}"[..], + b"not json", + b"[]", + br#"{"Chequebook":"0x11","Beneficiary":"0x22","CumulativePayout":1,"Signature":"AQ=="}"#, + br#"{"Chequebook":"0x1111111111111111111111111111111111111111","Beneficiary":"0x2222222222222222222222222222222222222222","CumulativePayout":"1","Signature":"AQ=="}"#, + ] { + decode_signed_cheque_json(body).expect_err("must reject"); + } + decode_signed_cheque_json(&vec![b'x'; MAX_CHEQUE_JSON + 1]).expect_err("oversized"); + } +} diff --git a/src/pusher.rs b/src/pusher.rs index c0a78e9..4bb0d39 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -92,6 +92,34 @@ const PUSH_POOL_TARGET_DEFAULT: usize = 128; const PUSH_POOL_TARGET_MAX: usize = 512; /// Per-chunk retry budget on the push path. const PUSH_MAX_RETRIES: usize = 20; +/// Default cap on concurrent inbound connections +/// (`HOVERFLY_PUSH_MAX_CONNS`). A `/v1/push` in flight holds its collected +/// body plus decoded frames — a couple of MiB at `PUSH_BATCH_MAX` — so an +/// uncapped accept loop is an uncapped memory commitment. +const PUSH_MAX_CONNS_DEFAULT: usize = 256; +/// Ceiling on the accept-loop retry backoff. +const ACCEPT_BACKOFF_MAX_MS: u64 = 1000; +/// How long a connection may take to send its request headers. Only the +/// header read — response streaming is unbounded by design. +const HEADER_READ_TIMEOUT_SECS: u64 = 30; +/// How long `/v1/push` may take to receive its (already size-capped) body. +/// Generous for ~2 MiB on a slow mobile link, but finite: the read happens +/// before any work is spawned, so an unbounded one is free to hold. +const PUSH_BODY_READ_TIMEOUT_SECS: u64 = 120; +/// Bound on the `batch_id → owner` cache. Batch ids are enumerable +/// on-chain (`BatchCreated`), so an unbounded map is a remote memory sink. +const OWNER_CACHE_CAP: usize = 4096; +/// How long a successful batch resolution stays cached. Bounded so a batch +/// that expires while cached stops being served indefinitely. +const OWNER_OK_TTL_SECS: u64 = 1800; +/// How long a *definitive* rejection (absent on-chain, or expired) stays +/// cached. This is what stops a flood of bogus batch ids from turning one +/// unauthenticated POST into one RPC round trip per frame. +const OWNER_BAD_TTL_SECS: u64 = 300; +/// Distinct batch resolutions that may reach the chain in a single POST. +/// Honest clients push one or a few batches per request; this bounds the +/// RPC amplification of a request that names 512 different ones. +const PUSH_MAX_BATCH_LOOKUPS: usize = 8; /// Recently-acked address cache: enough to cover several in-flight /// batches across every lane a client might hedge between. const RECENT_ACK_CAP: usize = 8192; @@ -125,6 +153,30 @@ pub struct PusherOpts { /// HOVERFLY_PUSHER_IDENTITY. `None` = reuse the stamp key. pub node_identity: Option, pub transport: TransportConfig, + /// Metered mode (`docs/pusher-incentives.md` Stage 1). `None` = `open`, + /// today's unmetered behaviour, which the production lanes keep running. + pub meter: Option, +} + +/// Everything `--meter` needs. Validated at startup: a relay that cannot +/// state its own origin, or whose parameters violate §10.1's invariant, +/// refuses to boot rather than serving a broken meter. +#[derive(Debug, Clone)] +pub struct MeterOpts { + /// `--origin`, one or more. **Required**, and never derived from a + /// request header (§11.1). + pub origins: Vec, + /// EOA that must appear as `Cheque.beneficiary`. The relay holds the + /// address only — never the key (§6). + pub beneficiary: [u8; 20], + /// Settlement chain. Pins the EIP-712 domain and the factory. + pub chain_id: u64, + pub params: crate::meter::Params, + /// Where the ledger and relay secret live. Required: metered mode + /// without durable state is an unbounded free-service loop (§11.4). + pub state_dir: PathBuf, + /// Stage 2. False = soft mode: meter and report, never refuse. + pub hard_mode: bool, } struct State { @@ -143,9 +195,82 @@ struct State { /// /v1/push requests (filled lazily on first push). `None` transport /// means the node key was unresolvable; /v1/push then 503s. push: Option, - /// `batch_id(hex) → on-chain owner`, so repeated pushes for one batch - /// cost a single RPC. - owner_cache: std::sync::Mutex>, + /// `batch_id(hex) → resolution`, so repeated pushes for one batch cost a + /// single RPC. Caches rejections too — see [`OwnerCache`]. + owner_cache: std::sync::Mutex, + /// Stage 0 shadow metering (`src/meter.rs`, incentives §14): counts what + /// a metered relay *would* have billed, bills nothing, and changes + /// nothing on the wire. Merged once per request, never per frame. + meter: std::sync::Mutex, + /// Stage 1 metered mode. `None` in `open` mode, which is every + /// production lane today. + metered: Option, +} + +/// Outcome of resolving a batch id on-chain. +#[derive(Clone)] +enum OwnerLookup { + /// Batch exists, is funded, and is owned by this address. Carries the + /// batch's total remaining value in PLUR (`remainingBalance × 2^depth`), + /// which costs nothing extra — `resolve_owner` already reads both halves + /// to check for expiry — and is what Stage 0 shadow metering prices a + /// credit line from (`src/meter.rs`, incentives §10.3). + Owner([u8; 20], u128), + /// Batch is *definitively* unusable: absent on-chain, or out of + /// balance. Carries the reason so the ack is unchanged. + Rejected(String), +} + +/// Bounded, TTL'd `batch_id → outcome` cache. +/// +/// Caching only *successes* was a live amplification bug. `stamp::validate` +/// checks that a signature recovers to a non-zero address and nothing else, +/// so any random key over any attacker-chosen batch id reaches the chain +/// read. With no negative entry, every frame naming a bogus batch re-issued +/// the RPC, so one unauthenticated POST of `PUSH_BATCH_MAX` frames became +/// that many serial `eth_call`s. Rejections are cached for a shorter TTL +/// than successes, so a batch that is topped up recovers quickly. +/// +/// Only definitive on-chain answers are cached. A transport error is *not* +/// cached — an RPC blip must not blacklist a live batch. +struct OwnerCache { + map: HashMap, + order: std::collections::VecDeque, + cap: usize, +} + +impl OwnerCache { + fn new(cap: usize) -> Self { + Self { + map: HashMap::new(), + order: std::collections::VecDeque::new(), + cap, + } + } + + fn get(&self, batch_id_hex: &str) -> Option { + let (entry, when) = self.map.get(batch_id_hex)?; + let ttl = match entry { + OwnerLookup::Owner(..) => OWNER_OK_TTL_SECS, + OwnerLookup::Rejected(_) => OWNER_BAD_TTL_SECS, + }; + (when.elapsed() < std::time::Duration::from_secs(ttl)).then(|| entry.clone()) + } + + fn insert(&mut self, batch_id_hex: &str, entry: OwnerLookup) { + if self + .map + .insert(batch_id_hex.to_string(), (entry, Instant::now())) + .is_none() + { + self.order.push_back(batch_id_hex.to_string()); + } + while self.order.len() > self.cap { + if let Some(old) = self.order.pop_front() { + self.map.remove(&old); + } + } + } } struct PushState { @@ -160,6 +285,10 @@ struct PushState { /// maintenance loop so the (sync) status handler never has to touch /// the async pool mutex. pool_live: AtomicUsize, + /// The node-identity key. Kept alongside the transport (which owns its + /// own clone) so the metered quote can be signed without rebuilding it + /// — it signs prices, never payments, and is not spendable (§6). + signer: crate::signer::SwarmSigner, /// This node's Kademlia overlay (node eth address + nonce). Published /// in `/v1/status` so a multi-lane client can route each chunk to the /// relay whose overlay is nearest the chunk's destination neighborhood @@ -171,21 +300,31 @@ struct PushState { /// The client turns `budget_remaining_gb` into a scheduling weight. budget_gb: Option, bytes_pushed: AtomicU64, - /// Addresses acked recently, so a duplicate frame (client hedging a - /// straggler across two lanes) is answered from cache instead of - /// paying a second real push. docs/pusher-design.md §7 "ChunkCache". + /// (Address, batch) pairs acked recently, so a duplicate frame (client + /// hedging a straggler across two lanes) is answered from cache + /// instead of paying a second real push. Keyed by batch so a dedup + /// hit can't substitute another uploader's stamp. + /// docs/pusher-design.md §7 "ChunkCache". recent: std::sync::Mutex, } -/// Bounded, TTL'd set of recently-acked chunk addresses. +/// Bounded, TTL'd set of recently-acked (chunk address, batch). +/// +/// Keyed on the batch too: a chunk address is content-derived, so it is +/// not unique across batch owners. Under a bare-address key a dedup hit +/// acks a frame `ok` while silently discarding the submitted stamp — one +/// uploader's dust batch could shadow another uploader's long-lived +/// batch for the TTL, and the victim's chunk is then garbage-collected +/// when the shadowing batch expires (docs/pusher-incentives.md §15). It +/// also fires spuriously between honest users uploading identical bytes. /// /// Insert order doubles as eviction order (a chunk's ack time only moves -/// forward), so a `VecDeque` of `(addr, when)` plus a `HashMap` index is -/// enough — no LRU bookkeeping, since re-acking an address doesn't need -/// to extend its life. +/// forward), so a `VecDeque` of `((addr, batch), when)` plus a `HashMap` +/// index is enough — no LRU bookkeeping, since re-acking an entry +/// doesn't need to extend its life. struct RecentAcks { - seen: HashMap<[u8; 32], Instant>, - order: std::collections::VecDeque<[u8; 32]>, + seen: HashMap<([u8; 32], [u8; 32]), Instant>, + order: std::collections::VecDeque<([u8; 32], [u8; 32])>, cap: usize, ttl: std::time::Duration, } @@ -200,13 +339,16 @@ impl RecentAcks { } } - fn contains(&self, addr: &[u8; 32]) -> bool { - self.seen.get(addr).is_some_and(|t| t.elapsed() < self.ttl) + fn contains(&self, addr: &[u8; 32], batch_id: [u8; 32]) -> bool { + self.seen + .get(&(*addr, batch_id)) + .is_some_and(|t| t.elapsed() < self.ttl) } - fn insert(&mut self, addr: [u8; 32]) { - if self.seen.insert(addr, Instant::now()).is_none() { - self.order.push_back(addr); + fn insert(&mut self, addr: [u8; 32], batch_id: [u8; 32]) { + let key = (addr, batch_id); + if self.seen.insert(key, Instant::now()).is_none() { + self.order.push_back(key); } while self.order.len() > self.cap { if let Some(old) = self.order.pop_front() { @@ -238,12 +380,23 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { warn!("push node identity unresolvable; /v1/push will 503 (probe/status still work)"); } + // Metered mode is validated *before* the listener binds. Every failure + // here is one that would otherwise be discovered by a paying client: + // a parameter set that bricks accounts (§10.1), an origin the relay + // cannot state (§11.1), a chain with no vetted factory (§6), or state + // it cannot persist (§11.4). Refuse to boot instead. + let metered = match build_metered(&opts) { + Ok(m) => m, + Err(e) => return Err(format!("--meter: {e}").into()), + }; + let listener = tokio::net::TcpListener::bind(opts.listen).await?; info!( - "pusher listening on http://{} (probe {}; push {}; {} known peers from {})", + "pusher listening on http://{} (probe {}; push {}; mode {}; {} known peers from {})", opts.listen, if opts.probe_enabled { "ON" } else { "off" }, if push.is_some() { "ON" } else { "off" }, + if metered.is_some() { "metered" } else { "open" }, peers_known, opts.peerlist.display(), ); @@ -255,7 +408,9 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { peers_known: AtomicUsize::new(peers_known), batch_cache: std::sync::Mutex::new(HashMap::new()), push, - owner_cache: std::sync::Mutex::new(HashMap::new()), + owner_cache: std::sync::Mutex::new(OwnerCache::new(OWNER_CACHE_CAP)), + meter: std::sync::Mutex::new(crate::meter::Meter::default()), + metered, }); // Background warm-pool maintenance: fill on startup and keep the pool @@ -266,26 +421,76 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { tokio::spawn(async move { push_maintenance(s).await }); } + // Bound concurrent connections. The permit is held for the connection's + // lifetime, so once `max_conns` are in flight the loop stops accepting and + // the OS backlog supplies the backpressure. Without this the accept loop + // spawned per connection with no cap at all, and (see the timer below) + // held those connections open forever. + let max_conns = std::env::var("HOVERFLY_PUSH_MAX_CONNS") + .ok() + .and_then(|s| s.parse::().ok()) + .map(|n| n.clamp(8, 4096)) + .unwrap_or(PUSH_MAX_CONNS_DEFAULT); + info!("pusher connection cap = {max_conns} (HOVERFLY_PUSH_MAX_CONNS to override)"); + let conns = Arc::new(tokio::sync::Semaphore::new(max_conns)); + let mut accept_backoff_ms = 0u64; + loop { - let (stream, _remote) = listener.accept().await?; + let Ok(permit) = conns.clone().acquire_owned().await else { + break Ok(()); // semaphore closed — shutting down + }; + let (stream, remote) = match listener.accept().await { + Ok(v) => { + accept_backoff_ms = 0; + v + } + Err(e) => { + // Per-connection accept errors (EMFILE, ENFILE, ECONNABORTED) + // are exactly what a connection burst produces against a + // process that also holds a warm libp2p pool. Propagating + // them with `?` killed the whole relay; back off instead. + let wait = accept_backoff_ms.max(10); + warn!("accept error: {e} — retrying in {wait}ms"); + tokio::time::sleep(std::time::Duration::from_millis(wait)).await; + accept_backoff_ms = (wait * 2).min(ACCEPT_BACKOFF_MAX_MS); + continue; + } + }; let io = hyper_util::rt::TokioIo::new(stream); let state = state.clone(); + // The peer address from the accept loop, never a client-supplied + // header: `/v1/challenge` rate-limits per IP, and a limiter keyed on + // anything the caller controls limits nothing. + let peer_ip = remote.ip().to_string(); tokio::spawn(async move { + // Held until the connection finishes, so the cap above is real. + let _permit = permit; let svc = service_fn(move |req| { let state = state.clone(); - async move { Ok::<_, Infallible>(handle(state, req).await) } + let peer_ip = peer_ip.clone(); + async move { Ok::<_, Infallible>(handle(state, req, &peer_ip).await) } }); - // Streamed probe responses outlive any sane header timeout; - // hyper's defaults are fine, errors here are just client - // disconnects. + // A timer MUST be installed or hyper's `header_read_timeout` + // default is silently inert: with `Time::Empty` the check logs + // "timeout has default, but no timer set" and returns `None`, so + // the previous builder had no header timeout whatsoever and a + // client could hold a connection open indefinitely by dribbling + // request headers. This bounds only the *request header* read — + // streamed probe and push responses are unaffected. let _ = hyper::server::conn::http1::Builder::new() + .timer(hyper_util::rt::TokioTimer::new()) + .header_read_timeout(std::time::Duration::from_secs(HEADER_READ_TIMEOUT_SECS)) .serve_connection(io, svc) .await; }); } } -async fn handle(state: Arc, req: Request) -> Response { +async fn handle( + state: Arc, + req: Request, + peer: &str, +) -> Response { // Browsers push cross-origin (a dApp on some origin → this relay), with a // custom content-type that triggers a CORS preflight. Answer OPTIONS and // tag every response with permissive CORS headers — the relay serves no @@ -296,10 +501,25 @@ async fn handle(state: Arc, req: Request) -> Respo } let mut resp = match (req.method(), req.uri().path()) { (&Method::GET, "/v1/status") => status_response(&state), + (&Method::GET, "/v1/meter") => meter_response(&state, req.headers()), + (&Method::GET, "/v1/account") => account_response(&state, req.headers()), + (&Method::GET, "/v1/challenge") => { + // Per-IP, because no account exists yet. `peer` comes from the + // accept loop, never from a client-supplied header. + challenge_response(state, req.uri().query(), peer).await + } + (&Method::POST, "/v1/pay") => pay_response(state, req).await, (&Method::POST, "/v1/probe") => probe_response(state, req.uri().query()), (&Method::POST, "/v1/tcpcheck") => tcpcheck_response(state, req.uri().query()), (&Method::POST, "/v1/push") => push_response(state, req).await, - (_, "/v1/probe") | (_, "/v1/status") | (_, "/v1/tcpcheck") | (_, "/v1/push") => { + (_, "/v1/probe") + | (_, "/v1/status") + | (_, "/v1/tcpcheck") + | (_, "/v1/push") + | (_, "/v1/meter") + | (_, "/v1/challenge") + | (_, "/v1/pay") + | (_, "/v1/account") => { json_line_response(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") } _ => json_line_response(StatusCode::NOT_FOUND, "not found"), @@ -308,6 +528,639 @@ async fn handle(state: Arc, req: Request) -> Respo resp } +/// What admission decided, carried into the push task so completion can +/// convert the reservation into debt. `None` in open mode. +/// +/// **This is an RAII guard, and it has to be.** A reservation is placed +/// before the body is read, but half a dozen things between there and the +/// push can fail — an oversize body, a read timeout, a frame-decode error, +/// an empty batch. Every one of those returns without ever reaching +/// `run_push`, which is the only place `commit` runs, and `commit` is the +/// only thing besides `release` that lowers `reserved_plur`. Paying a +/// cheque does not help: `Ledger::credit` reduces `owed`, never `reserved`. +/// +/// So a leaked reservation is permanent until restart, and it ratchets: +/// under hard mode the account eventually sits above its cap with **no +/// cheque able to clear it** — precisely the no-exit failure §10.1's +/// invariant exists to prevent — and under soft mode the leaks accumulate +/// against `MAX_LIVE_RESERVATIONS` until the relay sheds real clients. +/// +/// Releasing on drop makes every exit path correct by construction, +/// including ones added later, which is the whole reason it is a guard +/// rather than four hand-written `release` calls. +pub struct Admitted { + state: Arc, + account: [u8; 20], + batch: [u8; 32], + reserved_plur: u128, + settled: bool, +} + +impl Admitted { + /// Convert the reservation into debt for the bytes actually admitted, + /// releasing the remainder. Consumes the guard, so the `Drop` path + /// cannot double-release. + fn commit(mut self, billable_bytes: u64) { + let Some(m) = self.state.metered.as_ref() else { + return; + }; + let billed = m.cfg.params.price_bytes(billable_bytes); + let mut l = m.ledger.lock().expect("ledger poisoned"); + l.commit(self.account, self.reserved_plur, billed); + if let Err(e) = l.persist() { + // `owed` is written at batch completion, so a failed persist + // forfeits at most this batch — the safe direction (§10.2). + tracing::error!("ledger persist after commit failed: {e}"); + } + self.settled = true; + } +} + +impl Drop for Admitted { + fn drop(&mut self) { + if self.settled { + return; + } + if let Some(m) = self.state.metered.as_ref() { + m.ledger + .lock() + .expect("ledger poisoned") + .release(self.account, self.reserved_plur); + } + } +} + +/// Metered admission for `/v1/push` (§7.2). +/// +/// Runs before the body is read, and reads **no chain state** — the +/// challenge already carries the credit line, resolved once when it was +/// issued. That is what keeps up to 512 ecrecovers and an RPC round trip +/// off the front of every request. +fn admit_metered( + state: &Arc, + req: &Request, +) -> Result, Box>> { + let Some(m) = state.metered.as_ref() else { + return Ok(None); + }; + let raw = req + .headers() + .get(crate::challenge::CHALLENGE_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + // **Soft mode never refuses** (§7.1). A request with no challenge is an + // unmetered request, served exactly as `open` mode serves it, with + // Stage 0 still shadow-counting it. That is the whole point of shipping + // soft first: a relay can be flipped to `--meter` while the existing + // fleet keeps working, because clients that predate the protocol simply + // do not send the header. + // + // Requiring it unconditionally would 401 every current client on the + // lane the moment metering was enabled — the opposite of a staged + // rollout. Hard mode does require it, because by then there is a 402 to + // enforce and a client that cannot present a capability cannot be + // billed. + // + // A header that is *present but invalid* is refused in both modes: + // claiming a capability you do not hold is not the same as not claiming + // one, and letting it through would make the check bypassable by + // corrupting a byte. + if raw.is_empty() { + if m.cfg.hard_mode { + return Err(Box::new(json_line_response( + StatusCode::UNAUTHORIZED, + "metered relay: a challenge is required (GET /v1/challenge)", + ))); + } + return Ok(None); + } + let verified = m + .verify_header(raw, crate::challenge::now_unix()) + .map_err(|e| Box::new(json_line_response(StatusCode::UNAUTHORIZED, &e)))?; + if !m.allow_account(&verified.account) { + return Err(Box::new(json_line_response( + StatusCode::TOO_MANY_REQUESTS, + "slow down", + ))); + } + // The reservation ledger is attacker-influenced (one entry per batch in + // standing), so shed rather than grow without bound (§7.2). + if m.shed_reservations() { + return Err(Box::new(json_line_response( + StatusCode::SERVICE_UNAVAILABLE, + "too many accounts with live reservations", + ))); + } + // Bound the reservation by the *declared* body. Same quantity, same + // arithmetic as the eventual bill (§8), so there is no estimate to be + // wrong about — and a one-frame POST reserves one frame's worth rather + // than a flat PUSH_BATCH_MAX, which is what keeps small batches usable. + let declared = req + .headers() + .get(hyper::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + let Some(declared) = declared else { + return Err(Box::new(json_line_response( + StatusCode::LENGTH_REQUIRED, + "metered mode requires Content-Length so the reservation can be bounded", + ))); + }; + if declared > PUSH_MAX_BODY as u64 { + return Err(Box::new(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body exceeds limit", + ))); + } + let adm = m.reserve_for_body(verified.account, declared, verified.cap_plur); + if adm.over_cap { + if m.cfg.hard_mode { + // Hard mode: give the reservation back, then refuse. Keeping it + // would leak credit on every refusal. + m.ledger + .lock() + .expect("ledger poisoned") + .release(verified.account, adm.reserved_plur); + return Err(Box::new(json_response( + StatusCode::PAYMENT_REQUIRED, + &serde_json::json!({ + "error": "payment required", + "outstanding_plur": adm.outstanding_plur.to_string(), + "max_outstanding_plur": adm.cap_plur.to_string(), + "settle_every_plur": m.cfg.params.settle_every_plur.to_string(), + }), + ))); + } + // Soft mode: record and serve anyway. This is the instrument Stage 0 + // could not provide — how often a real client *would* have been + // 402'd, measured against live traffic before anyone is refused. + tracing::info!( + account = %hex::encode(verified.account), + outstanding_plur = %adm.outstanding_plur, + cap_plur = %adm.cap_plur, + "soft-mode overshoot: this request would 402 under hard mode" + ); + } + Ok(Some(Admitted { + state: state.clone(), + account: verified.account, + batch: verified.batch, + reserved_plur: adm.reserved_plur, + settled: false, + })) +} + +/// Validate `--meter` and build the metered state, or `None` for `open`. +/// +/// Every check here is one a paying client would otherwise discover the +/// hard way, so all of them are fatal rather than warnings. +fn build_metered(opts: &PusherOpts) -> Result, String> { + let Some(m) = &opts.meter else { + return Ok(None); + }; + m.params.validate()?; + if m.origins.is_empty() || m.origins.iter().any(|o| o.trim().is_empty()) { + return Err( + "--origin is required and must be non-empty: the relay compares a challenge's \ + origin against configuration, never against the Host header, and a relay that \ + cannot state its own hostname cannot close the cross-relay replay (§11.1)" + .into(), + ); + } + let factory = crate::batch::swap_factory_for_chain(m.chain_id).ok_or_else(|| { + format!( + "no vetted SimpleSwapFactory for chain {}: a factory address must never come \ + from the wire, so metered mode cannot run here (§6)", + m.chain_id + ) + })?; + if m.beneficiary == [0u8; 20] { + return Err("--beneficiary must be set: it is the EOA cheques are made out to".into()); + } + std::fs::create_dir_all(&m.state_dir) + .map_err(|e| format!("--state-dir {}: {e}", m.state_dir.display()))?; + let ledger = + crate::ledger::Ledger::load_or_create(m.state_dir.join("ledger.json")).map_err(|e| { + format!( + "ledger at {}: {e} — metered mode requires durable state, because losing \ + last_cumulative turns one signature into unlimited free service (§11.4)", + m.state_dir.display() + ) + })?; + info!( + "metered mode: origin(s) {} beneficiary 0x{} chain {} price {} PLUR/KiB ({} mode)", + m.origins.join(","), + hex::encode(m.beneficiary), + m.chain_id, + m.params.price_plur_per_kib, + if m.hard_mode { "hard" } else { "soft" }, + ); + Ok(Some(crate::metered::Metered::new( + crate::metered::MeterConfig { + origins: m.origins.clone(), + beneficiary: m.beneficiary, + chain_id: m.chain_id, + factory, + params: m.params, + hard_mode: m.hard_mode, + }, + ledger, + ))) +} + +/// `GET /v1/challenge?account=&batch=` — issue a capability (§7.2). +/// +/// This is where the chain reads happen: standing is resolved once, priced +/// into a credit line, and baked into the nonce, so `/v1/push` admission +/// touches no chain state at all. Amplification is bounded by *distinct +/// batch ids* rather than request count, because the owner cache answers +/// repeats — plus a per-IP limit on top. +async fn challenge_response( + state: Arc, + query: Option<&str>, + peer_ip: &str, +) -> Response { + let Some(m) = state.metered.as_ref() else { + return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); + }; + if !m.allow_challenge(peer_ip) { + return json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down"); + } + let q = parse_query(query); + let (Some(account_hex), Some(batch_hex)) = (q.get("account"), q.get("batch")) else { + return json_line_response(StatusCode::BAD_REQUEST, "need account= and batch="); + }; + let account = match parse_hex_array::<20>(account_hex) { + Ok(a) => a, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &format!("account: {e}")), + }; + let batch = match parse_hex_array::<32>(batch_hex) { + Ok(b) => b, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &format!("batch: {e}")), + }; + let batch_id_hex = hex::encode(batch); + let mut budget = 1usize; + let (owner, remaining_value) = match resolve_owner(&state, &batch_id_hex, &mut budget).await { + Ok(v) => v, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e), + }; + // No nonce for a batch this account does not own. Without this, + // admission would grant a reservation to any EOA that can sign, and + // free identities could occupy ledger entries without owning anything. + if owner != account { + return json_line_response( + StatusCode::FORBIDDEN, + "account is not the on-chain owner of that batch", + ); + } + let origin = m.cfg.origins.first().cloned().unwrap_or_default(); + match m.issue( + account, + batch, + remaining_value, + &origin, + crate::challenge::now_unix(), + ) { + Ok(issued) => json_response(StatusCode::OK, &issued.to_json()), + Err(e) => json_line_response(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +/// `POST /v1/pay` — accept a cheque (§11.6). +/// +/// Ordered cheapest-first, but the *bound* comes from the two free refusals +/// at the top: a challenge is required, and an account with no debt cannot +/// spend a single `eth_call`. Without those, every "free" check passes for +/// a cheque an attacker synthesizes at zero cost and each garbage POST buys +/// one `deployedContracts` call. +async fn pay_response( + state: Arc, + req: Request, +) -> Response { + let Some(m) = state.metered.as_ref() else { + return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); + }; + let header = req + .headers() + .get(crate::challenge::CHALLENGE_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string(); + let verified = match m.verify_header(&header, crate::challenge::now_unix()) { + Ok(v) => v, + Err(e) => return json_line_response(StatusCode::UNAUTHORIZED, &e), + }; + if !m.allow_account(&verified.account) { + return json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down"); + } + // No debt, no cheque — before parsing anything. Postpaid means an honest + // client always has debt by the time it settles, so this costs nothing + // legitimate and makes the endpoint useless to anyone who has not first + // done billable work. + let owed = m + .ledger + .lock() + .expect("ledger poisoned") + .owed(&verified.account); + if owed == 0 { + return json_line_response(StatusCode::BAD_REQUEST, "nothing owed on this account"); + } + + let body = match read_body_limited(req, crate::protocols::swap::MAX_CHEQUE_JSON).await { + Ok(b) => b, + Err(resp) => return resp, + }; + let cheque = match crate::protocols::swap::decode_signed_cheque_json(&body) { + Ok(c) => c, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), + }; + if cheque.beneficiary != m.cfg.beneficiary { + return json_line_response(StatusCode::BAD_REQUEST, "cheque is not made out to us"); + } + let cumulative: u128 = match u128::try_from(cheque.cumulative_payout) { + Ok(v) if v <= crate::ledger::MAX_CUMULATIVE_PLUR => v, + _ => { + return json_line_response(StatusCode::BAD_REQUEST, "cumulative payout is implausible"); + } + }; + let have = m + .ledger + .lock() + .expect("ledger poisoned") + .last_cumulative(&verified.account, &cheque.chequebook); + if cumulative <= have { + return json_line_response( + StatusCode::BAD_REQUEST, + &format!("cheque cumulative {cumulative} does not exceed the {have} already accepted"), + ); + } + // Against the floor this account can actually reach, not the configured + // one: a batch whose credit line is below `min_cheque_plur` would + // otherwise be refused for a cheque it is structurally incapable of + // writing (§10.1, `Params::effective`). + let floor = m.cfg.params.effective(verified.cap_plur).min_cheque_plur; + if cumulative - have < floor { + return json_line_response( + StatusCode::BAD_REQUEST, + &format!( + "cheque credits {} but the dust floor is {floor}", + cumulative - have, + ), + ); + } + // Every free check has passed; only now does this cost RPC. + match m.is_deployed(&state.opts.rpc_url, cheque.chequebook).await { + Ok(true) => {} + Ok(false) => { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook was not deployed by the canonical factory", + ); + } + Err(e) => return json_line_response(StatusCode::BAD_GATEWAY, &e), + } + let issuer_ok = crate::signer::recover_cheque_issuer( + &cheque.chequebook, + &cheque.beneficiary, + cheque.cumulative_payout, + m.cfg.chain_id, + &cheque.signature, + ); + let recovered = match issuer_ok { + Ok(a) => a, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), + }; + let cb_state = match m + .chequebook_state(&state.opts.rpc_url, cheque.chequebook) + .await + { + Ok(s) => s, + Err(e) => return json_line_response(StatusCode::BAD_GATEWAY, &e), + }; + if cb_state.bounced { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook has bounced a cheque before and is refused", + ); + } + if cb_state.issuer.into_array() != recovered { + return json_line_response( + StatusCode::BAD_REQUEST, + "cheque was not signed by the issuer", + ); + } + if cb_state.issuer.into_array() != verified.account { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook issuer is not the account that owns this batch", + ); + } + // The funding check, against `liquidBalanceFor(us)` rather than + // `balance()` — the latter counts other beneficiaries' hard deposits as + // our coverage, which is unsound (§11.2). + let paid_out = u128::try_from(cb_state.paid_out_to_us).unwrap_or(u128::MAX); + let liquid = u128::try_from(cb_state.liquid_for_us).unwrap_or(u128::MAX); + if liquid < cumulative.saturating_sub(paid_out) { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook cannot cover this cheque", + ); + } + match m.credit( + verified.account, + cheque.chequebook, + cumulative, + cheque.signature, + ) { + Ok(accepted) => { + // We just consumed part of what that balance covered; the next + // cheque should not be judged against the pre-credit reading. + m.invalidate_chequebook(&cheque.chequebook); + let l = m.ledger.lock().expect("ledger poisoned"); + json_response( + StatusCode::OK, + &serde_json::json!({ + "accepted_plur": accepted.to_string(), + "cumulative": cumulative.to_string(), + "owed_plur": l.owed(&verified.account).to_string(), + "outstanding_plur": l.outstanding(&verified.account).to_string(), + }), + ) + } + Err(e) => json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), + } +} + +/// `GET /v1/account` — the client's own ledger row (§7). +/// +/// Authenticated with the challenge header: unauthenticated it is a +/// per-identity volume oracle over on-chain-enumerable batch owners, and a +/// targeting oracle for tipping a victim into 402 at a chosen moment. +fn account_response(state: &State, headers: &hyper::HeaderMap) -> Response { + let Some(m) = state.metered.as_ref() else { + return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); + }; + let raw = headers + .get(crate::challenge::CHALLENGE_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + let verified = match m.verify_header(raw, crate::challenge::now_unix()) { + Ok(v) => v, + Err(e) => return json_line_response(StatusCode::UNAUTHORIZED, &e), + }; + let l = m.ledger.lock().expect("ledger poisoned"); + json_response( + StatusCode::OK, + &serde_json::json!({ + "account": format!("0x{}", hex::encode(verified.account)), + "owed_plur": l.owed(&verified.account).to_string(), + "reserved_plur": l.reserved(&verified.account).to_string(), + "outstanding_plur": l.outstanding(&verified.account).to_string(), + "max_outstanding_plur": verified.cap_plur.to_string(), + "settle_every_plur": m.cfg.params.settle_every_plur.to_string(), + "min_cheque_plur": m.cfg.params.min_cheque_plur.to_string(), + }), + ) +} + +/// Read a request body with a hard cap, so an oversized one costs nothing. +async fn read_body_limited( + req: Request, + max: usize, +) -> Result> { + use http_body_util::BodyExt; + use hyper::body::Body as _; + use std::time::Duration; + // Cheap rejection when the client declares an oversize body up front. + // This is an optimisation, NOT the bound: `size_hint().upper()` is + // `None` for a chunked body (and for HTTP/2, where length is unknown + // until END_STREAM), so a client that omits `Content-Length` skips it + // entirely. + if let Some(len) = req.body().size_hint().upper() + && len > max as u64 + { + return Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body too large", + )); + } + // The real bound. `Limited` enforces the cap *inside* `poll_frame`, so + // memory is held to `max` plus one frame. A bare `.collect()` would + // accumulate whatever the client streams until the timeout fires — + // ~30 s of link bandwidth per connection, times the connection cap — + // and only notice afterwards, which is no bound at all. + match tokio::time::timeout( + Duration::from_secs(HEADER_READ_TIMEOUT_SECS), + Limited::new(req.into_body(), max).collect(), + ) + .await + { + Ok(Ok(c)) => { + let b = c.to_bytes(); + if b.len() > max { + return Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body too large", + )); + } + Ok(b) + } + Ok(Err(_)) => Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body exceeds limit or read error", + )), + Err(_) => Err(json_line_response( + StatusCode::REQUEST_TIMEOUT, + "body read timed out", + )), + } +} + +fn parse_hex_array(s: &str) -> Result<[u8; N], String> { + let raw = hex::decode(s.trim_start_matches("0x")).map_err(|e| e.to_string())?; + if raw.len() != N { + return Err(format!("must be {N} bytes, got {}", raw.len())); + } + let mut out = [0u8; N]; + out.copy_from_slice(&raw); + Ok(out) +} + +/// Body of a maximal POST, for the credit-line comparison in `src/meter.rs`: +/// a batch whose line is under this has to split its uploads across smaller +/// requests (incentives §7.2). +fn full_post_kib() -> u64 { + (PUSH_BATCH_MAX * pushframe::MAX_FRAME_LEN).div_ceil(1024) as u64 +} + +/// Completed per-stream push outcomes since boot. +/// +/// Incentives §9.1 reads this over frames admitted as an egress multiplier, +/// and **it undercounts**: the counters are bumped after the await inside +/// the racing future (`src/client.rs:5818-5870`), and the dispatcher +/// cancels the losing racers as soon as it takes a receipt +/// (`src/client.rs:4806-4809`). A cancelled racer has already put its +/// Delivery on the wire — the relay pays that egress — but never reaches +/// the increment. Shallow retries and errors are counted; concurrent +/// losers are not, so the ratio floors near 1.0 whenever the race is won +/// promptly. Counting at dispatch, beside `inflight_pushes`, would fix it. +fn stream_attempts() -> u64 { + use crate::transport::diag; + use std::sync::atomic::Ordering; + diag::PUSH_OUTCOME_OK.load(Ordering::Relaxed) + + diag::PUSH_OUTCOME_SHALLOW.load(Ordering::Relaxed) + + diag::PUSH_OUTCOME_OVERDRAFT.load(Ordering::Relaxed) + + diag::PUSH_OUTCOME_ERROR.load(Ordering::Relaxed) +} + +/// `GET /v1/meter` — Stage 0 shadow-metering detail (incentives §14). +/// +/// **Open by default; set `HOVERFLY_PUSH_METER_TOKEN` to require a bearer +/// token.** Stage 0's rows are derived from state that is already public: +/// batch owners and their balances are on-chain and enumerable from +/// `BatchCreated`, and the stamp on every relayed chunk names its batch, +/// which retrieval hands back (incentives §2). All this endpoint adds is +/// relay attribution and timing, so gating it by default buys little and +/// costs a lot — an instrument nobody reads answers no questions, and +/// deciding whether to meter at all is the only reason Stage 0 exists. +/// +/// That flips at **Stage 1**, and the reason is worth recording because it +/// is not the obvious one. Once 402s are live, `/v1/account` exposes an +/// account's *outstanding* balance, which lets a reader time a stamp replay +/// (§11.1) to tip a victim over its cap mid-upload. That is an active +/// attack enabler rather than a privacy leak, and it is the real reason +/// incentives §7 authenticates that endpoint. +fn meter_response(state: &State, headers: &hyper::HeaderMap) -> Response { + if let Ok(want) = std::env::var("HOVERFLY_PUSH_METER_TOKEN") { + let got = headers + .get(hyper::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .unwrap_or_default(); + // Compare the *contents* in constant time — a byte-wise early exit + // would let the token be ground out one character at a time. Length + // is compared directly and so is observable, which is fine: it is a + // static property of the operator's config, not something an + // attacker can narrow down to a value. + let ok = got.len() == want.len() + && got + .bytes() + .zip(want.bytes()) + .fold(0u8, |acc, (a, b)| acc | (a ^ b)) + == 0; + if !ok { + return json_line_response(StatusCode::UNAUTHORIZED, "unauthorized"); + } + } + let body = + state + .meter + .lock() + .expect("meter poisoned") + .detail(full_post_kib(), stream_attempts(), 100); + json_response(StatusCode::OK, &body) +} + /// Add permissive CORS headers to a response. fn add_cors(h: &mut hyper::HeaderMap) { use hyper::header::HeaderValue; @@ -333,7 +1186,7 @@ fn cors_preflight() -> Response { ); h.insert( "access-control-allow-headers", - HeaderValue::from_static("content-type"), + HeaderValue::from_static("content-type, x-hoverfly-challenge"), ); h.insert("access-control-max-age", HeaderValue::from_static("86400")); resp @@ -377,6 +1230,20 @@ fn status_response(state: &State) -> Response { // log-only, so a deployed relay can be inspected without shell // access to it. "diag": diag::summary(), + // Stage 0 shadow metering (incentives §14): what a metered relay + // *would* have billed. Nothing is charged and no client behaviour + // changes. Aggregates only — per-account detail is behind + // /v1/meter, since it names identities (see `meter_response`). + // Metered-mode quote (§7.3), signed with the node-identity key so a + // price is not repudiable in either direction. Absent in open mode. + "payment": payment_quote(state), + "meter": state.push.as_ref().map(|_| { + state + .meter + .lock() + .expect("meter poisoned") + .summary(full_post_kib(), stream_attempts()) + }), }); json_response(StatusCode::OK, &body) } @@ -592,6 +1459,7 @@ fn build_push_state(opts: &PusherOpts) -> Option { }; let keypair = crate::inbound::libp2p_keypair_from_identity(&node_signer); let overlay = *node_signer.overlay(); + let quote_signer = node_signer.clone(); let snapshot = crate::protocols::status::StatusSnapshot::default(); let transport = Transport::new_with_keypair(node_signer, opts.transport.clone(), keypair) .with_status_snapshot(snapshot); @@ -619,6 +1487,7 @@ fn build_push_state(opts: &PusherOpts) -> Option { pool: tokio::sync::Mutex::new(None), pool_target, pool_live: AtomicUsize::new(0), + signer: quote_signer, overlay, budget_gb, bytes_pushed: AtomicU64::new(0), @@ -646,15 +1515,33 @@ async fn push_response( "push disabled (no node identity resolvable)", ); } - // Bounded body read — a whole batch, not a stream. - let bytes = match Limited::new(req.into_body(), PUSH_MAX_BODY).collect().await { - Ok(c) => c.to_bytes(), - Err(_) => { + // Metered admission, entirely before the body is read (§7.2). Nothing + // here touches the chain: the challenge already carries the credit line, + // which is the point of issuing it. + let admitted = match admit_metered(&state, &req) { + Ok(a) => a, + Err(resp) => return *resp, + }; + // Bounded body read — a whole batch, not a stream. Bounded in *time* as + // well as size: the size limit alone let a client dribble a body forever + // and hold the connection (and, under metering, its admission + // reservation) for free. + let read = tokio::time::timeout( + std::time::Duration::from_secs(PUSH_BODY_READ_TIMEOUT_SECS), + Limited::new(req.into_body(), PUSH_MAX_BODY).collect(), + ) + .await; + let bytes = match read { + Ok(Ok(c)) => c.to_bytes(), + Ok(Err(_)) => { return json_line_response( StatusCode::PAYLOAD_TOO_LARGE, "body exceeds limit or read error", ); } + Err(_) => { + return json_line_response(StatusCode::REQUEST_TIMEOUT, "body read timed out"); + } }; let chunks = match pushframe::decode_batch(&bytes, PUSH_BATCH_MAX) { Ok(c) => c, @@ -673,7 +1560,7 @@ async fn push_response( // sessions from it (maintain=false), so no per-push dial burst. let (tx, rx) = futures::channel::mpsc::unbounded::, Infallible>>(); tokio::spawn(async move { - run_push(state, chunks, tx).await; + run_push(state, chunks, tx, admitted).await; }); Response::builder() @@ -689,6 +1576,7 @@ async fn run_push( state: Arc, chunks: Vec, tx: futures::channel::mpsc::UnboundedSender, Infallible>>, + admitted: Option, ) { let push = state.push.as_ref().expect("push state present"); let mut dedup_hits = 0usize; @@ -704,31 +1592,37 @@ async fn run_push( } send_line(&v); }; - let ack_ok = |addr: &[u8; 32], info: crate::client::PushInfo| { - // `po` is the proximity order of the peer whose receipt we took — - // i.e. how deep into the chunk's own neighborhood it actually - // landed. This is the measurement that decides whether client-side - // proximity routing to a relay's overlay is worth anything at all - // (docs/pusher-design.md §7); without it that question can only be - // guessed at. - let mut v = serde_json::json!({ - "a": hex::encode(addr), - "s": "ok", - "po": info.po, - "ms": info.ms, - }); - if info.shallow { - v["shallow"] = serde_json::Value::Bool(true); - } - send_line(&v); + let ack_dedup = |addr: &[u8; 32]| { + // A dedup hit did no push work and is billed at zero (§8.2). Say so + // explicitly: without a marker it is indistinguishable from a real + // push, so a paying client counts bytes the relay never charged + // for, and its next cheque is refused as an overpayment. The claim + // only ever *lowers* what is owed, so it is safe for the client to + // take at face value. + send_line(&serde_json::json!({ + "a": hex::encode(addr), "s": "ok", "po": 0, "ms": 0, "dedup": true + })); }; // The stamp's batch_id must match the on-chain owner the signature // recovers to. All chunks in one upload share a batch; verify the // batch once, then check each chunk's recovered signer against it. let mut accepted: Vec = Vec::with_capacity(chunks.len()); - let mut batch_owner: Option<[u8; 20]> = None; + let mut batch_owner: Option<([u8; 20], u128)> = None; let mut batch_hex: Option = None; + // Stage 0 shadow metering, accumulated on this task's stack and merged + // once below. Per-frame locking would serialize concurrent POSTs on a + // counter nobody is billed from. + let mut tally = crate::meter::PostTally::default(); + // Body bytes that will actually be billed: admitted minus dedup hits, + // which did no push work and so cost nothing (§8.2). + let mut billable_bytes: u64 = 0; + // addr -> owning batch, for the recent-ack cache: dedup must be + // scoped per (addr, batch) so one uploader's frame can't be acked + // "ok" under another uploader's stamp (§15). + let mut batch_of: HashMap<[u8; 32], [u8; 32]> = HashMap::with_capacity(chunks.len()); + // Distinct on-chain batch resolutions this request may perform. + let mut rpc_budget = PUSH_MAX_BATCH_LOOKUPS; for chunk in chunks { let vs = match crate::stamp::validate(&chunk.addr, &chunk.stamp) { @@ -743,7 +1637,7 @@ async fn run_push( let owner = if batch_hex.as_deref() == Some(bid.as_str()) { batch_owner } else { - match resolve_owner(&state, &bid).await { + match resolve_owner(&state, &bid, &mut rpc_budget).await { Ok(o) => { batch_hex = Some(bid.clone()); batch_owner = Some(o); @@ -756,32 +1650,58 @@ async fn run_push( } }; match owner { - Some(o) if o == vs.signer => { + Some((o, batch_value)) if o == vs.signer => { // Duplicate suppression: a client hedging a straggler // sends the same frame to two lanes on purpose. Answering // from the recent-ack cache makes the loser of that race // free instead of a second real push through the pool. + // Scoped per (addr, batch) — a bare address would let the + // hit discard the submitted stamp (§15). + let mut batch_id = [0u8; 32]; + batch_id.copy_from_slice(vs.batch_id); + // One batch per request under metering (§6). Standing, the + // credit line and the reservation are all properties of a + // *batch*, so a POST that mixes them lets one good-standing + // frame carry 511 others from an account that is over its + // cap (§11.8). Rejected rather than billed to whoever it + // names. + if let Some(adm) = &admitted + && batch_id != adm.batch + { + ack( + &chunk.addr, + "err", + Some("frame batch does not match the challenge (one batch per request)"), + ); + continue; + } + // Body bytes this frame occupied — header plus wire — which + // is the unit incentives §8 bills. Recorded for every + // admitted frame, dedup hits included, because the relay + // received those bytes either way. + let key = crate::meter::AccountBatch { + owner: o, + batch: batch_id, + }; + let frame_bytes = (pushframe::HEADER_LEN + chunk.wire.len()) as u64; + tally.admit(key, batch_value, frame_bytes); + billable_bytes += frame_bytes; let dup = push .recent .lock() .expect("recent-ack cache poisoned") - .contains(&chunk.addr); + .contains(&chunk.addr, batch_id); if dup { dedup_hits += 1; - ack_ok( - &chunk.addr, - crate::client::PushInfo { - po: 0, - ms: 0, - shallow: false, - best_po: 0, - }, - ); + tally.dedup(key, batch_value, frame_bytes); + billable_bytes = billable_bytes.saturating_sub(frame_bytes); + ack_dedup(&chunk.addr); } else { + batch_of.insert(chunk.addr, batch_id); accepted.push(chunk); } } - Some(o) => ack( + Some((o, _)) => ack( &chunk.addr, "err", Some(&format!( @@ -794,6 +1714,25 @@ async fn run_push( } } + // Turn the reservation into debt for what was actually admitted, and + // release the rest (§10.2). Runs before the early return below, so a + // POST that admitted nothing still gives its reservation back — leaking + // it would ratchet the account toward a 402 it can never clear. + if let Some(adm) = admitted { + adm.commit(billable_bytes); + } + + // One lock for the whole request. Runs before the early return below so + // an all-dedup POST is still measured — those are exactly the requests + // §8.2 bills at zero, and their share is a number Stage 0 wants. + if !tally.is_empty() { + state + .meter + .lock() + .expect("meter poisoned") + .merge(std::mem::take(&mut tally)); + } + if accepted.is_empty() { send_line(&serde_json::json!({ "done": {"pushed": 0, "rejected": dedup_hits == 0, "dedup": dedup_hits} @@ -833,6 +1772,13 @@ async fn run_push( Arc::new(move |addr: &[u8; 32], res| { let v = match &res { Ok(info) => { + // `po` is the proximity order of the peer whose receipt + // we took — how deep into the chunk's own neighborhood + // it actually landed. This is the measurement that + // decides whether client-side proximity routing to a + // relay's overlay is worth anything at all + // (docs/pusher-design.md §7); without it that question + // can only be guessed at. let mut v = serde_json::json!({ "a": hex::encode(addr), "s": "ok", "po": info.po, "ms": info.ms, // Best proximity the dispatcher could reach for this @@ -880,10 +1826,15 @@ async fn run_push( for (a, ok) in &seen { if *ok { pushed += 1; + // A successful chunk belongs to exactly the batch it was + // admitted under; cache it under (addr, batch) so dedup stays + // scoped. `batch_of` is only missing a key if the chunk was + // admitted on a path that never mapped it — none currently. + let bid = batch_of.get(a).copied().unwrap_or([0u8; 32]); push.recent .lock() .expect("recent-ack cache poisoned") - .insert(*a); + .insert(*a, bid); } } if total > 0 { @@ -919,43 +1870,93 @@ async fn run_push( /// is "the batch is alive" (docs/pusher-design.md §5), so a batch whose /// `remainingBalance` has drained to zero is rejected — bee nodes would /// refuse its stamps anyway, and pushing them just burns relay egress. -/// The aliveness read happens once per batch (the cache never expires); -/// a batch that dies *while cached* only wastes its own push attempts — -/// bees reject the stamps downstream — and a pusher restart re-checks. -async fn resolve_owner(state: &State, batch_id_hex: &str) -> Result<[u8; 20], String> { - if let Some(o) = state +/// The aliveness read happens once per batch and is cached for +/// `OWNER_OK_TTL_SECS`; a batch that dies *while cached* only wastes its +/// own push attempts — bees reject the stamps downstream. +/// `rpc_budget` bounds how many *distinct* batch ids one request may push +/// through to the chain. A cache hit never spends it; only a genuine miss +/// does. Without it, negative caching alone still leaves a single POST +/// naming `PUSH_BATCH_MAX` different bogus batch ids able to issue that +/// many serial `eth_call`s, since every one of them is a first miss. +async fn resolve_owner( + state: &State, + batch_id_hex: &str, + rpc_budget: &mut usize, +) -> Result<([u8; 20], u128), String> { + if let Some(hit) = state .owner_cache .lock() .expect("owner cache poisoned") .get(batch_id_hex) { - return Ok(*o); + return match hit { + OwnerLookup::Owner(o, value) => Ok((o, value)), + OwnerLookup::Rejected(why) => Err(why), + }; + } + if *rpc_budget == 0 { + return Err(format!( + "batch {batch_id_hex}: too many distinct batches in one request \ + (limit {PUSH_MAX_BATCH_LOOKUPS}); split them across requests" + )); } + *rpc_budget -= 1; + + // Only *definitive* on-chain answers are cached. A transport error must + // not blacklist a live batch for OWNER_BAD_TTL_SECS, so those propagate + // uncached. let stamp_addr: alloy_primitives::Address = crate::batch::MAINNET_POSTAGE_STAMP .parse() .expect("hardcoded valid"); let info = crate::batch::read_batch(&state.opts.rpc_url, stamp_addr, batch_id_hex) .await .map_err(|e| format!("batch owner RPC: {e}"))?; + let reject = |state: &State, why: String| -> String { + state + .owner_cache + .lock() + .expect("owner cache poisoned") + .insert(batch_id_hex, OwnerLookup::Rejected(why.clone())); + why + }; if info.not_found { - return Err(format!("batch {batch_id_hex} not found on-chain")); + return Err(reject( + state, + format!("batch {batch_id_hex} not found on-chain"), + )); } let remaining = crate::batch::read_remaining_balance(&state.opts.rpc_url, stamp_addr, batch_id_hex) .await .map_err(|e| format!("batch balance RPC: {e}"))?; if remaining.is_zero() { - return Err(format!( - "batch {batch_id_hex} has expired (zero remaining balance) — bees would reject every stamp" + return Err(reject( + state, + format!( + "batch {batch_id_hex} has expired (zero remaining balance) — bees would reject every stamp" + ), )); } + // Total value still funded on this batch: `remainingBalance` is PLUR per + // chunk, and a batch of depth `d` covers 2^d chunks (incentives §6). + // Saturating rather than wrapping — a nonsense depth from a malformed + // read must not produce a small number that looks like a real answer. + let depth_factor = 1u128 + .checked_shl(u32::from(info.depth)) + .unwrap_or(u128::MAX); + let remaining_value_plur = u128::try_from(remaining) + .unwrap_or(u128::MAX) + .saturating_mul(depth_factor); let owner = info.owner.into_array(); state .owner_cache .lock() .expect("owner cache poisoned") - .insert(batch_id_hex.to_string(), owner); - Ok(owner) + .insert( + batch_id_hex, + OwnerLookup::Owner(owner, remaining_value_plur), + ); + Ok((owner, remaining_value_plur)) } /// Return the warm pool, filling/topping it up to `push.pool_target`. @@ -1409,3 +2410,130 @@ fn json_response(status: StatusCode, body: &serde_json::Value) -> Response Response { json_response(status, &serde_json::json!({"error": message})) } + +/// The signed `payment` block for `/v1/status` (incentives §7.3). +/// +/// An unsigned price is repudiable in both directions: the relay can serve +/// `P` and bill `10P`, the client can claim it saw `P/10`, and +/// reconciliation can detect the mismatch but never attribute it. +/// +/// It carries `node_eth_address` and `overlay_nonce` because +/// "pin `(url, overlay)`" is not implementable — an overlay is +/// `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`, so verifying a signature +/// yields the *eth address* while the nonce is neither transmitted nor +/// derivable. With both present a client can recompute the overlay and +/// check it against what the relay advertises, and pin the triple +/// `(url, node_eth_address, beneficiary)`. +fn payment_quote(state: &State) -> Option { + let m = state.metered.as_ref()?; + let push = state.push.as_ref()?; + let p = &m.cfg.params; + let mut body = serde_json::json!({ + "mode": "metered", + "enforcement": if m.cfg.hard_mode { "hard" } else { "soft" }, + "beneficiary": format!("0x{}", hex::encode(m.cfg.beneficiary)), + "node_eth_address": format!("0x{}", hex::encode(push.signer.eth_address())), + "overlay_nonce": format!("0x{}", hex::encode(state.opts.nonce)), + "origin": m.cfg.origins.first().cloned().unwrap_or_default(), + "chain_id": m.cfg.chain_id, + "factory": format!("0x{}", hex::encode(m.cfg.factory)), + "price_plur_per_kib": p.price_plur_per_kib.to_string(), + "min_cheque_plur": p.min_cheque_plur.to_string(), + "settle_every_plur": p.settle_every_plur.to_string(), + "max_outstanding_plur": p.max_outstanding_plur.to_string(), + "credit_ratio": p.credit_ratio, + "challenge_ttl_secs": crate::challenge::CHALLENGE_TTL_SECS, + }); + // Sign the canonical serialization of the block itself, so what the + // client verifies is exactly what it read. + let payload = body.to_string(); + match push.signer.sign_eip191(payload.as_bytes()) { + Ok(sig) => { + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + Some(body) + } + Err(e) => { + tracing::error!("cannot sign payment quote: {e}"); + None + } + } +} + +#[cfg(test)] +mod owner_cache_tests { + use super::*; + + fn owner_of(c: &OwnerCache, k: &str) -> Option<[u8; 20]> { + match c.get(k) { + Some(OwnerLookup::Owner(o, _)) => Some(o), + _ => None, + } + } + + fn value_of(c: &OwnerCache, k: &str) -> Option { + match c.get(k) { + Some(OwnerLookup::Owner(_, v)) => Some(v), + _ => None, + } + } + + #[test] + fn rejections_are_cached_so_a_bogus_batch_is_not_re_resolved() { + let mut c = OwnerCache::new(16); + assert!(c.get("deadbeef").is_none(), "cold cache must miss"); + c.insert("deadbeef", OwnerLookup::Rejected("not found".into())); + // The point of the fix: a second frame naming the same bogus batch + // finds a cached rejection instead of issuing another eth_call. + assert!( + matches!(c.get("deadbeef"), Some(OwnerLookup::Rejected(w)) if w == "not found"), + "rejection must be cached" + ); + } + + /// The batch's remaining value rides along on the cached success so + /// Stage 0 can price a credit line without a second `eth_call` + /// (`src/meter.rs`). A cache that dropped it would silently make every + /// batch look unpriced. + #[test] + fn the_cached_success_carries_the_batch_value() { + let mut c = OwnerCache::new(16); + c.insert("b0", OwnerLookup::Owner([7u8; 20], 100_000_000_000_000)); + assert_eq!(owner_of(&c, "b0"), Some([7u8; 20])); + assert_eq!(value_of(&c, "b0"), Some(100_000_000_000_000)); + } + + #[test] + fn eviction_is_bounded_by_cap() { + let mut c = OwnerCache::new(4); + for i in 0..64 { + c.insert(&format!("batch{i}"), OwnerLookup::Owner([i as u8; 20], 0)); + } + assert_eq!(c.map.len(), 4, "map must stay at cap"); + assert_eq!(c.order.len(), 4, "order must stay at cap"); + assert!(owner_of(&c, "batch0").is_none(), "oldest evicted"); + assert_eq!(owner_of(&c, "batch63"), Some([63u8; 20]), "newest retained"); + } + + #[test] + fn reinserting_a_key_does_not_grow_the_order_queue() { + let mut c = OwnerCache::new(8); + for _ in 0..32 { + c.insert("same", OwnerLookup::Owner([1u8; 20], 0)); + } + assert_eq!(c.order.len(), 1, "one order slot per distinct key"); + assert_eq!(owner_of(&c, "same"), Some([1u8; 20])); + } + + #[test] + fn entries_expire() { + // Zero-length TTLs are not reachable through the constants, so drive + // expiry by backdating the insert instant directly. + let mut c = OwnerCache::new(8); + c.insert("stale", OwnerLookup::Owner([2u8; 20], 0)); + let aged = Instant::now() + .checked_sub(std::time::Duration::from_secs(OWNER_OK_TTL_SECS + 1)) + .expect("clock supports backdating"); + c.map.get_mut("stale").expect("present").1 = aged; + assert!(c.get("stale").is_none(), "expired entry must not be served"); + } +} diff --git a/src/pushframe.rs b/src/pushframe.rs index 06e74e1..e100433 100644 --- a/src/pushframe.rs +++ b/src/pushframe.rs @@ -22,7 +22,11 @@ pub const STAMP_LEN: usize = 113; /// Max wire length: span(8) + max chunk data(4096). pub const MAX_WIRE_LEN: usize = 4104; /// Fixed frame header before the variable wire: addr + stamp + len(u16). -const HEADER_LEN: usize = 32 + STAMP_LEN + 2; +/// Public because it is the smallest a frame can be, which is what bounds +/// how many frames a body of a given `Content-Length` can hold — the basis +/// of the metered reservation (`docs/pusher-incentives.md` §7.2) and of +/// Stage 0's per-frame byte accounting. +pub const HEADER_LEN: usize = 32 + STAMP_LEN + 2; /// Upper bound on a single encoded frame. pub const MAX_FRAME_LEN: usize = HEADER_LEN + MAX_WIRE_LEN; diff --git a/src/pushsched.rs b/src/pushsched.rs index 4d7ae9d..36a5c06 100644 --- a/src/pushsched.rs +++ b/src/pushsched.rs @@ -83,6 +83,10 @@ pub enum LaneHealth { /// 35 s to first byte on a sleeping instance), and a cold lane must not /// be able to swallow a full-size batch before proving it is awake. Warming, + /// Over its credit line, or the client has no way to pay it. Ineligible + /// but **not terminal** — unlike `Retired`, which is permanent for the + /// run. Cleared by [`Scheduler::fund_lane`] once a cheque is accepted. + Unfunded, /// Serving normally. Live, /// Failing; not eligible until `until_ms`. @@ -109,6 +113,23 @@ pub struct LaneInfo { pub budget_remaining_gb: Option, /// Live warm sessions; the strongest available prior on throughput. pub pool_live: Option, + /// Price in PLUR per KiB of body, from a *verified* signed quote + /// (`docs/pusher-incentives.md` §7.3). `None` on an `open` lane, and + /// also `None` when the quote failed verification — an unverifiable + /// quote is treated as "not metered" rather than "free", so the lane is + /// simply not paid and not scheduled for payment. + pub price_plur_per_kib: Option, + /// True when the lane enforces 402 rather than metering softly. + /// + /// Deliberately *not* behind the `pusher` feature: a client that cannot + /// pay still has to recognise a lane that will refuse it, and the + /// browser build compiles without the payment stack entirely. + pub hard_enforcement: bool, + /// The whole verified quote, when this lane advertised one. Carried so + /// the payment loop has the beneficiary and parameters without + /// re-fetching and re-verifying `/v1/status`. + #[cfg(not(target_arch = "wasm32"))] + pub quote: Option, } /// Tunables. Defaults are the shipping configuration. @@ -229,6 +250,8 @@ impl Lane { match self.health { LaneHealth::Live | LaneHealth::Warming => true, LaneHealth::Backoff { until_ms } => now_ms >= until_ms, + // Ineligible until paid, but recoverable within this run. + LaneHealth::Unfunded => false, LaneHealth::Retired => false, } } @@ -307,6 +330,15 @@ pub enum BatchOutcome { Answered, /// Transport error, non-2xx, or an unparseable body. Failed(String), + /// `402 Payment Required` — the account is over its credit line on this + /// lane (`docs/pusher-incentives.md` §12). + /// + /// **Not a failure.** Mapping it to `Failed` would charge lane health + /// for a *routine settlement*: five of them retire a perfectly healthy + /// lane mid-upload, which is the opposite of what should happen when a + /// relay says "pay me". The lane is paused until a cheque clears and its + /// streak is left untouched. + PaymentRequired, } /// Why a run couldn't finish. @@ -333,6 +365,7 @@ pub enum LaneHealthKind { Warming, Live, Backoff, + Unfunded, Retired, } @@ -726,6 +759,61 @@ impl Scheduler { .retain(|&k| self.chunks[k].phase == ChunkPhase::Pending); } + /// Bring a lane back after a cheque has been accepted. + /// + /// The counterpart to `PaymentRequired`: because that outcome left + /// `fail_streak` and `backoff_exp` alone, settling restores the lane to + /// exactly the health it had before it ran out of credit, rather than + /// making it re-warm. + pub fn fund_lane(&mut self, lane: usize) { + if let Some(l) = self.lanes.get_mut(lane) + && l.health == LaneHealth::Unfunded + { + l.health = if l.bytes_total > 0 { + LaneHealth::Live + } else { + LaneHealth::Warming + }; + } + } + + /// Take a lane out of rotation permanently. + /// + /// For lanes that are unusable by configuration rather than by + /// behaviour — a lane enforcing payment this client cannot make — where + /// there is nothing to discover by trying and every attempt costs the + /// chunks one of their retries. + pub fn retire_lane(&mut self, lane: usize) { + if let Some(l) = self.lanes.get_mut(lane) { + l.health = LaneHealth::Retired; + } + } + + /// Re-clamp a lane's frames-per-POST between dispatches. + /// + /// A metered lane's affordable body shrinks as debt and in-flight bytes + /// accumulate and grows back as cheques clear, so the ceiling set at + /// startup goes stale immediately. Sizing the assignment here is what + /// keeps the client from building a body the relay will refuse — §7.2 + /// wants the POST sized to fit rather than the ceiling discovered as a + /// 402. + pub fn set_lane_batch_max(&mut self, lane: usize, max: usize) { + if let Some(l) = self.lanes.get_mut(lane) { + l.info.batch_max = Some(max.max(1)); + } + } + + /// Lanes currently paused for payment, so the driver knows which ones a + /// cheque would unblock. + pub fn unfunded_lanes(&self) -> Vec { + self.lanes + .iter() + .enumerate() + .filter(|(_, l)| l.health == LaneHealth::Unfunded) + .map(|(i, _)| i) + .collect() + } + /// Record the HTTP-level result of a dispatch. Must be called exactly /// once per [`Assignment`], after all of that batch's acks. pub fn on_batch_result(&mut self, batch: u64, outcome: BatchOutcome, now_ms: u64) { @@ -746,6 +834,13 @@ impl Scheduler { l.backoff_exp = 0; l.health = LaneHealth::Live; } + // A 402 says the relay is willing and the client is behind on + // payment. Pause the lane without touching `fail_streak` or + // `backoff_exp`, so settling restores it instantly and a long + // upload is not punished for crossing a settlement window. + BatchOutcome::PaymentRequired => { + self.lanes[lane].health = LaneHealth::Unfunded; + } _ => { let l = &mut self.lanes[lane]; l.fail_streak += 1; @@ -962,6 +1057,7 @@ impl Scheduler { LaneHealth::Warming => LaneHealthKind::Warming, LaneHealth::Live => LaneHealthKind::Live, LaneHealth::Backoff { .. } => LaneHealthKind::Backoff, + LaneHealth::Unfunded => LaneHealthKind::Unfunded, LaneHealth::Retired => LaneHealthKind::Retired, }), }) diff --git a/src/pushsched/tests.rs b/src/pushsched/tests.rs index 7c9bb76..cd4dfac 100644 --- a/src/pushsched/tests.rs +++ b/src/pushsched/tests.rs @@ -643,3 +643,133 @@ fn all_acked_policy_ignores_groups() { assert_eq!(sched.acked(), 10, "AllAcked must not stop at the threshold"); assert_eq!(sched.skipped(), 0); } + +// ── Metered lanes (docs/pusher-incentives.md §12) ──────────────────────── + +fn two_lanes() -> Scheduler { + let infos = vec![LaneInfo::default(), LaneInfo::default()]; + let mut s = Scheduler::new(infos, Config::default()); + s.admit(addrs(64, 7)); + s +} + +/// Drive one batch and hand back `(batch, lane)`. The scheduler chooses the +/// lane by weight, so tests follow its choice rather than dictating one. +fn dispatch_one(s: &mut Scheduler, now_ms: u64) -> (u64, usize) { + let a = s + .next(now_ms) + .expect("a lane with pending work must produce an assignment"); + (a.batch, a.lane) +} + +/// The failure §12 exists to prevent: five routine settlements retiring a +/// perfectly healthy lane mid-upload. +#[test] +fn repeated_402s_never_retire_a_lane() { + let mut s = two_lanes(); + let mut now = 0u64; + for _ in 0..20 { + let (b, lane) = dispatch_one(&mut s, now); + s.on_batch_result(b, BatchOutcome::PaymentRequired, now); + // Settling is what un-pauses it; without that the lane stays out. + s.fund_lane(lane); + now += 1000; + } + for (i, st) in s.lane_stats().iter().enumerate() { + assert_ne!( + st.health.expect("health"), + LaneHealthKind::Retired, + "lane {i} kept asking to be paid and must never be retired" + ); + } +} + +/// A 402 must not touch the failure streak — otherwise a lane that has +/// been asking for payment is one transport error away from backoff. +#[test] +fn a_402_does_not_charge_lane_health() { + let mut s = two_lanes(); + let mut victim = None; + for i in 0..10 { + let (b, lane) = dispatch_one(&mut s, 100 + i * 10); + // Charge every 402 to one lane, so any accumulation would show. + if victim.is_none() { + victim = Some(lane); + } + s.on_batch_result(b, BatchOutcome::PaymentRequired, 100 + i * 10); + s.fund_lane(lane); + } + // A single real failure now must still be just one failure, not the + // straw that tips an already-charged streak into backoff. + let (b, lane) = dispatch_one(&mut s, 500); + s.on_batch_result(b, BatchOutcome::Failed("boom".into()), 500); + let health = s.lane_stats()[lane].health.expect("health"); + assert!( + matches!(health, LaneHealthKind::Live | LaneHealthKind::Warming), + "one failure after many 402s must not have compounded: {health:?}" + ); +} + +/// `Unfunded` is ineligible but recoverable — unlike `Retired`, which is +/// permanent for the run. +#[test] +fn an_unfunded_lane_is_paused_then_restored_by_paying() { + let mut s = two_lanes(); + let (b, lane) = dispatch_one(&mut s, 0); + s.on_batch_result(b, BatchOutcome::PaymentRequired, 0); + + assert_eq!( + s.unfunded_lanes(), + vec![lane], + "the driver must see what to pay" + ); + for _ in 0..8 { + assert_ne!( + s.next(20).map(|a| a.lane), + Some(lane), + "an unfunded lane must not be dispatched to" + ); + } + s.fund_lane(lane); + assert!(s.unfunded_lanes().is_empty(), "paying clears the pause"); + assert!(s.next(30).is_some(), "and the run continues"); +} + +/// Work must not be stranded on a paused lane: chunks go back to pending +/// and another lane picks them up while the cheque is in flight. +#[test] +fn work_on_an_unfunded_lane_fails_over_rather_than_stalling() { + let mut s = two_lanes(); + let (b, lane) = dispatch_one(&mut s, 0); + s.on_batch_result(b, BatchOutcome::PaymentRequired, 0); + let other = s.next(10).expect("work must not strand on a paused lane"); + assert_ne!(other.lane, lane, "it fails over to the funded lane"); +} + +/// Paying is optional (§2): a fleet may mix `open`, soft-metered and +/// hard-metered lanes, and a client with no chequebook must keep using the +/// ones it can. +/// +/// A retired lane is out of rotation for good — the point is that nothing +/// is ever dispatched to it, since a lane that refuses for a missing +/// capability is not a 402 and would charge lane health once per chunk. +#[test] +fn a_retired_lane_never_receives_work() { + let infos: Vec = (0..3).map(|_| LaneInfo::default()).collect(); + let mut sched = Scheduler::new(infos, Config::default()); + sched.admit(addrs(600, 7)); + + // Lane 1 is the one this client cannot pay. + sched.retire_lane(1); + + let mut seen = [0usize; 3]; + while let Some(a) = sched.next(0) { + seen[a.lane] += 1; + sched.on_batch_result(a.batch, BatchOutcome::Answered, 0); + } + assert_eq!(seen[1], 0, "retired lane took {} assignments", seen[1]); + assert!( + seen[0] > 0 && seen[2] > 0, + "the payable lanes must still carry the upload: {seen:?}" + ); +} diff --git a/src/signer.rs b/src/signer.rs index d80c595..6fc8acb 100644 --- a/src/signer.rs +++ b/src/signer.rs @@ -453,6 +453,16 @@ pub fn recover_eth_address_from_handshake_v15( recover_eth_from_eip191(&payload, signature) } +/// Public wrapper over [`recover_eth_from_eip191`], for verifying the +/// relay's signed price quote client-side (`docs/pusher-incentives.md` +/// §7.3). An unsigned price is repudiable in both directions. +pub fn recover_eth_address_from_eip191( + payload: &[u8], + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + recover_eth_from_eip191(payload, signature) +} + /// Recover the 20-byte Ethereum address from a 65-byte (r || s || v) /// EIP-191 signature over `payload`. `v` is expected in Ethereum form /// (27 or 28); k256 normalises to 0/1 internally. @@ -498,3 +508,319 @@ fn recover_eth_from_eip191(payload: &[u8], signature: &[u8]) -> Result<[u8; 20], addr.copy_from_slice(&hash[12..]); Ok(addr) } + +// ────────────────────────────────────────────────────────────────────── +// Metered relay (docs/pusher-incentives.md Stage 1) +// ────────────────────────────────────────────────────────────────────── + +/// secp256k1n ÷ 2. `ERC20SimpleSwap.recoverEIP712` calls `ECDSA.recover` +/// from `@openzeppelin/contracts/cryptography/ECDSA.sol` at `^3.4.1`, which +/// requires `uint256(s) <= this` and `v ∈ {27, 28}` — so a signature outside +/// that range is *off-chain valid and on-chain uncashable*. See incentives +/// §11.6; rejecting it at the boundary is what stops a client buying service +/// with a cheque that can never be redeemed. +const SECP256K1N_HALF: [u8; 32] = [ + 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x5D, 0x57, 0x6E, 0x73, 0x57, 0xA4, 0x50, 0x1D, 0xDF, 0xE9, 0x2F, 0x46, 0x68, 0x1B, 0x20, 0xA0, +]; + +/// Reject a signature the deployed chequebook would refuse to honour. +/// +/// This is a *validity* check, not a malleability nicety. `alloy` recovers +/// high-`s` and `v ∈ {0,1}` happily, so without this the relay accepts +/// cheques it can never cash. +pub fn check_canonical_signature(sig: &[u8]) -> Result<(), SignerError> { + if sig.len() != 65 { + return Err(SignerError::Alloy(format!( + "signature must be 65 bytes, got {}", + sig.len() + ))); + } + if sig[64] != 27 && sig[64] != 28 { + return Err(SignerError::Alloy(format!( + "non-canonical v={}: ERC20SimpleSwap requires 27 or 28, so this \ + cheque would revert at cashout", + sig[64] + ))); + } + // Both are big-endian 32-byte magnitudes, so a lexicographic slice + // compare is a numeric compare. + if sig[32..64] > SECP256K1N_HALF[..] { + return Err(SignerError::Alloy( + "non-canonical high-s signature: ERC20SimpleSwap rejects \ + s > secp256k1n/2, so this cheque would revert at cashout" + .into(), + )); + } + Ok(()) +} + +/// The EIP-712 domain bee uses for cheques. Extracted so signing and +/// recovery cannot drift: a mismatched domain silently recovers a +/// *different* address rather than failing, which would look like "the +/// client signed with the wrong key". +pub fn chequebook_domain(chain_id: u64) -> Eip712Domain { + Eip712Domain { + name: Some("Chequebook".into()), + version: Some("1.0".into()), + chain_id: Some(alloy_primitives::U256::from(chain_id)), + verifying_contract: None, + salt: None, + } +} + +/// Domain for the relay's own admission challenge (incentives §7.2). +/// +/// Separate from `Chequebook` on purpose: the account key already signs +/// postage stamps (EIP-191 over raw stamp bytes) and cheques (EIP-712, +/// `Chequebook`). A third scheme over the same key without its own domain +/// invites cross-scheme confusion, where a signature gathered for one +/// purpose is replayable as another. +pub fn pusher_domain(chain_id: u64) -> Eip712Domain { + Eip712Domain { + name: Some("HoverflyPusher".into()), + version: Some("1".into()), + chain_id: Some(alloy_primitives::U256::from(chain_id)), + verifying_contract: None, + salt: None, + } +} + +sol! { + /// Signed by the *client* to prove it holds the account key the relay + /// issued a challenge to. `origin` is the host the client dialled and is + /// what stops a signature gathered at relay A being replayed at relay B + /// (incentives §11.1) — but only if the relay compares it against its own + /// configured hostname rather than a request header. + struct PushChallenge { + bytes32 nonce; + string origin; + address account; + bytes32 batchId; + uint256 expiry; + } +} + +/// Recover the issuer of an EIP-712 cheque. +/// +/// Mirrors bee's `RecoverCheque` (`chequestore.go:190`). Rejects +/// non-canonical signatures first — a cheque the chain would refuse is not +/// a cheque, however well it recovers locally. +pub fn recover_cheque_issuer( + chequebook: &[u8; 20], + beneficiary: &[u8; 20], + cumulative_payout: alloy_primitives::U256, + chain_id: u64, + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + check_canonical_signature(signature)?; + let cheque = Cheque { + chequebook: alloy_primitives::Address::from(*chequebook), + beneficiary: alloy_primitives::Address::from(*beneficiary), + cumulativePayout: cumulative_payout, + }; + recover_typed(&cheque, &chequebook_domain(chain_id), signature) +} + +/// Recover the account that signed a `PushChallenge`. +pub fn recover_push_challenge( + challenge: &PushChallenge, + chain_id: u64, + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + // The challenge is never cashed, so canonical form is not a *validity* + // requirement here — but accepting only one encoding per signature keeps + // the replay surface a single value rather than four. + check_canonical_signature(signature)?; + recover_typed(challenge, &pusher_domain(chain_id), signature) +} + +fn recover_typed( + value: &T, + domain: &Eip712Domain, + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + use k256::ecdsa::{RecoveryId, Signature as K256Sig, VerifyingKey}; + let digest = value.eip712_signing_hash(domain); + let v = signature[64] - 27; + let k_sig = K256Sig::from_slice(&signature[..64]) + .map_err(|e| SignerError::Alloy(format!("k256 sig: {e}")))?; + let rec_id = + RecoveryId::try_from(v).map_err(|e| SignerError::Alloy(format!("recovery id: {e}")))?; + let vk = VerifyingKey::recover_from_prehash(digest.as_slice(), &k_sig, rec_id) + .map_err(|e| SignerError::Alloy(format!("recover: {e}")))?; + let point = vk.to_encoded_point(false); + let hash: [u8; 32] = Keccak256::digest(&point.as_bytes()[1..]).into(); + let mut addr = [0u8; 20]; + addr.copy_from_slice(&hash[12..]); + Ok(addr) +} + +impl SwarmSigner { + /// Sign a `PushChallenge` (incentives §7.2). Client side. + pub fn sign_push_challenge( + &self, + challenge: &PushChallenge, + chain_id: u64, + ) -> Result<[u8; 65], SignerError> { + let sig = self + .inner + .sign_typed_data_sync(challenge, &pusher_domain(chain_id)) + .map_err(|e| SignerError::Alloy(e.to_string()))?; + let mut bytes = sig.as_bytes(); + if bytes[64] < 27 { + bytes[64] += 27; + } + Ok(bytes) + } +} + +#[cfg(test)] +mod metered_tests { + use super::*; + + const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + + fn signer() -> SwarmSigner { + SwarmSigner::from_hex_with_nonce(KEY, &format!("0x{}", hex::encode([0u8; 32])), 1) + .expect("valid key") + } + + #[test] + fn a_cheque_recovers_to_the_key_that_signed_it() { + let s = signer(); + let cb = [0x11u8; 20]; + let bn = [0x22u8; 20]; + let amount = alloy_primitives::U256::from(1_000_000u64); + let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + let got = recover_cheque_issuer(&cb, &bn, amount, 100, &sig).expect("recover"); + assert_eq!(got, *s.eth_address(), "issuer must be the signing key"); + } + + /// The domain separator carries the chain id, so a cheque signed for + /// Gnosis must not validate against a relay pinning Sepolia — otherwise + /// one signature pays on every chain at once. + #[test] + fn a_cheque_does_not_recover_across_chains() { + let s = signer(); + let (cb, bn) = ([0x11u8; 20], [0x22u8; 20]); + let amount = alloy_primitives::U256::from(7u64); + let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + let other = recover_cheque_issuer(&cb, &bn, amount, 11155111, &sig).expect("recovers"); + assert_ne!( + other, + *s.eth_address(), + "wrong chain must not recover the issuer" + ); + } + + /// Changing any signed field must move the recovered address, or the + /// relay could be paid with a cheque made out to someone else. + #[test] + fn a_cheque_binds_every_field() { + let s = signer(); + let (cb, bn) = ([0x11u8; 20], [0x22u8; 20]); + let amount = alloy_primitives::U256::from(500u64); + let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + let me = *s.eth_address(); + assert_ne!( + recover_cheque_issuer(&cb, &[0x33u8; 20], amount, 100, &sig).expect("rec"), + me, + "beneficiary is bound" + ); + assert_ne!( + recover_cheque_issuer(&[0x44u8; 20], &bn, amount, 100, &sig).expect("rec"), + me, + "chequebook is bound" + ); + assert_ne!( + recover_cheque_issuer(&cb, &bn, alloy_primitives::U256::from(501u64), 100, &sig) + .expect("rec"), + me, + "cumulative payout is bound" + ); + } + + #[test] + fn a_push_challenge_recovers_to_the_account() { + let s = signer(); + let c = PushChallenge { + nonce: alloy_primitives::B256::from([9u8; 32]), + origin: "relay-a.example".into(), + account: alloy_primitives::Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from([5u8; 32]), + expiry: alloy_primitives::U256::from(1_700_000_000u64), + }; + let sig = s.sign_push_challenge(&c, 100).expect("sign"); + assert_eq!( + recover_push_challenge(&c, 100, &sig).expect("recover"), + *s.eth_address() + ); + } + + /// The whole point of binding `origin`: a signature gathered at relay A + /// must not verify at relay B (§11.1). + #[test] + fn a_push_challenge_binds_the_origin() { + let s = signer(); + let mut c = PushChallenge { + nonce: alloy_primitives::B256::from([9u8; 32]), + origin: "relay-a.example".into(), + account: alloy_primitives::Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from([5u8; 32]), + expiry: alloy_primitives::U256::from(1_700_000_000u64), + }; + let sig = s.sign_push_challenge(&c, 100).expect("sign"); + c.origin = "relay-b.example".into(); + assert_ne!( + recover_push_challenge(&c, 100, &sig).expect("recovers something"), + *s.eth_address(), + "a challenge signed for relay A must not recover the account at relay B" + ); + } + + /// A cheque and a challenge signed over structurally similar data must + /// live in different domains, or one could be replayed as the other. + #[test] + fn cheque_and_challenge_domains_are_distinct() { + assert_ne!( + chequebook_domain(100).separator(), + pusher_domain(100).separator() + ); + } + + #[test] + fn non_canonical_signatures_are_rejected() { + let s = signer(); + let (cb, bn) = ([0x11u8; 20], [0x22u8; 20]); + let amount = alloy_primitives::U256::from(1u64); + let good = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + check_canonical_signature(&good).expect("alloy always produces canonical"); + + let mut bad_v = good; + bad_v[64] = 1; // the 0/1 encoding alloy accepts and the contract does not + let e = check_canonical_signature(&bad_v).expect_err("v=1 must be rejected"); + assert!(format!("{e}").contains("non-canonical v"), "got: {e}"); + + let mut high_s = good; + high_s[32] = 0xFF; // s > secp256k1n/2 + let e = check_canonical_signature(&high_s).expect_err("high-s must be rejected"); + assert!(format!("{e}").contains("high-s"), "got: {e}"); + + for len in [0usize, 64, 66] { + check_canonical_signature(&vec![1u8; len]).expect_err("bad length must be rejected"); + } + } + + /// Exactly the boundary OpenZeppelin enforces: `s <= n/2` passes, + /// `s = n/2 + 1` does not. + #[test] + fn the_high_s_boundary_matches_openzeppelin() { + let mut sig = [0u8; 65]; + sig[64] = 27; + sig[32..64].copy_from_slice(&SECP256K1N_HALF); + check_canonical_signature(&sig).expect("s == n/2 is allowed"); + sig[63] += 1; + check_canonical_signature(&sig).expect_err("s == n/2 + 1 is not"); + } +} diff --git a/src/wasm.rs b/src/wasm.rs index b860e96..71161ca 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -1139,8 +1139,18 @@ impl UploadSession { /// Feed a lane's `/v1/status` JSON. Per lane and idempotent: a relay /// that is asleep when polled simply keeps its default priors instead /// of degrading routing for every other lane. + /// + /// Returns `false` when the lane was taken out of rotation instead of + /// scheduled — today that means it enforces payment + /// (`docs/pusher-incentives.md` §7.3) and this build has no way to pay. + /// The browser stamps but never settles: the chequebook lives in the + /// native client, so a hard lane would answer every POST with 401 for a + /// missing capability, and that is not a 402 — it counts against lane + /// health and burns one attempt per chunk rediscovering something the + /// lane announced up front. Soft-metered lanes are kept, since those + /// bill and serve; an unpaying client is served exactly like on `open`. #[wasm_bindgen(js_name = "setLaneStatus")] - pub fn set_lane_status(&mut self, lane: usize, status: JsValue) -> Result<(), JsError> { + pub fn set_lane_status(&mut self, lane: usize, status: JsValue) -> Result { // Round-trip through JSON text rather than pulling in // serde-wasm-bindgen for one struct. let txt = js_sys::JSON::stringify(&status) @@ -1152,6 +1162,21 @@ impl UploadSession { .and_then(|s| s.as_str()) .and_then(|s| hex::decode(s.trim_start_matches("0x")).ok()) .and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok()); + // The advertised `enforcement` is read without verifying the quote's + // signature, unlike the native path. Verification protects a *payer* + // from being overcharged; there is no payment here to protect, and + // the only thing this flag can do is make us decline a lane. A lane + // lying its way out of our traffic is a lane denying its own + // service. + let payment = v.get("payment"); + let hard_enforcement = payment + .and_then(|p| p.get("enforcement")) + .and_then(|x| x.as_str()) + == Some("hard"); + let price_plur_per_kib = payment + .and_then(|p| p.get("price_plur_per_kib")) + .and_then(|x| x.as_str()) + .and_then(|s| s.parse::().ok()); self.sched.set_lane_info( lane, crate::pushsched::LaneInfo { @@ -1170,9 +1195,15 @@ impl UploadSession { .and_then(|p| p.get("live")) .and_then(|x| x.as_u64()) .map(|x| x as usize), + price_plur_per_kib, + hard_enforcement, }, ); - Ok(()) + if hard_enforcement { + self.sched.retire_lane(lane); + return Ok(false); + } + Ok(true) } /// Next POST to issue, or `undefined` if nothing is dispatchable right