From 9e6adceadcb766264261be4ea2951701f87900b6 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Thu, 6 Aug 2026 15:14:29 +0300 Subject: [PATCH 01/27] feat(pusher): metered relay mode + Stage 0 shadow metering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements docs/pusher-incentives.md Stages 0 and 1 in soft mode: the relay can be paid, but never refuses. All of it is behind `--meter`, so open mode — every production lane — is byte-identical to before. Billing unit is **bytes admitted**, not pushsync receipts. The client cannot lie about it because the client produced the bytes and the relay counted them: no third-party attestation to forge, no chain state to disagree about. That choice deletes the staking-registry anchor, the set-valued invoice and the sampled audit an earlier draft needed, along with every attack against them. Relay side: challenge.rs capability MAC, domain-tagged length-prefixed preimage ledger.rs owed/reserved/cumulative/binding, durable except `reserved` — no in-flight POST survives a restart to release it, so restoring it leaks credit permanently metered.rs endpoint logic + state, keeping pusher.rs a router inbound_limit.rs fail-closed limiter: only full buckets are evictable, so flooding keys cannot mint budget meter.rs Stage 0 shadow accounting, /v1/meter Client side: payer.rs signed-quote verification, lane pinning on (url, node_eth_address, beneficiary), POST sizing against the credit line, local owed tracking pushsched.rs BatchOutcome::PaymentRequired + non-terminal LaneHealth::Unfunded — a 402 is a bill, not a fault, and must not retire a healthy lane cheques.rs total_issued + `relay:` key namespace, so lanes sharing a beneficiary share one cumulative Verified against swap-swear-and-swindle rather than inferred from bee: cheque signatures must be canonical (OZ 3.4.1 rejects high-s and v∉{27,28}, so a non-canonical cheque buys service and reverts at cashout), and the funding check reads liquidBalanceFor rather than balance, which counts other beneficiaries' hard deposits as our coverage. 90 new tests. Not wired: the driver does not yet act on a 402, and the /v1/pay chain reads have not been exercised against a real chequebook. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-design.md | 107 ++- docs/pusher-incentives.md | 1495 +++++++++++++++++++++++++++++++++++++ src/batch.rs | 160 +++- src/bin/hoverfly.rs | 119 +++ src/challenge.rs | 310 ++++++++ src/cheques.rs | 140 ++++ src/client.rs | 63 +- src/inbound_limit.rs | 196 +++++ src/ledger.rs | 556 ++++++++++++++ src/lib.rs | 18 + src/meter.rs | 780 +++++++++++++++++++ src/metered.rs | 694 +++++++++++++++++ src/payer.rs | 681 +++++++++++++++++ src/protocols/pushsync.rs | 132 ++++ src/protocols/swap.rs | 233 ++++++ src/pusher.rs | 1114 +++++++++++++++++++++++++-- src/pushframe.rs | 6 +- src/pushsched.rs | 61 ++ src/pushsched/tests.rs | 98 +++ src/signer.rs | 322 ++++++++ 20 files changed, 7225 insertions(+), 60 deletions(-) create mode 100644 docs/pusher-incentives.md create mode 100644 src/challenge.rs create mode 100644 src/inbound_limit.rs create mode 100644 src/ledger.rs create mode 100644 src/meter.rs create mode 100644 src/metered.rs create mode 100644 src/payer.rs diff --git a/docs/pusher-design.md b/docs/pusher-design.md index 1d94739..5efe84c 100644 --- a/docs/pusher-design.md +++ b/docs/pusher-design.md @@ -646,7 +646,10 @@ worked without it: ### Deferred / watchlist - PR-sized follow-ups: `--push-quota`/`--push-challenge` hardening, P2 - workerd port (unlocks Deno Deploy), attribution-log tooling. + workerd port (unlocks Deno Deploy), attribution-log tooling. Note that + the metered-relay design (§12) subsumes `--push-quota` outright — price + is a strictly better quota — and promotes `--push-challenge` from + optional hardening to a correctness requirement. - Contiguous-arc lane assignment + deep pool specialization — only if receipt data shows forwarding depth is a real cost (§7). - WS/WT bindings of the frame format — only on demonstrated need (§4). @@ -656,7 +659,107 @@ worked without it: dial storage nodes directly and the pusher's raison d'être shrinks to constrained networks. -Deferred/watchlist: +## 12. Incentives — paying for relay + +Specified separately in **[`pusher-incentives.md`](./pusher-incentives.md)** +(status: design only, nothing implemented). + +The problem it addresses: a relay absorbs a cost it did not incur. In a +native upload the user's own machine is the peer bee debits for every +chunk; put a relay in the middle and that debt moves wholesale to the +relay, while the browser client that caused the traffic pays only postage. +§6 books this as an accepted risk and §10 identifies the dedicated egress +IP as the thing that is actually scarce. + +The design adds an optional **`metered`** relay mode (today's behaviour +becomes `open`, and the four production lanes stay there) in which a +client pays with off-chain SWAP cheques over the existing HTTPS channel. +Both counterparties are hoverfly — bee is not a party, and only the +chequebook contract and EIP-712 cheque format are borrowed, not the swap +protocol. Payment is out-of-band (`POST /v1/pay`), the account is the +batch-owner EOA already established by push auth, and the relay holds only +the beneficiary *address* — never a spendable key. + +**The trust model is one-directional, and it drives everything else.** +Relays are a curated set pinned by URL (`PUSHER_URLS`, +`apps/upload/src/config.ts:18-25`); clients are anonymous. So the design +protects the *relay* from the *client*, and a misbehaving lane is handled +socially — removed from the list — rather than cryptographically. An +earlier revision built two-sided verification for this one-sided +relationship and paid for it with a forgeable billing unit and an +unbounded residual; see incentives §2. + +Findings from that doc that constrain this one: + +- **The billing unit is bytes admitted, not receipts or acks.** The client + cannot lie about it, because the client produced the bytes and the relay + counted them — no third-party attestation to forge, no chain state to + disagree about (incentives §8). This matters here because it settles + what receipt forwarding is *for*: forwarding `PushsyncReceipt` into the + ack is still worth doing — `PushInfo` currently drops the signature, so + acks are pure relay assertion, and it is the first cryptographic signal + a relay client has ever had — but it is **telemetry that feeds lane + weighting, not evidence that feeds an invoice**. A receipt signs the + bare chunk address (`bee/pkg/pushsync/pushsync.go:277`), so it is + forgeable with any throwaway key and carries no freshness; anything that + prices work by receipt inherits both problems. +- **`--push-challenge` becomes mandatory under metering.** Swarm stamps + are public (§6), so an attacker can replay a victim's stamps at a + metered relay and have the work billed to the victim. The signed payload + must bind the relay's **origin**, not its beneficiary, and must be an + EIP-712 typed struct — the same account key already signs stamps and + cheques, so a third raw-bytes scheme over it invites confusion. And the + origin the relay compares against must come from **configuration, never + from the `Host` header** — the header is supplied by the same party + supplying the challenge, so checking one against the other is a no-op + that silently reopens the replay (incentives §11.1). +- **"A batch owner is a costly identity" is false as stated.** The relay + checks batch *liveness*, and the cheapest batch the contract accepts + costs a fraction of a cent — so any flat per-account credit line is + roughly self-financing for an attacker. The fix is to scale the credit + line to the batch's remaining on-chain value rather than gate on it + (incentives §10.3). Worth knowing here because it is the same mistake + any future per-batch quota would make. +- **`--push-quota` should be struck** if metered mode ships; price is a + strictly better quota than a volume cliff. +- **Six live bugs in open-mode code, found during those reviews — all + fixed** (incentives §16). None depend on metering; they were in + production the whole time. + - *Stamp substitution via the recent-ack cache.* Dedup was keyed on + chunk address alone, so a hit acked a frame `ok` while silently + discarding the submitted stamp. Since addresses are content-derived, + one uploader's dust batch could shadow another's year-long batch for + the 120 s TTL — the victim's chunk then garbage-collected when the + attacker's batch expires. It also fired accidentally between honest + users uploading the same file. Now keyed on `(addr, batch_id)` + (`src/pusher.rs:286`). + - *Unauthenticated RPC amplification on `/v1/push`.* `resolve_owner` + cached only successes, unbounded, with no per-request budget, so one + anonymous POST naming bogus batches became up to 512 serial + `eth_call`s — and `EthRpc::new` built a fresh HTTP client per read. + Now a bounded cache with negative caching, TTLs, and an 8-lookup + budget, over one shared client. + - *No connection limit.* The accept loop spawned per connection with + nothing bounding it, despite §3 listing a cap as table stakes. Now a + 256-permit semaphore acquired before `accept()`. + - *One transient accept error killed the relay.* `accept().await?` + propagated `EMFILE`/`ECONNABORTED` out of `run`. Now logged, backed + off, and continued. + - *No HTTP timeouts.* The comment claiming hyper's defaults sufficed was + backwards: `header_read_timeout` is inert unless a timer is installed, + so nothing was enforced. Now a timer plus a 30 s header timeout and a + 120 s body timeout. + - *Pushsync receipt addresses were never validated* — the one that + matters here, because §7's whole receipt-forwarding idea rests on it. + `receipt.address` was neither length-checked (callers + `copy_from_slice` it into `[u8; 32]`, so a missized address was a + remote panic) nor compared to the address actually pushed (so a peer + could store nothing and sign for a different address deep in its own + neighbourhood, and `is_shallow` would call it a perfect delivery). + Checked at the protocol boundary now, with regression tests. + +### Deferred / watchlist (cont.) + - Contiguous-arc lane assignment + deep pool specialization — only if receipt data shows forwarding depth is a real cost (§7). - WS/WT bindings of the frame format — only on demonstrated need (§4). diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md new file mode 100644 index 0000000..7fcee0f --- /dev/null +++ b/docs/pusher-incentives.md @@ -0,0 +1,1495 @@ +# Pusher incentives: paying for relay with SWAP cheques + +Status: **Stages 0 and 1 shipped (soft mode); Stages 2–3 outstanding.** A relay can be paid, but never +refuses: soft mode meters, reports, and accepts cheques, and 402 +enforcement is Stage 2. See §14 for exactly what is and is not in. This doc +specifies an +optional *metered* mode for `hoverfly pusher` in which a client pays the +relay for bandwidth with off-chain SWAP cheques. It is a companion to +`docs/pusher-design.md`, which stays the index for the pusher subsystem; +read §§1–7 there first. + +The short version: **both counterparties are hoverfly.** A client pays a +relay. Bee is not a party to the payment — we borrow SWAP's *contracts and +cheque format*, not its protocol. + +**This is a rewrite.** Three rounds of adversarial review went into the +previous version, and the third round's most important finding was +structural rather than local: the doc was building *two-sided* +cryptographic verification for a *one-sided* trust relationship. Relays +are a curated, pinned set (`PUSHER_URLS`, `apps/upload/src/config.ts:18-25` +— four URLs, all ours); clients are anonymous. Roughly half the old +document defended the client against the relay, which meant defending our +own infrastructure from itself, and it paid for that with a forgeable +billing unit, an unbounded residual, and a ship-blocking measurement +nobody had taken. + +This version points every defence in one direction — **the relay is what +gets protected, from the client** — and picks a billing unit the client +cannot lie about. The result is smaller, has no kill criterion, and closes +the attacks that involve actual money. §16 records the live bugs that +review found in open-mode code; they are the durable artifact of the +process and are all fixed. + +## 1. Why + +Today the relay eats a cost it did not incur. + +In a native upload the user's own machine opens the pushsync streams, and +bee debits *it* for every chunk — `price(po) = (32 − po) × 10 000` +accounting units (`bee/pkg/pricer/pricer.go:34-36`, mirrored at +`src/transport.rs:94-97`). Put a relay in the middle and that debt moves +wholesale to the relay: it is the peer bee sees, so it is the peer bee +charges. The browser client that caused the traffic pays nothing but +postage. + +`docs/pusher-design.md` §6 currently books this as an accepted risk — +*"worst case = the platform's free egress for the month (~70–100 GB) +burned, $0 lost"* — and §10 identifies the thing that is actually scarce: +a dedicated egress IP with an unthrottled dial budget. Free-tier lanes +starve at a fraction of a dedicated host's throughput, and the only way to +get more of them is for someone to volunteer. + +Metering is the answer to both. It puts the cost back on the party that +caused it, converts §6's accepted quota-drain risk into a priced one, and +makes running a dedicated-IP relay a rational act rather than a donation. + +Nobody gets rich. §9's economics are thin and §9.3 is explicit that a +single small upload does not cover its own cashout gas. The target is a +self-sustaining lane federation funded by repeat and bulk traffic, not +profit. + +## 2. Trust model — read this before anything else + +Every design decision below follows from one asymmetry. + +> **Relays are known. Clients are anonymous.** +> +> A relay is an entry in a hardcoded list, run by a named operator, pinned +> by URL over HTTPS. A client is whoever POSTs. Defences point *from* the +> relay *at* the client. The security goal is: **a client cannot obtain +> relay service without paying for it, and cannot lie to the relay about +> what it owes.** + +The inverse — a relay defrauding a client — is explicitly **out of scope +for this iteration**. Not because it is impossible, but because it is +governed socially rather than cryptographically: a lane that misbehaves is +removed from `PUSHER_URLS`. If the federation ever opens to unvetted +operators that becomes a real problem and §14 records what it would take. + +Three consequences worth stating so they don't get re-litigated: + +**No confidentiality is claimed or implied.** The pusher sees plaintext +chunks, the stamp, and the batch-owner address. That is inherent to +relaying, and Swarm is not an anonymity system in the first place — chunks +carry a signed postage stamp, addresses are content-derived and publicly +retrievable, and retrieval hands the stamp back +(`src/protocols/retrieval.rs:27`, `:42-80`). Metering changes none of +this. The one operational consequence: **a relay holds every stamp it has +ever relayed**, which makes it the richest possible source of harvestable +stamps and is an argument for §7.2's origin-bound challenge. + +**The client is not asked to verify the relay's work cryptographically.** +It verifies *arithmetic* — it knows exactly how many bytes it sent — and +it measures *outcomes*, which `src/pushsched.rs` already does by +deweighting lanes that underperform. That is the same protection open mode +has today, and open mode works. + +**Receipts are telemetry, not evidence.** The relay forwards the pushsync +receipt in the ack because it is genuinely useful — it is the first +cryptographic signal a relay client has ever had about where its chunk +landed, and it feeds lane weighting — but **it does not enter the +invoice.** An earlier draft billed per verified receipt, which required +anchoring each receipt in the staking registry to stop forgery, and still +left a replay hole that no off-chain check could close. §8 explains why +the billing unit moved. + +## 3. Scope and non-goals + +**In scope:** a client (native CLI first, browser later) pays a +`hoverfly pusher` for relayed bytes, over the existing HTTPS channel, +denominated in BZZ. + +Explicit non-goals, each with its reason: + +- **Not bee's swap protocol.** No `/swarm/swap/1.0.0/swap` stream, no + `Handshake`/`EmitCheque` framing, no `exchange`/`deduction` headers, no + priceoracle. Client and relay already speak HTTP; there is nothing to + gain from tunnelling a libp2p settlement protocol through it. We reuse + the *cheque* (§4), not the transport. +- **Not bee's accounting model.** No `paymentTolerance`, no ghost + balances, no trust ramp, no blocklist state machine. But note §10.2: + bee's *reservation* concept is mandatory and an early draft wrongly + dropped it. +- **Not "the relay uses the income to pay bees."** `PERFORMANCE.md` + measured paid at 160 KiB/s against unpaid at 195 KiB/s ("verdict: not + confirmed at this workload"), and the arithmetic agrees: bee grants + 4 500 000 accounting units/s per peer via pseudosettle, ≈ 18 chunks/s at + PO 8, ≈ 2 400 chunks/s across a 128-session pool — against ~150 chunks/s + actually measured. The relay is session- and RTT-bound, not + credit-bound, so buying credit buys nothing. Relay→bee settlement stays + free pseudosettle. (The pusher does not even wire `SwapConfig` today — + `build_push_state` (`src/pusher.rs:724-772`) never calls `.with_swap`, + so the global `--chequebook` flag has no effect on a relay.) +- **Not confidentiality, not anonymity.** §2. +- **Not defence against a malicious relay.** §2, §14. +- **Not encrypted uploads.** 64-byte references are unsupported throughout + (`src/feed.rs:150`; the erasure coder is non-encrypted-path only at + `src/erasure/mod.rs:145`, `:365`, `:408`) and adding a key-management + surface is out of proportion to the problem. +- **Not a new contract.** See §13. + +## 4. What we borrow from SWAP, and what we drop + +| Borrowed | Where it comes from | Why | +|---|---|---| +| `ERC20SimpleSwap` chequebook + canonical factory | `bee/pkg/config/chain.go:89` (Gnosis `0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420`), `chain.go:66` (Sepolia `0x0fF044F6bB4F684a5A149B46D7eC03ea659F98A1`) | Audited, deployed, in production. Nothing to write. | +| EIP-712 cheque | `bee/pkg/settlement/swap/chequebook/cheque.go:32-38` | Domain `{name:"Chequebook", version:"1.0", chainId}` — no `verifyingContract`, no `salt`. Type `Cheque(address chequebook, address beneficiary, uint256 cumulativePayout)`. Already implemented at `src/signer.rs:309-337`. | +| Cumulative-payout monotonicity | `chequestore.go:133-138` (`ErrChequeNotIncreasing`) | Makes cheques loss-tolerant and replay-proof (§8.3). | +| Funding check | `chequestore.go:172-179` (`ErrBouncingCheque`) | But **use `liquidBalanceFor(us)`, not `balance()` — bee's version is unsound**, see §11.6. | +| Reservation against concurrent issuance | `chequebook.go:163-178` (`reserveTotalIssued`) | Needed on both sides. See §10.2. | +| Payee-only role | `bee/pkg/node/node.go:592-625`, `bee/pkg/node/chain.go:206` | The beneficiary is a plain **EOA**, not a chequebook. A payee needs no contract of its own. | +| Cheque JSON encoding | `src/protocols/swap.rs:84-106` (`encode_signed_cheque_json`) | Byte-exact Go `encoding/json` compatibility already solved. | + +| Dropped | Reason | +|---|---| +| swap libp2p stream, `Handshake`, `EmitCheque` framing | We're on HTTP. | +| priceoracle, `exchange`, `deduction` | Relay quotes PLUR directly. No oracle dependency, no new-peer ramp. | +| accounting-unit indirection | Prices are PLUR. One unit, no conversion. | +| `ErrChequeValueTooLow` as inherited | Bee's floor is "≥ 1 accounting credit" (`chequestore.go:37-38`), denominated in the oracle indirection we dropped. Our `min_cheque_plur` (§10.1) is a *different* mechanism with a different justification; do not treat bee as precedent for its sizing. | +| ghost/shadow balances, tolerance, trust ramp | No analogue in request/response. | +| `StakeRegistry` / staked-set snapshot | An earlier draft needed this to make forged pushsync receipts expensive. With receipts out of the invoice (§8) there is nothing to anchor, and Stage 1 loses its two largest work items. | + +## 5. Relay modes + +A relay runs in exactly one of two modes, advertised in `/v1/status`: + +- **`open`** — today's behaviour, unmetered. The four production lanes in + `apps/upload/src/config.ts:18-25` keep running this. Auth stays "stamp + signer is the live batch's on-chain owner" and nothing is billed. +- **`metered`** — every byte admitted is billed (§8). There is no free + allowance. + +Four consequences worth stating up front: + +**Metering subsumes the deferred `--push-quota`.** Design §6 proposed +capping each batch at its own effective volume per TTL. Under metering the +price *is* the quota, and a better one: it scales continuously instead of +cliff-edging at a volume boundary, and it doesn't require the relay to +model batch TTL semantics. `--push-quota` should be struck from the +watchlist if metered mode ships. + +**Metering makes `--push-challenge` mandatory, not optional** — and it +does double duty as the admission mechanism (§7.2, §11.1). + +**Metered mode requires durable storage.** Not a recommendation. §11.4 +shows that an ephemeral filesystem turns one signature into unlimited free +service. Free-tier hosts with ephemeral disks (Render free, the reference +deployment in design §11) **must** run `open`. + +**Metered mode requires a chain RPC.** Open mode already needs one for +batch-owner resolution; metering adds batch-value reads (§10.3) and cheque +verification (§11.6). Only on the relay — the client needs no chain access +to compute its own bill, which is the point of §8. + +## 6. Identity: the account is the batch owner, the credit line is the batch + +The relay's existing auth already establishes a strong, on-chain identity +for every push (`src/pusher.rs:895-960`): the stamp signature must recover +to the address that the `PostageStamp` contract reports as the batch's +owner, and the batch must be alive. That recovered address is the natural +account key. + +> **Account = the batch-owner EOA.** Cheques, the cumulative and the +> settlement ledger are keyed on it. +> **A cheque is valid for that account iff its chequebook's on-chain +> `issuer()` equals the same EOA.** +> **Credit is keyed one level finer, on the *batch*.** Standing (below), +> the credit line (§10.3) and admission (§7.2) are all per batch, because +> one owner can hold many batches of wildly different value and the +> cheapest of them must not buy the credit of the dearest. + +This binds payment to identity cryptographically, with no session tokens, +no registration step, and no extra protocol message. It lines up with the +CLI's existing semantics: `--chequebook`'s documented precondition is +already *"`issuer()` == `--key`'s address"* (`src/bin/hoverfly.rs:90-105`), +and `--key` is the stamp key. And in a browser it means the session key — +which already owns the batch — is the chequebook issuer, so cheques sign +with **zero wallet prompts**. + +**One *batch* per request.** `run_push` currently re-resolves the owner +whenever a frame's batch id changes (`src/pusher.rs:905-925`), so a single +POST can legally mix batches and owners. Metered mode must forbid this: +standing, the credit line and the reservation are all properties of a +*batch*, not of an owner who may hold many. Every frame in a POST must +carry the exact batch id named in the challenge, and a mismatched frame is +rejected rather than billed to whoever it names. Without this the +admission check in §7.2 is trivially bypassed by prefixing one frame from +a good-standing batch (§11.8). + +Tightening account → batch costs nothing and buys three alignments: the +recent-ack cache is already keyed on `(addr, batch_id)` (§16.1), admission +resolves standing once per POST instead of once per frame, and the +reservation has exactly one credit line to check against. A client mixing +batches splits them across POSTs, which the scheduler already pipelines. + +**The relay never holds spendable key material.** It needs the +beneficiary's *address* only. `cashChequeBeneficiary(recipient, cumulativePayout, signature)` +must be called *by* the beneficiary (`cashout.go:137`, and +`_cashChequeInternal` takes `msg.sender` as the beneficiary), so cashing +out happens later, elsewhere, from a machine that does hold the key. A +relay box holds nothing worth stealing — the property that makes today's +pusher safe (design §1, "not a signer") survives metering intact. The +relay *does* hold its node-identity key (`HOVERFLY_PUSHER_IDENTITY`), +which is not spendable and is used to sign quotes and challenges (§7.3). + +**The factory check is security, not compatibility.** The relay must +verify `factory.deployedContracts(chequebook) == true` +(`bee/pkg/settlement/swap/chequebook/factory.go:101-118`) against a +factory address **hardcoded per chain**, never one supplied by the client. +Skip it and a client presents an arbitrary contract that returns a forged +`issuer()` and `liquidBalanceFor()` and implements `cashChequeBeneficiary` +as a no-op — total compromise for one `eth_call` saved. Optionally also +check the deployed bytecode hash against `AcceptedChequebookBytecodeHashes` +(`bee/pkg/config/chain.go:96-98`). + +**Liveness is not enough: metering needs batch *standing*.** Open-mode +code now caches `batch_id → owner` through a bounded `OwnerCache` with +separate TTLs for successes and definitive rejections +(`src/pusher.rs:181-239`, `:1106`) — an amplification fix (§16.2) that +happens to give metering a staleness bound. Two gaps remain. + +The first is that the TTL is tuned for amplification, not standing. +`OWNER_OK_TTL_SECS = 1800` bounds how stale an aliveness answer can be; +§10.3's Sybil bound needs the *credit line* re-read on the same schedule. +Metered mode extends the cache entry rather than adding a second cache. + +The second is that liveness is a *boolean* and the thing it stands in for +is a *quantity*. "Alive" is satisfied by the cheapest batch that clears +the contract's minimum — minimum depth, minimum validity — which costs a +fraction of a cent. So metered mode reads standing, not liveness: + +``` +standing(B) = (owner, depth, remaining_value_plur) +remaining_value_plur = remainingBalance(B) × 2^depth +``` + +Both reads already exist — `read_batch` returns owner and depth +(`src/batch.rs:248`), `read_remaining_balance` returns PLUR per chunk +still funded (`:288`) — so this is one extra `eth_call` over what open +mode already does, cached per batch under the same TTL. §10.3 turns +`remaining_value_plur` into that account's credit line, which is what +makes the Sybil bound hold by construction rather than by assumption. + +Known limitation: one account = one batch owner = one chequebook. A client +uploading under several batches with distinct owners needs a chequebook +per owner. A signed authorization linking extra batch owners to one +chequebook is possible but deferred (§15). + +## 7. Wire protocol + +Push frames (`src/pushframe.rs`) are **unchanged**. Payment is +out-of-band, for the same reason bee keeps swap on a separate stream from +pushsync: it must not sit on the hot path, and a payment failure must not +fail a push. + +| Endpoint | Shape | +|---|---| +| `GET /v1/status` | new `payment` block, signed (§7.3); `mode: "open" \| "metered"` | +| `GET /v1/challenge?account=&batch=` | `{nonce, expires_ms, max_outstanding_plur}` — stateless MAC, issued only to a batch in good standing (§7.2) | +| `GET /v1/account` | requires the challenge header. `{owed_plur, reserved_plur, outstanding_plur, kib_admitted, kib_dedup, cumulative_received_plur, settle_every_plur, max_outstanding_plur}` | +| `POST /v1/pay` | requires the challenge header; body = `SignedCheque` JSON → `{accepted_plur, cumulative, outstanding_plur}` | +| `POST /v1/push` | requires the challenge header; `402 Payment Required` when over cap | + +`/v1/account` is authenticated because unauthenticated it is a +per-identity volume oracle over on-chain-enumerable batch owners, and a +targeting oracle for tipping a victim into 402 at a chosen moment. + +### 7.1 Rollout + +Two-phase. *Soft mode* first: the relay meters and reports `owed` in the +`done` line of `/v1/push` but never refuses. Existing clients ignore +unknown fields and keep working, exactly as the stage-B relays did against +stage-C clients (design §7, "mixed version"). Only once clients in the +wild can pay does a relay flip to hard mode. + +Soft mode is an *instrument*, not a migration path for clients: the `done` +line lands after the whole batch (`src/pusher.rs:1090`), so a client +cannot use it to pace itself. Its job is to tell the operator what §14 +Stage 0 needs to know. + +### 7.2 Admission — the challenge carries account, batch and credit line + +An early draft claimed 402 was easy because `/v1/push` spawns its body +(`src/pusher.rs:835`), so the status is committed before any chunk is +processed. That is true and it is the *problem*: at that moment the relay +does not yet know whose account to check. The account only exists after +`stamp::validate` (`:900`) and `resolve_owner` (`:912`, two `eth_call`s on +a miss), both inside the spawned task. Hoisting them means up to 512 +ecrecovers (~40 ms) plus a possible RPC round-trip synchronously in front +of every response — the exact unauthenticated amplification surface §11.6 +spends a section defending, and instantly over budget on the CF Workers +profile (design §9: 10 ms CPU/req). + +**The challenge solves this by moving the chain reads off the POST path +entirely.** Standing (§6) is resolved once when the challenge is *issued*, +and the resulting credit line is baked into the nonce. `/v1/push` +admission then reads no chain state at all: + +``` +GET /v1/challenge?account=A&batch=B + → resolve standing(B) (cached per batch, TTL; §6) + → require owner(B) == A, else 403 — no nonce is issued + → cap = credit_line(standing(B)) (§10.3) + → nonce = HMAC(relay_secret, preimage(A, B, origin, expiry, cap)) + → {nonce, expires_ms, max_outstanding_plur: cap} +``` + +**The MAC preimage is fixed-width and domain-tagged, not a +concatenation.** `origin` is variable-length, so a bare concatenation +makes `("host.a", "bc")` and `("host.ab", "c")` share a preimage, and a +relay serving several hostnames would issue one nonce valid for two of +them. In bytes rather than a typed struct, because only the relay ever +parses it: + +``` +preimage = "hoverfly-pusher-challenge-v1" // domain tag, fixed 28 B + ‖ A (20 B) + ‖ B (32 B) + ‖ expiry_be ( 8 B) + ‖ cap_be (16 B) + ‖ len(origin) ( 2 B, big-endian) + ‖ origin (variable, last) +``` + +Two operational rules come with it. Compare the MAC in **constant time** — +it is the only secret in the exchange and the client controls every other +field, so a byte-wise early exit is a forgery oracle. And **persist +`relay_secret` alongside the ledger** (§11.4): a secret regenerated at +boot invalidates every outstanding challenge, which on a host that sleeps +and cold-starts (design §7 measured a 35.2 s wake) turns every restart +into a 403 storm for clients mid-upload. + +The nonce is therefore a **capability**: possessing one is proof the relay +already checked standing and already priced the credit line. Admission +becomes: + +1. Verify the MAC over `preimage(A, B, origin, expiry, cap)` — symmetric, + no RPC, constant-time compare. +2. Verify `origin` against the relay's **configured** hostname — never + against the request's `Host` or `X-Forwarded-Host` (see below). +3. Verify the client's signature over the challenge struct → recovers `A` + (1 ecrecover, no RPC, no body). +4. `reserve = ceil(Content-Length / 1024) × price_plur_per_kib`; if + `outstanding(A) + reserve > cap` → **402**, before reading the body. +5. Otherwise commit the reservation atomically, return 200, spawn. +6. On completion, convert reservation → `owed` for bytes actually admitted + (§8) and release the remainder. + +**`origin` must be configured, not derived.** This is the single +easiest-to-get-wrong line in the design. The obvious implementation reads +the `Host` header and compares the challenge's `origin` against it — and +that is a no-op, because `Host` is *supplied by the same client that +supplies the challenge*. An attacker replaying a victim's signature at +relay B simply sends `Host: relay-b.example`, the comparison passes, and +§11.1's cross-relay replay is restored in full while the doc claims it is +closed. `X-Forwarded-Host` is worse: attacker-set on any relay not behind +a proxy that overwrites it, which the reference deployment is not. So the +relay takes its hostname from a required `--origin` flag and compares +against that constant. A relay reachable at several hostnames configures +the list; a relay that cannot state its own origin must not run metered. + +**The reservation is exactly the billing unit.** Because §8 bills bytes, +the quantity reserved at admission and the quantity billed at completion +are the same thing, computed the same way — `Content-Length` before, bytes +counted after. No estimation, no conversion, no over-reserve. (An earlier +draft reserved a flat `PUSH_BATCH_MAX × price`, which was ~10× a dust +batch's entire credit line and 402'd every POST from it regardless of +size.) A request with no `Content-Length` is refused; the existing client +always sets one, and `PUSH_MAX_BODY` (`src/pusher.rs:64`, ≈ 2.08 MiB) +caps it anyway. + +**Small batches are the client's job to size.** A 0.01 BZZ dust batch gets +~208 KiB of credit (§10.3), so it cannot push a full 512-frame POST +(~2 125 KiB) in one go. The challenge returns `max_outstanding_plur`, so +the client knows its ceiling before it builds the request and simply sends +smaller POSTs. This is a normal flow-control interaction, not a failure. + +Issuing the challenge only to a batch in good standing also closes a hole +an earlier draft left open: admission previously granted a reservation to +*any* EOA that could sign, since standing was established per frame inside +the spawned task. Free identities could occupy reservation-ledger entries +without ever owning a batch. Now they cannot obtain a nonce at all. + +**Nonces are stateless.** No server-side table, so no unbounded-memory DoS +from a free `GET /v1/challenge`, and no per-POST round trip that would +serialize the pipeline the scheduler exists to exploit (`inflight_max` up +to 8, `src/pusher.rs:517`). Replay within the window is bounded by the +reservation, not by nonce uniqueness. Two limits follow from making the +challenge endpoint do chain work: + +- **Rate-limit and cache `/v1/challenge`.** Amplification is bounded by + *distinct batch ids*, not request count, so cache standing per batch + under the §6 TTL and negative-cache unknown batches. Per-IP limiting on + top (§11.6's inbound limiter). +- **Bound live-reservation cardinality.** The in-memory + `account → reserved` map is attacker-influenced (one entry per batch in + standing), so cap the number of accounts holding live reservations and + shed beyond it. Persist a row only once `owed > 0`. + +**The challenge is EIP-712, not a concatenation.** The account key already +signs postage stamps (EIP-191 over stamp bytes) and cheques (EIP-712, +domain `Chequebook`). A third raw-bytes scheme over the same key invites +cross-scheme confusion. Use a typed struct with its own domain: + +```solidity +// domain: {name: "HoverflyPusher", version: "1", chainId} +struct PushChallenge { + bytes32 nonce; + string origin; // host the client dialled; relay checks its own + address account; + bytes32 batchId; + uint256 expiry; +} +``` + +`quote_valid_secs` governs the quote; the challenge gets its own, much +shorter `challenge_ttl_secs` (~300 s), since re-signing is one local +ecrecover's worth of work and a short window shrinks the replay surface to +near nothing. + +**Browser blocker:** the CORS preflight currently allows exactly one +request header, `content-type` (`src/pusher.rs:468-486`, header list at +`:481`). A custom challenge header fails preflight in every browser until +that list grows. Push *frames* are unchanged, but the challenge is still a +wire change. + +### 7.3 The quote is signed + +`/v1/status` is unsigned JSON today (`src/pusher.rs:488-528`). An unsigned +price is repudiable in both directions: the relay can serve `P` and bill +`10P`, the client can claim it saw `P/10`, and reconciliation can detect +the mismatch but never attribute it. So the `payment` block is signed with +the node-identity key — which the relay already holds and already +publishes as `overlay` (`:499`) — and the signed blob is echoed verbatim +in every 402. + +**The pin is on the node's Ethereum address, not on its overlay.** An +overlay is `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`; verifying a +signature yields the **eth address**, and the nonce is neither transmitted +nor derivable, so "pin `(url, overlay)`" is not implementable — the +recovered address and the pinned overlay are values in different spaces. +The signed block therefore carries `node_eth_address` and `overlay_nonce`, +so any client can recompute `overlay` and check it against what +`/v1/status` advertises, and clients pin +**`(url, node_eth_address, beneficiary)`**. `PUSHER_URLS` is already a +hardcoded list, so extending each entry costs nothing. + +```jsonc +// GET /v1/status → new field (the whole object is covered by `sig`) +"payment": { + "mode": "metered", + "beneficiary": "0x…", // EOA that must appear in Cheque.beneficiary + "node_eth_address": "0x…", // recovers from `sig`; client pins this + "overlay_nonce": "0x…", // 32 B; lets the client recompute `overlay` + "origin": "relay-a.example", // must equal the configured --origin (§7.2) + "chain_id": 100, + "factory": "0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420", + "price_plur_per_kib": "480000000", + "min_cheque_plur": "3900000000000", + "settle_every_plur": "15600000000000", + "max_outstanding_plur": "62200000000000", // ceiling; actual cap is per + // batch, see §10.3 + "credit_ratio": 1000, // credit line = batch value ÷ this + "quote_valid_secs": 86400, + "challenge_ttl_secs": 300, + "sig": "0x…" // node-identity key over the above +} +``` + +Parameters are derived in §9 and §10.1; the ordering +`min_cheque < settle_every < max_outstanding` is a hard invariant (§10.1). + +## 8. The billing unit: bytes admitted + +> **`owed = (kib_admitted − kib_dedup) × price_plur_per_kib`**, where +> `kib_admitted` is the body bytes the relay accepted under a valid +> challenge, rounded up to KiB. + +That is the whole billing rule. It has one property that matters more than +everything else in this document: + +> **The client cannot lie about it, because the client is the one who +> produced the bytes and the relay is the one who counted them.** + +There is no third-party attestation to forge, no signature to replay, no +chain state to disagree about, and no relay assertion the client has to +take on faith. The client committed to `Content-Length` when it built the +request; the relay counted what arrived. Both numbers are known to both +parties before any push work happens, and they must agree or the request +was malformed. + +Compare what this replaces. An earlier draft billed per *verified pushsync +receipt*, which forced a five-clause predicate anchored in the staking +registry (to stop trivial receipt forgery), a set-valued invoice (to stop +self-replay), a sampled retrieval audit (to catch fabrication), a staked-set +log sweep on both sides, and a shared predicate module that had to be +byte-identical between client and relay or it would generate disputes with +no adjudicator. All of that existed to make a *third party's* signature +into a billing input. Removing the receipt from the invoice removes the +entire apparatus and every attack against it. + +### 8.1 Why bytes rather than successful pushes + +Because bytes are what the relay spends money on. §9.1's cost basis is +egress, and egress is incurred on *attempts* — the 3-way peer race and the +shallow retries happen whether or not a chunk ultimately lands. Billing +successes would mean the relay eats the cost of every failure, which is +what made an earlier draft's "shallow-cascade arbitrage" (§11.5) an +unresolvable tension between verifiable billing and cost recovery. Billing +attempts dissolves it: the relay charges for what it spends, and the +*client* protects itself against a lane that spends without succeeding by +deweighting it in the scheduler, which is a mechanism that already exists +and already works. + +Two mechanisms, each doing what it is good at, instead of one mechanism +trying to do both and failing at the second. + +### 8.2 Dedup hits are billed at zero + +A frame served from the recent-ack cache (`src/pusher.rs:928-948`) does no +push work, so its bytes are subtracted. The ack already reports dedup, and +the `done` line carries the count (`:969-971`), so the client can +reconstruct the invoice exactly. + +This is the one place a relay assertion enters the bill — the relay claims +"this was a dedup hit". It is safe because the claim only ever *lowers* +the amount owed, so a relay has no incentive to make it falsely, and a +client that disagrees is disagreeing in its own favour. + +### 8.3 Cumulative cheques, and why not per-chunk + +A cheque is a cumulative running total for one `(chequebook, beneficiary)` +pair. Three properties fall out, all of which we want: + +- **Loss-tolerant.** A dropped or failed `/v1/pay` costs nothing; the next + cheque supersedes it. No retry state machine. +- **Replay-proof within a live relay.** Strict monotonicity means a + re-presented cheque credits zero. (Across a *restart*, see §11.4.) +- **Cheap, and gas-amortizing by construction.** One signature per + `settle_every_plur`, and the relay cashes only the *latest* cumulative, + so on-chain gas is paid once per account, not once per cheque. This is + why the dust floor (§10.1) is about RPC cost, not gas. + +**Attaching cheques to chunks or frames is rejected.** ~137 bytes per +4 KiB chunk, an EIP-712 signature on the per-chunk hot path, and — fatally +— cumulative payouts are *serial* per `(issuer, beneficiary)` pair, so +per-chunk cheques would force a total order on chunks within a lane, +destroying the concurrent multi-POST pipelining the scheduler depends on +(design §7). + +**Two lanes may share one beneficiary, and the client must handle it.** A +cumulative is per `(chequebook, beneficiary)` — the beneficiary is what +the contract keys `paidOut` on — while a *lane* is a URL, and one operator +running four lane URLs behind one beneficiary EOA is the obvious +deployment. If the client tracks cumulatives per lane, that configuration +**bricks**: `src/cheques.rs` keys `payouts` on the peer *overlay*, with a +comment explaining that the overlay is the only stable cross-run identity +(`:64-76`) — correct for bee peers, wrong for relay beneficiaries. Lane 1 +issues cumulative 10; lane 2, counting from its own zero, issues 8; the +relay applies `ErrChequeNotIncreasing` and rejects it, forever. + +> **Key the client's cumulative store on `(chequebook, beneficiary)`, not +> on lane or overlay.** Two lanes advertising one beneficiary are one +> settlement channel; sum their `owed` before signing, and send the same +> cheque to both. + +Detecting the sharing is free — the beneficiary is in the signed quote +(§7.3), so the client sees it before its first push. + +Independently, lanes with *distinct* beneficiaries still **share one +chequebook balance**, so the client must track the sum. `src/cheques.rs` +needs a `total_issued` mirroring bee's `reserveTotalIssued` +(`chequebook.go:163-178`). Without it a cheque to the second lane silently +exceeds the balance and bounces. + +### 8.4 Reconciliation + +`GET /v1/account` returns `kib_admitted` and `kib_dedup`. The client +compares them against what it sent. Because the client's byte count comes +from its own request construction and not from anything the relay returns, +this is immune to the failure mode that made an earlier draft's +reconciliation useless: ack sends are fire-and-forget +(`src/pusher.rs:855-860`), so a client that hangs up mid-stream legitimately +receives fewer acks than the relay emitted — but it still knows exactly how +many bytes it sent. There is nothing to dispute. + +A relay over-reporting `kib_admitted` is therefore immediately visible and +attributable. The client's response is to withhold the next cheque and +deweight the lane, which is §2's social enforcement rather than a protocol +mechanism, and is adequate for a pinned lane set. + +## 9. Pricing + +### 9.1 Cost basis + +Design §9's *"egress ≈ payload × 1.4× at race=1; in-pusher racing is off +by design"* is wrong on its premise. **In-pusher racing is on:** +`CHUNK_PEER_PARALLELISM = 3` (`src/client.rs:4086`), and three peers are +seeded concurrently at dispatch, each writing a full Delivery before any +receipt is read. Design §7 says so in its own words — *"the deliberate +3-way peer race"*, measured at 2.2 PO of receipt depth for 2–3× +throughput. + +Real cost per 4 KiB chunk relayed: + +| | | +|---|---| +| Delivery on the wire | addr 32 + stamp 113 + span/data ≤ 4104 ≈ 4249 B, +~5 % protobuf/yamux/noise/TCP ≈ **4.4 KiB** | +| Peer race | **×3** | +| Shallow retries at pool 128 | **×1.15** (design §10: 0.44 shallow/chunk) | +| **Egress per chunk relayed** | **≈ 15 KiB** | +| **Egress per GiB of payload** | **≈ 3.7 GiB** (262 144 chunks/GiB) | + +### 9.2 Price + +| | per GiB of payload | +|---|---| +| Cheap VPS bandwidth (~€1/TB) | ≈ €0.0037 | +| AWS egress ($0.09/GB) | ≈ $0.36 | +| **Suggested price** | **$0.02** | +| Postage (buys a *year* of storage) | orders of magnitude more | + +$0.02/GiB is ~5× a VPS's raw bandwidth cost — a real margin covering CPU, +the dedicated IP, and gas — and ~18× cheaper than AWS egress. Against what +the user already paid for postage it is a rounding error. + +**Read that AWS row as a constraint, not a favourable comparison.** At +§9.1's 3.7 GiB of real egress per GiB of payload, per-GB-billed clouds +cost the relay ~$0.33/GiB against $0.02 of revenue. Metered mode is only +rational on **flat-rate or included bandwidth** — the same class of host +§5 already requires for durable storage, and the same class §1 says +metering exists to fund. A relay on metered egress should run `open` and +eat the quota, or not run at all. + +At $0.40/BZZ: `$0.02 / $0.40 = 0.05 BZZ/GiB`; `0.05 × 10¹⁶ = 5e14` PLUR +per GiB, ÷ 1 048 576 KiB = + +> **`price_plur_per_kib ≈ 4.8 × 10⁸`** (1 BZZ = 10¹⁶ PLUR). + +A full push frame is `HEADER_LEN(147) + wire(4104) = 4251` B ≈ 4.15 KiB, +so a 4 KiB chunk costs ≈ `2.0 × 10⁹` PLUR all in. Framing overhead is +billed because the relay received those bytes. + +Flat per KiB, deliberately. Any pricing curve steeper than flat +re-introduces a per-item number for the two sides to disagree about, and +the entire point of §8 is that there is exactly one number and both +parties measure it directly. + +### 9.3 Gas, and who is actually profitable + +Cashout is `GetGasLimitWithDefault(ctx, 300_000)` (`cashout.go:145`) ≈ +**$0.0005** on Gnosis. Issuing a cheque costs nothing +(`chequebook.go:190-250` sends no transaction); only cashing out touches +the chain. Because cheques are cumulative, gas is paid **once per +account**, whenever the relay decides to cash. + +Suggested cashout threshold **0.25 BZZ ≈ $0.10 ≈ 5 GiB relayed by one +account**, which puts gas at ~0.5 % of realized revenue. + +The uncomfortable corollary: **a one-shot user is not profitable.** Design +§11's flagship metric is a 71 MB browser upload, worth +`71/1024 × $0.02 ≈ $0.0014` — about 3× the cashout gas, and only if it is +ever cashed. Accounts below the threshold that never return are written +off. Metered relay economics work on repeat and bulk accounts; cumulative +cheques are what make a returning user amortize. A relay whose traffic is +entirely one-shot browser users should run `open`. + +## 10. Credit and settlement + +**Postpaid.** The client accrues `owed` and settles when it crosses +`settle_every_plur`. Not prepaid: an anonymous browser prepaying a relay +is exposed to outright theft, whereas postpaid exposes the relay to at +most one cap of bandwidth. The asymmetry is correct — the relay's risk is +a fraction of a cent, the client's would be its deposit. + +### 10.1 Parameters, and the invariant to hold + +> **Invariant: `min_cheque_plur ≤ settle_every_plur < max_outstanding_plur`.** +> A client that is 402'd must always be able to clear it with a cheque for +> exactly what it owes. + +An early draft published `min_cheque_plur` 87× larger than +`settle_every_plur`. Every metered account would have bricked: accrue → +cross `settle_every` → sign a cheque → **rejected as dust** → keep +accruing → 402 → the only cheque that clears the 402 is 21× what is owed, +which the no-prepayment rule forbids. No exit. The error came from sizing +the dust floor against *cashout gas*, which cumulative cheques amortize +separately (§8.3). + +`min_cheque_plur` exists only to bound RPC cost per unit of value (§11.6 +lists up to 4 `eth_call`s per cheque, of which `liquidBalanceFor()` and +`paidOut()` cannot be cached). A quarter of `settle_every` is sufficient. + +At $0.02/GiB → 4.8e8 PLUR/KiB: + +| parameter | value (PLUR) | in payload | rationale | +|---|---|---|---| +| `price_plur_per_kib` | 4.8e8 | 1 KiB | §9.2 | +| `min_cheque_plur` | 3.9e12 | ~8 MiB | ≥ 4 `eth_call`s' worth of value | +| `settle_every_plur` | 1.56e13 | ~32 MiB | ~2–3 cheques per 71 MB upload | +| `max_outstanding_plur` | 6.22e13 | ~127 MiB | 4 × settle_every — a *ceiling*, not the cap; §10.3 | +| cashout threshold | 2.5e15 | ~5 GiB | §9.3 | + +The unsecured credit at risk per account is at most ~127 MiB ≈ **$0.0024**, +and less for any batch worth under ~6.2 BZZ, since §10.3 caps it at a +thousandth of the batch's remaining on-chain value. `settle_every` at +32 MiB also keeps the honest path off §11.6's amplification profile: ~2–3 +`/v1/pay` calls per 71 MB upload, not ~71. + +### 10.2 Reservation: bee's `reserve` was needed after all + +A monotone debit counter per account is not sufficient. `/v1/push` +deliberately does not serialize (`src/pusher.rs:785-846`: *"serializing +them … forced needless failover churn"*), so N concurrent POSTs each read +`outstanding` before any of them debits. + +N is now bounded — §16.3 added a 256-permit semaphore acquired before +`accept()` (`src/pusher.rs:377-415`) — but 256 concurrent full-size POSTs +is still `256 × 2 125 KiB × 4.8e8 = 2.6e14` PLUR, **4× +`max_outstanding_plur`**. The connection cap turns an unbounded overshoot +into a merely large one; it does not remove the need for a reservation. A +*polite* client at the relay's own advertised `inflight_max` of 8 +overshoots a 4-×-`settle_every` cap on its own. + +Fix, per §7.2: reserve `ceil(Content-Length / 1024) × price_plur_per_kib` +atomically at admission and release the unused remainder at completion — +bee's `reserveTotalIssued` applied to the receiving side. + +There are four coupled per-account quantities, not one: `owed`, +`reserved`, `last_cumulative[chequebook]`, and the `chequebook → account` +binding. They are written by N spawned tasks and read by admission, so +they need one lock. + +**But `reserved` must not be persisted.** A reservation belongs to an +in-flight POST, and no in-flight POST survives a restart — there is no +task left to release it. Restoring `reserved` from disk leaks credit +permanently and can brick an account into 402 with no way out, which is +exactly what §10.1's invariant exists to prevent. + +> **Persist `owed`, `last_cumulative` and the chequebook binding +> atomically. Reconstruct `reserved` as zero at boot.** + +The exposure from zeroing is one body's worth of over-admission +immediately after a restart — cents of egress, against an accounting +corruption that never self-heals. `owed` is written at batch completion, +so a crash forfeits at most the batch in flight, the same safe direction. + +### 10.3 Sybil bound: derive the credit line, don't assert it + +"An account is a batch owner, a live batch costs real BZZ, so the margin +is three orders of magnitude" is false. The relay checks **liveness**, and +liveness is satisfied by the cheapest batch the `PostageStamp` contract +will accept — minimum depth, minimum validity — which costs a fraction of +a cent. At a flat $0.0024 credit line the real margin is of order **1×**: +a throwaway batch buys roughly its own value in free relay. + +A hard floor on batch depth and TTL would fix the arithmetic at the cost +of excluding small legitimate users, which is the wrong trade for a system +whose flagship case is a 71 MB browser upload. So instead of gating on +batch size, **scale the credit line to it**: + +> **`max_outstanding(A, B) = min(remaining_value_plur(B) ÷ credit_ratio, +> max_outstanding_plur)`** + +with `credit_ratio = 1000` and `remaining_value_plur` from §6's standing +read. The Sybil margin is then **1000× by construction and independent of +batch size**: any attacker, holding any mix of batches, obtains total +credit equal to one thousandth of the on-chain value they actually funded. +There is no cheap corner of the parameter space, because the ratio is the +invariant rather than a consequence of a particular batch being expensive. + +Concretely: a batch needs ~6.2 BZZ (≈ $2.49) of remaining value to +saturate the global $0.0024 ceiling; a 0.01 BZZ dust batch earns +`1e11 ÷ 4.8e8 ≈ 208 KiB` of credit — enough to be useful, far too little +to farm. The line decays on its own as the batch is spent down or +approaches expiry, and §6's TTL is what makes that decay visible. + +This is the same move as §8's: replace an asserted constant with a +quantity read from chain, so the property holds by construction rather +than by assumption about what attackers will bother to buy. + +## 11. Attack surface + +Everything here is **a client attacking a relay**, per §2. Severity is +from the relay's point of view. "Inherited" means the issue exists in +bee's SWAP too; "introduced" means metering creates it. + +### 11.1 CRITICAL — stamp replay becomes billing griefing (introduced) + +**Swarm stamps are public.** Design §6 says so — *"anyone who saw your +chunks holds valid `(addr, stamp, wire)` triples"* — and they are +recoverable from the network, since retrieval returns the stamp alongside +the data (`src/protocols/retrieval.rs:27`, `:42-80`). A relay holds every +stamp it has ever relayed, so it is the densest source of all (§2). + +In open mode this is harmless: re-pushing is idempotent. **Under metering +it is an attack.** An attacker harvests a victim's stamps and replays them +at a metered relay. Auth passes — the stamps genuinely recover to the +batch owner — so the work is billed to the *victim's* account. Cost to the +attacker: zero. + +**Mitigation, mandatory:** the challenge of §7.2, plus §6's rule that +every frame must carry the exact batch id named in the challenge. The +challenge must be *signed by the account*, so possession of a harvested +stamp is not enough — the attacker would need the batch owner's key, which +is the same key that signs the stamps it is replaying. + +**The signed payload binds the relay's origin, not its beneficiary.** An +earlier draft signed `(nonce ‖ beneficiary ‖ account)` so a challenge could +not be replayed across relays. That fails: nothing authenticates a +beneficiary, so relay A can advertise honest relay B's beneficiary *and* +serve B's nonce, collect a victim's signature during a normal upload +through A, and replay it at B alongside V's stamps. Bind **`origin`** — +the host the client dialled — and have the relay verify it against its own +**configured** hostname (§7.2). Deriving it from a request header makes +the binding a no-op. + +### 11.2 CRITICAL — the withdraw race: cheques are unsecured (inherited) + +bee deploys chequebooks with a hard-deposit timeout of zero +(`init.go:169` passes `big.NewInt(0)` to `Deploy`, whose parameter is +`defaultHardDepositTimeoutDuration`, `factory.go:35`, `:62`) and never +places hard deposits, so the whole balance stays liquid and the issuer can +`withdraw()` at any time. The relay's funding check is therefore true **at +acceptance time, not at cashout time**. + +Attack: accrue debt → sign a covering cheque → relay verifies and keeps +serving → withdraw the chequebook → the relay's cashout collects less than +it is owed. Bee has the identical exposure; it is inherent to +SimpleSwap-as-deployed. + +**What reading `ERC20SimpleSwap` directly established:** + +*Cashout does not revert.* `_cashChequeInternal` pays +`totalPayout = Math.min(requestPayout, liquidBalanceFor(beneficiary))`, +credits `paidOut[beneficiary]`, and only then, if +`requestPayout != totalPayout`, sets `bounced = true` and emits +`ChequeBounced()`. An underfunded cashout **succeeds partially** and takes +whatever is there. The exposure is the shortfall, not the whole claim, and +`bounced` is *readable contract state* set contract-wide, so one bounce +marks the chequebook permanently. + +*Hard deposits work, but are inert as bee deploys them.* `withdraw()` is: + +```solidity +function withdraw(uint amount) public { + require(msg.sender == issuer, "not issuer"); + require(amount <= liquidBalance(), "liquidBalance not sufficient"); + require(token.transfer(issuer, amount), "transfer failed"); +} +``` + +with `liquidBalance() = balance() − totalHardDeposit`, so a deposit +genuinely locks funds. But decreasing uses + +```solidity +uint timeout = hardDeposit.timeout == 0 ? defaultHardDepositTimeout : hardDeposit.timeout; +hardDeposit.canBeDecreasedAt = block.timestamp + timeout; +``` + +and `decreaseHardDeposit` requires +`block.timestamp >= canBeDecreasedAt && canBeDecreasedAt != 0`. With bee's +`defaultHardDepositTimeout = 0` and a fresh deposit's `timeout == 0`, both +conditions hold in the *same block*: prepare, decrease, withdraw. A hard +deposit on a bee chequebook is this attack with two extra calls in front +of it. + +`setCustomHardDepositTimeout(beneficiary, timeout, beneficiarySig)` is the +way out — issuer-submitted, but requiring an EIP-712 signature **from the +beneficiary**, and the hash binds `address(this)`, the client's chequebook, +so it cannot be pre-signed. Secured mode therefore costs one interaction +with the cold beneficiary key per client chequebook. Deferred to §14 +Stage 2 as an account-tier feature, not a default. + +**Mitigations, in force:** + +1. **The per-account cap** (`max_outstanding_plur`), bounding the yield to + $0.0024 per account and one postage batch per account. This is the real + defence — **and it only works if §10.2's reservation ships.** +2. **Cash out at the threshold and treat sub-threshold value as at risk.** + At 5 GiB/account (§9.3) most accounts are never cashed, so the exposure + is chronic rather than acute. Bounded by mitigation 1, not by + promptness. +3. **Blocklist bouncing issuers.** Read the `bounced` flag — a cheap + `eth_call`, not a log subscription you had to be watching at the time. + +Do not treat an accepted cheque as settled revenue. It is a claim. + +### 11.3 HIGH — Sybil beneficiaries and aggregate exposure (introduced) + +Nothing proves a relay controls the beneficiary EOA it advertises; that +would require holding the key, destroying the property in §6. Since a +cumulative is per `(chequebook, beneficiary)`, **one operator running N +lane URLs with N beneficiary EOAs presents N independent accounts**, and +the client has no aggregate view until `src/cheques.rs` grows +`total_issued` (§8.3). + +Under §2 this is not a relay-fraud concern — it is a client-side budgeting +concern, and the fix is client-side: `total_issued` as a hard ceiling +across all beneficiaries, and pinning `(url, node_eth_address, beneficiary)` +in config rather than reading the beneficiary from `/v1/status` at runtime. + +### 11.4 HIGH — relay state loss is an unbounded free-service loop (introduced) + +If the relay loses `last_cumulative[chequebook]`, a client re-presents its +most recent cheque and is credited the full cumulative instead of the +delta. + +Seeding `last_cumulative` from on-chain `paidOut(beneficiary)` is inert at +§9.3's parameters: with a 5 GiB cashout threshold, almost no account is +ever cashed, so `paidOut` is permanently 0 and the seed is 0. The attack +is then: pay one cheque, consume service, wait for a restart, re-present +**the same cheque**, repeat — unbounded free service from a single +signature at zero incremental cost. Restarts are not incidental on free +tiers that sleep and cold-start (design §7 measured a 35.2 s wake). + +**Mitigation: metered mode requires durable storage** (§5). Persist +`owed`, `last_cumulative`, the `chequebook → account` binding **and +`relay_secret`** (§7.2) atomically together — losing only +`last_cumulative` over-credits, losing only `owed` under-bills, losing +`relay_secret` 403s every live client. Seed from `paidOut` on first run as +a floor, never as the primary source. A relay that cannot guarantee +durable state must run `open`. + +**`reserved` is deliberately excluded** and reconstructed as zero at boot +(§10.2). + +### 11.5 MEDIUM — shallow-cascade cost griefing (introduced) + +For a chunk nothing will store: 3 peers are raced immediately +(`src/client.rs:4086`); candidates are topped up to +`cap = max_retries.min(order.len())` with `PUSH_MAX_RETRIES = 20` +(`src/pusher.rs:94`, applied at `src/client.rs:4315`); if every outcome is +shallow or overdraft with no hard error the dispatcher **walks past `cap`** +over the whole eligible pool (`:4573-4602`, up to 128 sessions); +shallow-only then returns a *retryable* outcome and the outer loop +re-dispatches up to `MAX_CHUNK_RETRIES = 60` (`:4104`). The attacker aims: +the relay publishes its overlay (`src/pusher.rs:499`), `pool.live` +(`:511-514`) and the `pool_po` mean inside `diag` (`:525`), so addresses +can be mined into arcs the pool covers poorly. Design §10 measured 8.1 +shallow receipts/chunk at pool 32 as the *accidental* rate. + +**Bytes-admitted billing prices most of this away.** The attacker pays for +every byte it sends regardless of outcome, so it can no longer buy cheap +egress by aiming at badly-covered arcs — the arbitrage that made this +economically rational under receipt billing is gone. What remains is pure +griefing: the attacker burns its own credit to make the relay burn more. +The amplification factor is the cost control that matters, so metered mode +caps delivery *attempts* per chunk well below open mode's fallback and +disables the past-`cap` walk. + +### 11.6 HIGH — RPC amplification on `/v1/pay` (introduced) + +Accepting a cheque costs up to 4 `eth_call`s. Order the checks +cheapest-first and reach the chain only after every free check passes: + +1. Parse and length-check. +2. **Reject non-canonical signatures**: length ≠ 65, + `s > secp256k1n/2`, or `v ∉ {27, 28}`. Free, and mandatory — see below. +3. EIP-712 recover (local, ~80 µs). +4. `beneficiary == ours`, `chain_id == ours`. +5. `cumulative > last_cumulative[chequebook]` and + `amount ≥ min_cheque_plur`. +6. *Then* RPC: `deployedContracts` (cache forever per chequebook), + `issuer()` (**cache per chequebook** — bee refetches it on every cheque + at `chequestore.go:149` despite its own comment saying it never + changes; do not copy that), `liquidBalanceFor(ourBeneficiary)` and + `paidOut(ourBeneficiary)` (uncacheable — they are the §11.2 check). + Read `bounced` in the same batch and refuse the chequebook if set. + +**That ordering bounds nothing on its own.** Every "free" check passes for +a cheque an attacker synthesizes at zero cost: it reads `beneficiary` and +`chain_id` from the public quote, picks a random chequebook address, signs +with a throwaway key — step 3 recovers *some* address, and nothing +compares it to anything until `issuer()` — and sets `cumulative` above +`min_cheque_plur`, which clears step 5 trivially because `last_cumulative` +for an unseen chequebook is zero. Each garbage POST costs one +`deployedContracts` call. Cheapest-first is a latency optimisation; the +bound must come from somewhere the attacker cannot reach: + +- **No debt, no cheque.** Refuse any cheque for an account with + `owed == 0`, before parsing. Postpaid (§10) means an honest client + always has debt by the time it settles, so this costs nothing legitimate + and makes the endpoint useless to anyone who has not first done billable + work under a challenge. +- **Require the challenge header on `/v1/pay`**, as on `/v1/push`. The + caller must hold a batch in standing to reach any code path, pricing the + amplifier at one live postage batch. +- **Negative-cache non-deployed chequebooks** — bounded LRU with a TTL, + the structure the owner cache grew in §16.2. +- **Rate-limit per account.** **`src/ratelimit.rs` cannot be reused** — it + is a per-peer *outbound libp2p dial* GCRA pacer that parks rather than + refuses (`src/ratelimit.rs:1-30`), with no inbound, per-account or HTTP + concept. Stage 1 needs a new inbound limiter, also covering + `/v1/challenge`. + +**On step 2 — this is a validity check, not a malleability nicety.** +`ERC20SimpleSwap.recoverEIP712` calls `ECDSA.recover` from +`@openzeppelin/contracts/cryptography/ECDSA.sol` at `^3.4.1-solc-0.7-2` +(the repo's `package.json`), which is unconditionally strict: + +```solidity +require(uint256(s) <= 0x7FFF…5D576E7357A4501DDFE92F46681B20A0, "ECDSA: invalid signature 's' value"); +require(v == 27 || v == 28, "ECDSA: invalid signature 'v' value"); +``` + +plus a 65-byte length check and a zero-address check. `alloy` recovers +high-`s` and `v ∈ {0,1}` happily, so **a client can issue cheques that +verify off-chain, buy service, and revert at cashout** — free relay, with +the relay's own acceptance as the evidence it was paid. OZ's comment notes +the alternative of rewriting as `(n − s, v^1)`, but there is no reason to +accept a client that cannot produce a canonical signature, and `alloy` +always does, so rejection costs honest clients nothing. + +### 11.7 MEDIUM — hedging costs money (introduced) + +The scheduler hedges stragglers to a rank-#2 lane (design §7). Both relays +receive the bytes, so both bill. The recent-ack cache does not absorb it: +`push.recent` is populated only **after the whole batch drains** +(`src/pusher.rs:1050-1065`), not in the per-chunk callback (`:1000-1025`), +so two concurrent POSTs containing the same address — including the +design's own *"re-POST unacked frames"* retry model (design §3) — are not +deduped even on the same relay. + +Under §8 this is not a dispute: the client knows it sent those bytes +twice, and it did. **`hedge_fraction` becomes an economic parameter, not +just a latency one**, and clients should lower it against metered lanes. + +### 11.8 MEDIUM — mixed-account batches bypass admission (introduced) + +`run_push` handles frames from different batches with different owners in +one POST (`src/pusher.rs:905-925`), while §7.2's admission necessarily +picks one account. Prefix one frame from a good-standing account and fill +the remaining 511 from a 402'd account: admission passes, billing lands on +the wrong account. Closed by §6's one-batch-per-request rule. + +### 11.9 LOW + +| Threat | Assessment | +|---|---| +| Cross-relay cheque replay | Impossible: `beneficiary` is inside the signed EIP-712 struct. | +| Cross-chain replay | Impossible: `chainId` is in the domain separator; the relay pins its own. | +| Reorg invalidating `deployedContracts`/`liquidBalanceFor` | Negligible on Gnosis at these values; re-verified at cashout. | +| Paying off someone else's debt | Requires their chequebook key. Not a threat. | +| Price change mid-flight | Bill at the signed quote in force when the work was done; `quote_valid_secs` must exceed one settlement period (§10.1's sizing gives 24 h against a ~32 MiB period). | +| `/v1/account` as a volume/targeting oracle | Closed by requiring the challenge header (§7). | +| Challenge replay inside its own window | Bounded by the reservation (§7.2), a 300 s `challenge_ttl_secs`, and origin binding. Only the relay sees the header over TLS, so it must never be logged or echoed back. | +| Client under-reports bytes | Not possible. The relay counts what it received; `Content-Length` is a commitment the client made before the relay did any work. | +| Client disputes the invoice | Nothing to dispute — §8.4. | + +## 12. Client scheduler integration + +- `LaneInfo` (`src/pushsched.rs:101-112`) gains `price_plur_per_kib` and + `mode`. All five existing fields are already `Option`, so parsing is + free — but *scheduling* is not: `weight()` (`:257-278`) multiplies + `rate × budget × concurrency` against a `MIN_WEIGHT` floor (`:65-68`), + and an unknown price has no safe default. Treating it as free prefers + unadvertised lanes; treating it as expensive pushes warming lanes under + `MIN_WEIGHT`, where they can never revise their EWMA — the exact failure + `MIN_WEIGHT` exists to prevent. Probably "unknown price ⇒ treat as the + median of known lanes". +- **A 402 needs its own outcome.** `BatchOutcome` is + `Answered | Failed(String)` (`:304-310`), so a 402 maps to `Failed` → + `fail_streak++` → `Backoff` → `Retired` after 5 doublings (`:60-63`, + `:88-91`). "Pay, then retry" would cost lane health on every routine + settlement and retire a healthy lane mid-upload. Add + `BatchOutcome::PaymentRequired`, which pauses the lane without touching + the streak. +- **"Cannot pay" needs a non-terminal ineligible state.** `eligible()` + (`:228-234`) is a pure function of `LaneHealth`, and the only ineligible + terminal state is `Retired`, which is permanent for the run — wrong for + "temporarily out of chequebook balance". +- **POST sizing must respect `max_outstanding_plur`.** The challenge + returns the cap; the client sizes its body to fit rather than + discovering the ceiling as a 402 (§7.2). +- Lower `hedge_fraction` against metered lanes (§11.7). +- Lane weighting should prefer `open` lanes when both are eligible and the + client has no chequebook — otherwise a browser client with no payment + path spends its first POST discovering a 402. +- **Lane health is the client's protection**, per §2. A lane that takes + bytes and delivers poorly shows up in the existing EWMA and gets + deweighted. No new mechanism. + +## 13. Rejected alternatives + +**Billing per verified pushsync receipt.** The previous design. Rejected +in §8: it makes a third party's signature into a billing input, which +forces a staking-registry anchor, a set-valued invoice, a sampled +retrieval audit, and a shared predicate module that must not drift between +the parties — and after all of it, a receipt still carries no freshness, +so a relay could replay one for content already in the swarm and no +off-chain check could tell. Bytes-admitted deletes the input and the +entire apparatus with it. + +**Per-chunk / per-frame cheques.** §8.3. Serial by construction; kills +multi-lane pipelining. + +**Billing per ack.** Every field of an ack is a relay assertion +(`src/pusher.rs:1005-1020`). Under §2's trust model this is *nearly* +acceptable — but bytes-admitted is strictly better at the same cost, since +the client can verify it without trusting anything. + +**Relay earns like a bee forwarder.** Bee's ledger does pay forwarders — a +forwarder earns `(PO_next − PO_self) × 10 000` per chunk — but cheques are +issued *only* against `originatedBalance` +(`bee/pkg/accounting/accounting.go:472-484`). Non-originated debt settles +purely in free, rate-capped refreshments, so the spread is never +monetised. + +**Postage in kind** (client stamps chunks the relay names). Pays in +storage for a service whose cost is bandwidth. Cute, weak. + +**Direct on-chain BZZ prepay.** Simpler — no chequebook deployment — but +gas on every top-up and no off-chain micropayments. Kept as a fallback if +chequebook onboarding proves fatal in the browser. + +**A shared multi-tenant escrow contract.** Would remove per-user chequebook +deployment: one contract, many depositors, cumulative vouchers to any +beneficiary. Attractive UX, but a new contract to write, audit and +maintain, forfeiting reuse of an audited deployment. Not worth it unless +per-user deployment measurably kills conversion. + +**Cheque-as-credential, never cashed.** The relay accepts cheques purely +as funded, non-replayable proof of identity and never cashes them. Zero +cashout machinery, zero gas, no §11.2 exposure, no §9.3 profitability +problem — and it subsumes `--push-challenge`. The cost is that it prices +nothing: it gates abuse but does not fund the dedicated egress IP §1 says +is actually scarce. Kept as the fallback if Stage 0 shows consumption too +thin to bother metering. + +## 14. Staged plan + +**Stage 0 — metering instrument, zero protocol change. Shipped: +`src/meter.rs`.** Shadow accounting in the relay: per-`(owner, batch)` +bytes admitted, what they would owe at §9.2's candidate price, and the +credit line §10.3 would have granted each batch. Nothing is billed, no +request is refused, and no client-visible behaviour changes. + +Where to read it: + +- **`GET /v1/status` → `meter`** — aggregates only. Accounts and batches + seen, KiB admitted and deduped, total owed in PLUR and USD, how many + accounts would reach one settlement or the cashout threshold, the + observed egress multiplier, and the credit-line distribution. Names + nobody, so it stays public alongside the `bytes_pushed` total already + published there. +- **`GET /v1/meter`** — per-account and per-batch rows. **Open by default**; + set `HOVERFLY_PUSH_METER_TOKEN` to require an `Authorization: Bearer` + header. Everything in it derives from state that is already public — + batch owners and balances are on-chain and enumerable from + `BatchCreated`, and the stamp on every relayed chunk names its batch, + which retrieval hands back (§2) — so it adds only relay attribution and + timing. + + **This changes at Stage 1, for a reason worth stating precisely.** §7 + authenticates `/v1/account` on two grounds, and only the second is load + bearing: "per-identity volume oracle" is weak against a network where + stamps are public anyway, but "targeting oracle for tipping a victim into + 402 at a chosen moment" is real — knowing an account's *outstanding* + balance lets an attacker time a stamp replay (§11.1) to break a victim's + upload at a chosen instant. That is an active attack enabler, not a + privacy leak, and it does not exist in Stage 0 because there is no 402 + and no cap enforcement. Do not carry the Stage 1 conclusion backwards. + +Three properties worth knowing when reading the output: + +- **Costs no extra RPC.** `resolve_owner` already reads both `read_batch` + and `read_remaining_balance` to check for expiry, so the batch's total + value (`remainingBalance × 2^depth`) simply rides along on the cached + success rather than being discarded. +- **Costs one lock per POST**, not per frame: a request accumulates into a + stack-local `PostTally` and merges once at completion. +- **Bounded and in-memory.** 4 096 `(owner, batch)` rows with FIFO + eviction, for the same reason the owner cache is bounded (§16.2) — the + key is attacker-chosen. Evicted rows keep contributing to the totals, so + headline volume stays honest; only their detail is lost. State resets on + restart, which biases "do accounts return?" downward — check + `window_secs` before drawing conclusions. + +**What gates Stage 1:** the **distribution of batch remaining value across +accounts**, reported as `credit.batches_below_one_full_post` and the +`credit_kib` percentiles. It calibrates `credit_ratio` (§10.3) — +specifically, what fraction of real users would be capped below the global +ceiling, and whether §7.2's POST-sizing interaction bites in practice. + +Stage 0 also settles §9.1's cost basis, which the doc currently *models* +rather than measures: `egress.attempts_per_frame` against the modelled +3.45, since the push path counts every per-stream attempt including losing +racers and shallow retries (`src/client.rs:5361-5363`). A materially +different number moves the price in §9.2. + +*(The previous design had four gating measurements including a kill +criterion — the staked fraction of receipt signers, which could have +vetoed metering outright. Bytes-admitted removes the dependency.)* + +**Stage 1 — the relay can be paid (native client only), soft mode.** +**Relay side shipped.** Enabled with `--meter --origin --beneficiary +<0x…> --state-dir `; without `--meter` nothing below is reachable and +the relay behaves exactly as it does today. Modules: `src/challenge.rs`, +`src/ledger.rs`, `src/metered.rs`, `src/inbound_limit.rs`, plus cheque +recovery in `src/signer.rs`, a `SignedCheque` decoder in +`src/protocols/swap.rs` and chequebook bindings in `src/batch.rs`. +Endpoints: `GET /v1/challenge`, `GET /v1/account`, `POST /v1/pay`, and +metered admission on `POST /v1/push`. + +**Client side shipped** (`src/payer.rs`): signed-quote verification with +lane pinning on `(url, node_eth_address, beneficiary)`, challenge parsing +and signing, POST sizing against the returned cap, and local `owed` +tracking computed from bytes sent. `BatchOutcome::PaymentRequired` and a +non-terminal `LaneHealth::Unfunded` are in `src/pushsched.rs`, and +`src/cheques.rs` gained `total_issued` plus a `relay:` key namespace. + +Two things remain before hard mode: the driver does not yet *act* on a 402 +by issuing a cheque and calling `Scheduler::fund_lane` (the pieces exist +and are tested; the loop that connects them does not), and the `/v1/pay` +chain reads have never been exercised against a real chequebook. Soft mode +is what makes that ordering safe: a relay can run `--meter` against today's +clients without breaking them, because nothing is refused. + + +- Byte accounting per account with a **body-bounded reservation** (§7.2, + §10.2), durably persisted except `reserved` (§11.4). +- EIP-712 **recovery** in `src/signer.rs`, mirroring `sign_cheque` + (`:309-337`). The `sol!` `Cheque` type exists at `:21-27`; the + `Eip712Domain` is built inline at `:321-327` and needs extracting to be + shared. +- `SignedCheque` JSON **decoder** in `src/protocols/swap.rs` — only the + encoder exists (`encode_signed_cheque_json`, `:84-106`). It must reject + non-canonical signatures at parse time (§11.6): it is the one place + every cheque passes through. +- `ERC20SimpleSwap` + factory `sol!` bindings — `issuer()`, + `liquidBalanceFor(address)`, `paidOut(address)`, `bounced()`, + `deployedContracts(address)`. `EthRpc::call_view` (`src/batch.rs:726`) + is **private**, so these either live in `batch.rs` or it gains + visibility. +- **Re-key `src/cheques.rs` on `(chequebook, beneficiary)`** instead of + peer overlay (`:64-76`), and add `total_issued` (§8.3). +- Stateless MAC challenge with standing baked in, over a length-prefixed + domain-tagged preimage compared in constant time; EIP-712 + `PushChallenge` verification; `relay_secret` persisted with the ledger + (§7.2). CORS `allow-headers` update (`src/pusher.rs:468-486`). +- Signed quote carrying `node_eth_address`, `overlay_nonce` and `origin` + (§7.3); a new inbound per-account rate limiter (§11.6), also covering + `/v1/challenge` and `/v1/pay`. +- `/v1/pay` hardening: challenge header required, `owed == 0` refused + before parsing, non-deployed chequebooks negative-cached (§11.6). +- One-**batch**-per-request enforcement; batch *standing* reads with a TTL + on the owner cache (§6); credit line derivation (§10.3). +- Metered retry policy (§11.5). +- Forward the pushsync receipt into the ack as **telemetry** (§2). Not + billing-relevant, but independently the first cryptographic evidence a + relay client has ever had, and it feeds lane weighting. + +Pusher flags: `--meter`, `--origin` (**required under `--meter`**; §7.2, +§11.1), `--beneficiary`, `--price-plur-per-kib`, `--settle-every-plur`, +`--max-outstanding-plur`, `--min-cheque-plur`, `--credit-ratio`, +`--state-dir`. + +Client: the native path already has `--chequebook` / `--cheques-file` / +`--chequebook-chain-id`, so it needs the 402 handler, the challenge +signature, POST sizing against the returned cap, lane pinning on +`(url, node_eth_address, beneficiary)` (§7.3), and `total_issued` (§8.3). + +**Stage 2 — hard mode and cashout.** 402 enforcement; scheduler changes +(§12); a standalone `hoverfly cashout` run from a machine holding the +beneficiary key, never the relay. Cashout reads `bounced` and +`liquidBalanceFor` rather than `balance` (§11.2), and optionally offers +**secured mode** for high-volume accounts: the beneficiary signs a +`setCustomHardDepositTimeout` for that client's chequebook, the client +calls `increaseHardDeposit`, and the cheque stops being an unsecured +claim. Interactive by nature — it needs the cold key once per client +chequebook — so it is an account-tier feature, not a default. + +**Stage 3 — browser.** Deferred. Unblockers: export `sign_cheque` through +`#[wasm_bindgen]` (it already compiles on wasm — `src/signer.rs` has no +`cfg` gates — it is simply not bound in `src/wasm.rs`); an IndexedDB or +localStorage cumulative store, anticipated at `src/cheques.rs:24-26`; and +an opt-in dApp flow where the wallet deploys a chequebook with +`issuer = sessionKey` and funds it — two transactions on top of today's +approve + `createBatch`, ≈ $0.001 of Gnosis gas, one-time per user and +reused across every upload and lane. The dApp's four public lanes stay in +`open` mode throughout, so §9.3's one-shot-user problem does not bite +them. + +One Stage 3 hazard Stage 1 does not have: **a session-key chequebook can +strand funds permanently.** `withdraw()` is issuer-only, and the issuer +here is a key living in localStorage. Clearing site data destroys the only +key that can recover the remaining balance. Fund in small increments, warn +on deposit, and offer an explicit "drain chequebook" action before the key +is discarded. A wallet-owned chequebook with the session key as a mere +*signer* would avoid this, but SimpleSwap has no such split. + +## 15. Residual risks and open questions + +Carried knowingly: + +- **§11.2** — cheques are unsecured claims, bounded by the per-account cap. + Hard deposits work but are inert as bee deploys them; secured mode costs + a cold-key signature per client chequebook. +- **§9.3** — one-shot users do not cover their own cashout gas. +- **§11.3** — aggregate exposure across beneficiaries is invisible to the + client until `total_issued` ships. +- **§11.5** — griefing by aiming at badly-covered arcs still forces the + relay to spend more than the attacker pays, even though the attacker now + pays. Bounded by attempt caps, not eliminated. +- **A relay can take payment and drop chunks.** Out of scope by §2 and + governed socially: lanes are pinned in `PUSHER_URLS` and a + misbehaving one is removed. This is adequate for a curated set and + **not** adequate for an open one. + +Open: + +- What would it take to open the relay set? The honest answer is a + freshness-bearing proof of delivery, which pushsync does not provide — + the storer signs the bare chunk address + (`bee/pkg/pushsync/pushsync.go:277`), so a receipt is valid forever, for + everybody, and a relay can replay one for content already in the swarm. + Two upstream asks would fix it, in increasing order of difficulty: + **(a)** a read-only "do you hold `(address, batch)`?" query — bee's + reserve is *already* indexed exactly that way + (`BatchRadiusItem.ID() = BatchID ‖ Bin ‖ Address ‖ StampHash`, + `pkg/storer/internal/reserve/items.go:35-37`), so this is a lookup on an + existing index; **(b)** having the storer sign + `keccak(chunk_address ‖ stamp_hash)` instead of the bare address. Both + have value well beyond metering — any uploader could prove *its own* + stamp reached a neighbourhood — and (a) is a much easier sell. +- Price discovery across a federation: fixed per-operator quotes, or + something the client aggregates? +- Should a relay accept cheques from a chequebook whose `issuer()` is not + the batch owner, via a signed authorization (§6)? Needed by anyone + uploading under multiple batch owners. +- Is the batch owner the right account key at all, given §6's TTL problem + and the fact that batches expire? +- Is `credit_ratio = 1000` (§10.3) right? Chosen for a comfortable margin, + not derived. Stage 0's batch-value distribution should justify it or + move it. + +## 16. Found during review: six live open-mode bugs (all fixed) + +All independent of metering, all in code running in production. They are +recorded here rather than in a commit message because the design argument +keeps citing them: §6, §10.2 and §11.6 all describe relay behaviour these +fixes changed. + +### 16.1 The recent-ack cache let one uploader substitute another's stamp + +**Status: fixed** — `RecentAcks` is now keyed on `(addr, batch_id)` +(`src/pusher.rs:286`, applied at `:928-948` and `:1050-1065`). + +Dedup was keyed on `chunk.addr` alone. On a hit the frame is acked +`{"s":"ok"}` and is **never added to `accepted`** — the submitted stamp +never reaches the swarm. Swarm addresses are content-derived, so two +uploaders of the same bytes collide by construction. + +Attack: push chunk X stamped with a dust batch expiring tomorrow. Within +`RECENT_ACK_TTL_SECS = 120` (`:129`) the victim pushes X stamped with its +year-long batch, is told `ok`, and the chunk lives under the attacker's +dying stamp. When that batch expires the chunk is garbage-collected and +the victim's manifest 404s. The victim has a successful upload record and +no way to know. The same thing happens *accidentally* between two honest +users uploading the same file within 120 s on one lane. + +Two notes on the applied fix, neither a defect today: + +- The completion path resolves the batch via `batch_of.get(a)` with an + all-zero sentinel fallback (`:1050-1065`). No admitted chunk currently + reaches it unmapped, but a sentinel that could in principle alias a + batch id is worth replacing with an `if let Some` that skips caching. +- If one POST carries the same address under two different batches, only + the last-inserted mapping is cached. That under-dedups rather than + mis-dedups — the safe direction — and metered mode forbids the case + outright under §6's one-batch-per-request rule. + +### 16.2 `/v1/push` was an unauthenticated RPC amplifier + +**Status: fixed.** `resolve_owner` cached only *successes*, in an unbounded +`HashMap`, with no per-request budget. A batch id that failed to resolve — +not found, or zero remaining balance — was re-resolved on every mention, +so one anonymous POST of `PUSH_BATCH_MAX` frames naming bogus batches +became that many serial `eth_call`s against the operator's RPC endpoint. +Nothing about the request needed to be valid: the amplification happens +*before* any push work. `EthRpc::new` made it worse by building a fresh +`reqwest::Client` per read, so every call also paid a new connection pool +and TLS handshake. + +Fixed by a bounded `OwnerCache` (`src/pusher.rs:181-239`, `:1106`) that +caches definitive rejections under a shorter TTL than successes but never +caches transport errors — an RPC blip must not blacklist a live batch — is +capped at 4 096 entries with FIFO eviction, and by a per-request budget of +`PUSH_MAX_BATCH_LOOKUPS = 8` distinct lookups. `EthRpc` now shares one +process-wide client (`src/batch.rs`). + +This is the same defect §11.6 identifies in metered `/v1/pay`, which is +why that section stopped treating cheapest-first ordering as a bound. + +### 16.3 The accept loop had no connection limit + +**Status: fixed.** The loop spawned per connection with no semaphore and +no cap anywhere, despite design §3 listing one as table stakes. Now a +`PUSH_MAX_CONNS_DEFAULT = 256` semaphore permit is acquired **before** +`accept()` and held for the connection's life (`src/pusher.rs:377-415`, +`HOVERFLY_PUSH_MAX_CONNS` to override). Acquiring before accepting is what +applies backpressure to the kernel queue instead of admitting everything +and queueing internally. + +§10.2's reservation argument is written against the fixed behaviour: 256 +is a bound, and it is still 4× `max_outstanding_plur`. + +### 16.4 One transient accept error killed the relay + +**Status: fixed.** `listener.accept().await?` propagated out of `run`, so a +single `EMFILE` or `ECONNABORTED` — both transient and both routine under +load — terminated the process. The loop now logs, backs off exponentially +to `ACCEPT_BACKOFF_MAX_MS = 1000`, and continues. + +### 16.5 There were no HTTP timeouts at all + +**Status: fixed.** The pre-existing comment said hyper's defaults were +fine. They are not, and the reason is worth recording: `header_read_timeout` +is **silently inert unless a timer is installed**. Without `.timer(...)` +hyper's `Time` is `Empty`, its `check()` returns `None`, and the +configured timeout never fires — so the code read as protected while +accepting connections that could hold a slot forever sending one header +byte at a time. + +Now `TokioTimer` is installed, `HEADER_READ_TIMEOUT_SECS = 30` applies, +and the `/v1/push` body read is wrapped in a +`PUSH_BODY_READ_TIMEOUT_SECS = 120` `tokio::time::timeout` returning +`408 Request Timeout`. Combined with §16.3's cap, a slowloris now costs an +attacker 256 connections for 30 seconds rather than indefinitely. + +### 16.6 Pushsync receipt addresses were never checked + +**Status: fixed** (`src/protocols/pushsync.rs:142-148`, with regression +tests). + +`receipt.address` was copied straight off the wire and never compared to +the address that was pushed. Two things followed, and any peer in the pool +could reach both — pool membership comes from hive gossip and the seed +list, so it is not a trusted set: + +- **Remote panic.** Callers build a `[u8; 32]` from it with + `copy_from_slice` (`src/client.rs:5380`, `:5398`), which panics on any + other length. A 0- or 33-byte address unwound the push task. +- **Address substitution.** A peer could accept the Delivery, store + nothing, and sign a receipt for some *other* address deep inside its own + neighbourhood. `is_shallow` and the `po` computation both read + `r.address`, so the forged receipt looked like a perfect deep delivery + and the chunk was acked `ok` having never been stored. + +Receipts no longer carry money under this design, but they still carry the +`po` and shallow signals the client schedules on, so a peer able to forge +them could steer traffic. Checking at the protocol boundary means every +`PushsyncReceipt` in the codebase carries exactly the 32 bytes that were +pushed. diff --git a/src/batch.rs b/src/batch.rs index aa215ee..fdc1e40 100644 --- a/src/batch.rs +++ b/src/batch.rs @@ -655,14 +655,30 @@ struct EthRpc { http: reqwest::Client, } +/// One process-wide HTTP client for every `eth_call`. +/// +/// `EthRpc::new` is called per read (`read_batch`, `read_remaining_balance`, +/// `read_last_price`, …), and building a fresh `reqwest::Client` each time +/// means a fresh connection pool, so every read paid a full TLS handshake +/// and none were ever reused. On the pusher's hot path that is one handshake +/// per batch resolution; under a flood of unresolvable batch ids it was one +/// per frame. `reqwest::Client` is an `Arc` internally, so cloning it shares +/// the pool. +fn shared_http() -> &'static reqwest::Client { + static HTTP: std::sync::OnceLock = std::sync::OnceLock::new(); + HTTP.get_or_init(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("reqwest client") + }) +} + impl EthRpc { fn new(url: String) -> Self { Self { url, - http: reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .build() - .expect("reqwest client"), + http: shared_http().clone(), } } @@ -1053,3 +1069,139 @@ mod tests { assert_eq!(amount_for_duration(1, 86400), U256::from(17290u64)); } } + +// ────────────────────────────────────────────────────────────────────── +// Chequebook reads (metered relay — docs/pusher-incentives.md Stage 1) +// ────────────────────────────────────────────────────────────────────── + +sol! { + // ERC20SimpleSwap / SimpleSwapFactory, from + // github.com/ethersphere/swap-swear-and-swindle. + function issuer() external view returns (address); + function paidOut(address beneficiary) external view returns (uint256); + function bounced() external view returns (bool); + // NOT `balance()`. `liquidBalance() = balance() - totalHardDeposit`, and + // `liquidBalanceFor(b) = liquidBalance() + hardDeposits[b].amount`, which + // is what `_cashChequeInternal` actually pays against. Bee checks + // `balance()`, counting *other* beneficiaries' hard deposits as our + // coverage — unsound in general, invisible only because bee never places + // any (incentives §11.2). + function liquidBalanceFor(address beneficiary) external view returns (uint256); + function deployedContracts(address who) external view returns (bool); +} + +/// Canonical `SimpleSwapFactory` per chain (`bee/pkg/config/chain.go:66,89`). +/// **Hardcoded on purpose.** A factory address supplied by the client lets it +/// present a contract that returns a forged `issuer()` and +/// `liquidBalanceFor()` and implements `cashChequeBeneficiary` as a no-op — +/// total compromise for one `eth_call` saved (incentives §6). +pub const GNOSIS_SWAP_FACTORY: &str = "0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420"; +pub const SEPOLIA_SWAP_FACTORY: &str = "0x0fF044F6bB4F684a5A149B46D7eC03ea659F98A1"; + +/// The factory for a chain, or `None` if we have no vetted address for it — +/// in which case metered mode must not run, rather than fall back to +/// something the client names. +pub fn swap_factory_for_chain(chain_id: u64) -> Option
{ + let s = match chain_id { + 100 => GNOSIS_SWAP_FACTORY, + 11155111 => SEPOLIA_SWAP_FACTORY, + _ => return None, + }; + s.parse().ok() +} + +/// What the relay needs to know about a chequebook to accept a cheque +/// drawn on it. +#[derive(Debug, Clone, Copy)] +pub struct ChequebookState { + pub issuer: Address, + /// Everything this beneficiary could actually cash right now. + pub liquid_for_us: U256, + /// Already cashed by this beneficiary; the cheque's cumulative must + /// exceed it or there is nothing left to draw. + pub paid_out_to_us: U256, + /// Set permanently, contract-wide, the first time any cheque could not + /// be paid in full. Readable state rather than an event you had to have + /// been watching for (incentives §11.2). + pub bounced: bool, +} + +/// Is this address a chequebook the canonical factory deployed? +/// +/// Cache this forever per address on a `true`, and negative-cache a `false` +/// — it is the first chain read an unauthenticated `/v1/pay` can reach, so +/// an uncached miss is a one-RPC-per-request amplifier (incentives §11.6). +pub async fn is_deployed_chequebook( + rpc_url: &str, + factory: Address, + chequebook: Address, +) -> Result { + EthRpc::new(rpc_url.to_string()) + .call_view(factory, deployedContractsCall { who: chequebook }) + .await +} + +/// Read the four values that decide whether a cheque is worth anything. +/// +/// `issuer` is cacheable per chequebook — it cannot change — but the balance +/// and `paidOut` are not: they *are* the funding check, and caching them is +/// what would let the withdraw race (§11.2) go unnoticed. +pub async fn read_chequebook_state( + rpc_url: &str, + chequebook: Address, + beneficiary: Address, +) -> Result { + let rpc = EthRpc::new(rpc_url.to_string()); + let issuer = rpc.call_view(chequebook, issuerCall {}).await?; + let liquid_for_us = rpc + .call_view( + chequebook, + liquidBalanceForCall { + beneficiary, + }, + ) + .await?; + let paid_out_to_us = rpc + .call_view(chequebook, paidOutCall { beneficiary }) + .await?; + let bounced = rpc.call_view(chequebook, bouncedCall {}).await?; + Ok(ChequebookState { + issuer, + liquid_for_us, + paid_out_to_us, + bounced, + }) +} + +#[cfg(test)] +mod chequebook_binding_tests { + use super::*; + + /// Selectors are what the node dispatches on, so a wrong one silently + /// reads a *different* function rather than failing. Pinned against + /// `keccak256(signature)[..4]`. + #[test] + fn selectors_match_the_solidity_signatures() { + use alloy_sol_types::SolCall; + for (got, sig) in [ + (issuerCall::SELECTOR, "issuer()"), + (paidOutCall::SELECTOR, "paidOut(address)"), + (bouncedCall::SELECTOR, "bounced()"), + (liquidBalanceForCall::SELECTOR, "liquidBalanceFor(address)"), + (deployedContractsCall::SELECTOR, "deployedContracts(address)"), + ] { + let want: [u8; 32] = ::digest(sig.as_bytes()).into(); + assert_eq!(got, want[..4], "selector drift for {sig}"); + } + } + + /// A relay must never accept a factory address from the wire, and must + /// refuse to run metered on a chain it has no vetted factory for. + #[test] + fn only_known_chains_have_a_factory() { + assert!(swap_factory_for_chain(100).is_some(), "gnosis"); + assert!(swap_factory_for_chain(11155111).is_some(), "sepolia"); + assert!(swap_factory_for_chain(1).is_none(), "mainnet: no vetted factory"); + assert!(swap_factory_for_chain(31337).is_none(), "local devnet"); + } +} diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index a18dc40..02f6339 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -507,6 +507,70 @@ enum Commands { value_name = "URL" )] rpc_url: String, + + // ── Metered mode (docs/pusher-incentives.md Stage 1) ────────── + /// Bill clients for relayed bytes with off-chain SWAP cheques. + /// Off by default: `open` is today's unmetered behaviour and is + /// what the production lanes run. + #[arg(long)] + meter: bool, + + /// Hostname(s) this relay is reached at. **Required with + /// `--meter`.** The admission challenge binds the origin a client + /// dialled, and the relay compares it against *this* value — never + /// against the `Host` header, which the same client supplies and + /// which would make the check a no-op that silently reopens + /// cross-relay replay. + #[arg(long, value_name = "HOST")] + origin: Vec, + + /// EOA that cheques are made out to. The relay holds this address + /// only, never the key: cashing out happens elsewhere. + #[arg(long, value_name = "0xADDR")] + beneficiary: Option, + + /// Settlement chain. Pins the EIP-712 domain and selects the + /// hardcoded SimpleSwapFactory; a chain with no vetted factory + /// cannot run metered. + #[arg(long, default_value_t = 100, value_name = "ID")] + chequebook_chain: u64, + + /// Directory for the ledger and relay secret. **Required with + /// `--meter`**: without durable state a client re-presents its + /// last cheque after every restart and is credited the full + /// cumulative again, which is unlimited free service from one + /// signature. + #[arg(long, value_name = "DIR")] + state_dir: Option, + + /// PLUR per KiB of body admitted. + #[arg(long, value_name = "PLUR")] + price_plur_per_kib: Option, + + /// Smallest cheque accepted, to bound RPC cost per unit of value. + #[arg(long, value_name = "PLUR")] + min_cheque_plur: Option, + + /// Debt at which a client is expected to settle. + #[arg(long, value_name = "PLUR")] + settle_every_plur: Option, + + /// Global ceiling on a credit line. The actual cap is per batch: + /// `min(batch_remaining_value / credit_ratio, this)`. + #[arg(long, value_name = "PLUR")] + max_outstanding_plur: Option, + + /// Credit line = batch remaining on-chain value ÷ this. The Sybil + /// margin is this ratio by construction, independent of batch size. + #[arg(long, value_name = "N")] + credit_ratio: Option, + + /// Enforce 402 when an account is over its cap. Default is soft + /// mode: meter, report, and serve anyway — which is what Stage 1 + /// ships, so the overshoot rate can be measured against live + /// traffic before anyone is refused. + #[arg(long)] + meter_hard: bool, }, /// Run a long-lived daemon that holds a warm session pool across @@ -2063,6 +2127,17 @@ async fn main() -> Result<(), Box> { peerlist, probe, rpc_url, + meter, + origin, + beneficiary, + chequebook_chain, + state_dir, + price_plur_per_kib, + min_cheque_plur, + settle_every_plur, + max_outstanding_plur, + credit_ratio, + meter_hard, } => { // Premined overlay nonce. On ephemeral-FS hosts (Render, // Lambda) there is no persistent `--nonce-file`, so a random @@ -2092,6 +2167,49 @@ async fn main() -> Result<(), Box> { let node_identity = std::env::var("HOVERFLY_PUSHER_IDENTITY") .ok() .filter(|s| !s.trim().is_empty()); + // Metered mode is all-or-nothing: every precondition is checked + // here and in `build_metered`, and a failure refuses to start + // rather than serving a half-configured meter that a paying + // client would discover the hard way. + let meter_opts = if meter { + let mut params = hoverfly::meter::Params::default(); + if let Some(v) = price_plur_per_kib { + params.price_plur_per_kib = v; + } + if let Some(v) = min_cheque_plur { + params.min_cheque_plur = v; + } + if let Some(v) = settle_every_plur { + params.settle_every_plur = v; + } + if let Some(v) = max_outstanding_plur { + params.max_outstanding_plur = v; + } + if let Some(v) = credit_ratio { + params.credit_ratio = v; + } + let beneficiary = beneficiary + .as_deref() + .ok_or("--meter requires --beneficiary")?; + let raw = hex::decode(beneficiary.trim_start_matches("0x")) + .map_err(|e| format!("--beneficiary: {e}"))?; + if raw.len() != 20 { + return Err("--beneficiary must be a 20-byte address".into()); + } + let mut b = [0u8; 20]; + b.copy_from_slice(&raw); + Some(hoverfly::pusher::MeterOpts { + origins: origin, + beneficiary: b, + chain_id: chequebook_chain, + params, + state_dir: state_dir + .ok_or("--meter requires --state-dir (durable state is mandatory)")?, + hard_mode: meter_hard, + }) + } else { + None + }; hoverfly::pusher::run(hoverfly::pusher::PusherOpts { listen, peerlist, @@ -2101,6 +2219,7 @@ async fn main() -> Result<(), Box> { rpc_url, node_identity, transport: cfg, + meter: meter_opts, }) .await?; } diff --git a/src/challenge.rs b/src/challenge.rs new file mode 100644 index 0000000..784ed3c --- /dev/null +++ b/src/challenge.rs @@ -0,0 +1,310 @@ +//! Admission challenge for metered relay — `docs/pusher-incentives.md` §7.2. +//! +//! A challenge is a **capability**. Holding one proves the relay already +//! resolved the batch's standing on-chain and already priced its credit +//! line, so `/v1/push` admission reads no chain state at all — which is the +//! whole reason the design can afford to check standing before accepting a +//! body (§7.2's amplification argument). +//! +//! It is stateless: the relay keeps no table of issued nonces, so a free +//! `GET /v1/challenge` cannot exhaust memory. The nonce is a MAC over the +//! fields it authorises, and the fields travel back with the request. +//! +//! Two independent checks run at admission, and conflating them is the +//! easiest way to get this wrong: +//! +//! 1. **The relay's MAC over the nonce** proves *this relay* issued the +//! capability, with these exact fields. Symmetric, no RPC. +//! 2. **The client's EIP-712 signature** over the same fields proves the +//! caller holds the account key — and, because `origin` is inside the +//! signed struct, that it signed for *this* relay. That is what stops a +//! signature gathered during a normal upload through relay A being +//! replayed at relay B alongside the victim's harvested stamps (§11.1). +//! +//! Check 2 is only worth anything if `origin` is compared against a +//! **configured** hostname. Comparing it to the request's `Host` header +//! compares one attacker-supplied value to another and silently restores +//! the replay — see [`verify`]. + +use sha3::{Digest, Keccak256}; + +/// Domain tag, fixed 28 bytes. First field after the secret so no other +/// scheme using the same key can collide with this one. +pub const DOMAIN_TAG: &[u8; 28] = b"hoverfly-pusher-challenge-v1"; + +/// Default lifetime of a challenge (§7.2). Short on purpose: re-issuing +/// costs one local ecrecover, and a narrow window shrinks the replay +/// surface to near nothing. +pub const CHALLENGE_TTL_SECS: u64 = 300; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChallengeFields { + pub account: [u8; 20], + pub batch: [u8; 32], + /// The host the client dialled. Compared against configuration, never + /// against a request header. + pub origin: String, + pub expiry_unix: u64, + /// The credit line §10.3 granted this batch, in PLUR. Inside the MAC so + /// a client cannot present a nonce issued for a rich batch alongside a + /// cheap one's id. + pub cap_plur: u128, +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum ChallengeError { + #[error("challenge nonce is not ours")] + BadMac, + #[error("challenge expired {0}s ago")] + Expired(u64), + #[error("challenge origin {got:?} is not this relay ({want:?})")] + OriginMismatch { got: String, want: String }, + #[error("challenge origin is empty")] + EmptyOrigin, + #[error("origin too long: {0} bytes (max 65535)")] + OriginTooLong(usize), +} + +/// Bytes the MAC covers. +/// +/// **Fixed-width and length-prefixed, not a concatenation.** `origin` is +/// variable-length, so a bare `A ‖ B ‖ origin ‖ …` makes `("host.a", "bc")` +/// and `("host.ab", "c")` share a preimage — a relay serving several +/// hostnames would issue one nonce valid for two of them. Every fixed field +/// comes first at a known width, then a 2-byte length, then `origin` last. +pub fn preimage(f: &ChallengeFields) -> Result, ChallengeError> { + if f.origin.is_empty() { + return Err(ChallengeError::EmptyOrigin); + } + let olen = f.origin.len(); + if olen > u16::MAX as usize { + return Err(ChallengeError::OriginTooLong(olen)); + } + let mut out = Vec::with_capacity(28 + 20 + 32 + 8 + 16 + 2 + olen); + out.extend_from_slice(DOMAIN_TAG); + out.extend_from_slice(&f.account); + out.extend_from_slice(&f.batch); + out.extend_from_slice(&f.expiry_unix.to_be_bytes()); + out.extend_from_slice(&f.cap_plur.to_be_bytes()); + out.extend_from_slice(&(olen as u16).to_be_bytes()); + out.extend_from_slice(f.origin.as_bytes()); + Ok(out) +} + +/// `keccak256(secret ‖ preimage)`. +/// +/// A prefix-MAC rather than HMAC, which is sound here specifically because +/// Keccak is a sponge and has no length-extension weakness — the property +/// that forces HMAC's nested construction on Merkle–Damgård hashes like +/// SHA-256. The secret is a fixed 32 bytes and every field after it is +/// fixed-width or length-prefixed, so no two distinct inputs share a +/// preimage. (This is the same reasoning KMAC is built on.) +pub fn nonce(secret: &[u8; 32], f: &ChallengeFields) -> Result<[u8; 32], ChallengeError> { + let mut h = Keccak256::new(); + h.update(secret); + h.update(preimage(f)?); + Ok(h.finalize().into()) +} + +/// Verify a presented nonce against the fields it claims to authorise. +/// +/// `origins` is the relay's **configured** hostname list (`--origin`). It +/// must never be derived from `Host` or `X-Forwarded-Host`: those are +/// supplied by the same client supplying the challenge, so comparing them +/// is a no-op that leaves §11.1's cross-relay replay wide open while +/// appearing to close it. +pub fn verify( + secret: &[u8; 32], + f: &ChallengeFields, + presented: &[u8], + now_unix: u64, + origins: &[String], +) -> Result<(), ChallengeError> { + if !origins.iter().any(|o| o == &f.origin) { + return Err(ChallengeError::OriginMismatch { + got: f.origin.clone(), + want: origins.join(","), + }); + } + let want = nonce(secret, f)?; + if !constant_time_eq(presented, &want) { + return Err(ChallengeError::BadMac); + } + // Expiry last: a caller holding a valid-but-stale capability learns + // only that it is stale, while a forged one learns nothing about which + // field was wrong. + if now_unix > f.expiry_unix { + return Err(ChallengeError::Expired(now_unix - f.expiry_unix)); + } + Ok(()) +} + +/// Equal-length, data-independent comparison. A byte-wise early exit on the +/// MAC would let a nonce be ground out one byte at a time, which is exactly +/// the forgery this whole construction exists to prevent. +pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// Seconds since the Unix epoch, saturating at 0 on a clock before 1970. +pub fn now_unix() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SECRET: [u8; 32] = [7u8; 32]; + + fn fields() -> ChallengeFields { + ChallengeFields { + account: [1u8; 20], + batch: [2u8; 32], + origin: "relay-a.example".into(), + expiry_unix: 1_000_000, + cap_plur: 62_200_000_000_000, + } + } + + fn origins() -> Vec { + vec!["relay-a.example".into()] + } + + #[test] + fn a_nonce_we_issued_verifies() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + verify(&SECRET, &f, &n, 999_999, &origins()).expect("must verify"); + } + + #[test] + fn a_nonce_from_another_relays_secret_does_not() { + let f = fields(); + let n = nonce(&[9u8; 32], &f).expect("nonce"); + assert_eq!( + verify(&SECRET, &f, &n, 999_999, &origins()), + Err(ChallengeError::BadMac) + ); + } + + /// Every field is inside the MAC, so tampering with any of them after + /// issue must fail. The `cap` case is the sharp one: without it a client + /// could present a rich batch's credit line alongside a dust batch's id. + #[test] + fn every_field_is_covered_by_the_mac() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + let mut cases = Vec::new(); + + let mut g = f.clone(); + g.account = [0xAA; 20]; + cases.push(("account", g)); + let mut g = f.clone(); + g.batch = [0xBB; 32]; + cases.push(("batch", g)); + let mut g = f.clone(); + g.expiry_unix += 1; + cases.push(("expiry", g)); + let mut g = f.clone(); + g.cap_plur *= 1000; + cases.push(("cap", g)); + + for (what, g) in cases { + assert_eq!( + verify(&SECRET, &g, &n, 999_999, &origins()), + Err(ChallengeError::BadMac), + "{what} must be covered by the MAC" + ); + } + } + + /// The ambiguity the doc rejects for the client's signature and then + /// very nearly repeated in the relay's own MAC. `("host.a","bc")` and + /// `("host.ab","c")` must not collide. + #[test] + fn the_preimage_is_unambiguous_across_a_variable_length_origin() { + let mut a = fields(); + a.origin = "host.a".into(); + let mut b = fields(); + b.origin = "host.ab".into(); + assert_ne!( + preimage(&a).expect("a"), + preimage(&b).expect("b"), + "a shorter origin must not be a prefix-collision of a longer one" + ); + assert_ne!(nonce(&SECRET, &a).unwrap(), nonce(&SECRET, &b).unwrap()); + } + + /// The origin the relay compares against is configuration. A nonce + /// issued for another host must not verify here even though the MAC + /// itself is intact — this is the check that stops §11.1. + #[test] + fn an_origin_outside_the_configured_set_is_refused() { + let mut f = fields(); + f.origin = "relay-b.example".into(); + let n = nonce(&SECRET, &f).expect("nonce"); + let got = verify(&SECRET, &f, &n, 999_999, &origins()); + assert!( + matches!(got, Err(ChallengeError::OriginMismatch { .. })), + "got {got:?}" + ); + } + + #[test] + fn a_relay_serving_several_hostnames_accepts_each_of_them() { + let all: Vec = vec!["relay-a.example".into(), "alias.example".into()]; + for host in &all { + let mut f = fields(); + f.origin = host.clone(); + let n = nonce(&SECRET, &f).expect("nonce"); + verify(&SECRET, &f, &n, 999_999, &all).expect("each configured host verifies"); + } + } + + #[test] + fn an_expired_challenge_is_refused() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + verify(&SECRET, &f, &n, f.expiry_unix, &origins()).expect("valid up to the instant"); + assert_eq!( + verify(&SECRET, &f, &n, f.expiry_unix + 1, &origins()), + Err(ChallengeError::Expired(1)) + ); + } + + #[test] + fn a_truncated_or_padded_nonce_is_refused() { + let f = fields(); + let n = nonce(&SECRET, &f).expect("nonce"); + assert_eq!( + verify(&SECRET, &f, &n[..31], 1, &origins()), + Err(ChallengeError::BadMac) + ); + let mut long = n.to_vec(); + long.push(0); + assert_eq!( + verify(&SECRET, &f, &long, 1, &origins()), + Err(ChallengeError::BadMac) + ); + assert_eq!(verify(&SECRET, &f, &[], 1, &origins()), Err(ChallengeError::BadMac)); + } + + #[test] + fn an_empty_origin_is_rejected_rather_than_hashed() { + let mut f = fields(); + f.origin = String::new(); + assert_eq!(preimage(&f), Err(ChallengeError::EmptyOrigin)); + } + + #[test] + fn constant_time_eq_agrees_with_equality() { + assert!(constant_time_eq(b"abc", b"abc")); + assert!(!constant_time_eq(b"abc", b"abd")); + assert!(!constant_time_eq(b"abc", b"ab")); + assert!(constant_time_eq(b"", b"")); + } +} diff --git a/src/cheques.rs b/src/cheques.rs index 4d435cc..87571bc 100644 --- a/src/cheques.rs +++ b/src/cheques.rs @@ -175,3 +175,143 @@ impl ChequeStore { Ok(()) } } + +// ────────────────────────────────────────────────────────────────────── +// Metered relays (docs/pusher-incentives.md §8.3) +// ────────────────────────────────────────────────────────────────────── + +/// Key for a metered relay's cumulative. +/// +/// **Namespaced by beneficiary, not by lane or overlay**, and deliberately +/// distinct from the bare-overlay keys the bee settlement path uses. +/// +/// A cumulative is per `(chequebook, beneficiary)` — the beneficiary is what +/// `paidOut` is keyed on in the contract — while a *lane* is a URL. One +/// operator running four lane URLs behind one beneficiary EOA is the obvious +/// deployment, and keying per lane would deadlock it: lane 1 issues +/// cumulative 10, lane 2 counts from its own zero and issues 8, the relay +/// applies `ErrChequeNotIncreasing`, and the client recomputes 8 from the +/// same local counter forever. Two lanes sharing a beneficiary collapse to +/// one key here, which is exactly right — they are one settlement channel. +/// +/// The overlay key stays correct for bee peers, where the overlay *is* the +/// stable cross-run identity, so the two namespaces coexist without a +/// migration. +pub fn relay_key(beneficiary: &[u8; 20]) -> String { + format!("relay:{}", hex::encode(beneficiary)) +} + +impl ChequeStore { + /// Everything promised against this chequebook, across every payee. + /// + /// Lanes with distinct beneficiaries are independent claims on **one** + /// balance, so without this a cheque to the second lane silently + /// exceeds it and bounces. Mirrors bee's `reserveTotalIssued` + /// (`chequebook.go:163-178`) on the issuing side. + pub fn total_issued(&self) -> u128 { + self.payouts.values().copied().fold(0u128, u128::saturating_add) + } + + /// Would raising `key` to `cumulative` push the total past `balance`? + /// + /// Checked *before* signing: an over-committed cheque is not refused by + /// the relay, it is accepted and then fails at cashout, which looks like + /// the relay's fault and costs the lane's trust rather than the + /// client's. + pub fn would_exceed_balance(&self, key: &str, cumulative: u128, balance: u128) -> bool { + let others = self + .total_issued() + .saturating_sub(self.cumulative(key)); + others.saturating_add(cumulative) > balance + } + + /// Set an absolute cumulative, for payees where the client computes the + /// running total itself (metered relays) rather than accruing deltas. + /// Refuses to move backwards — that would produce a cheque the payee + /// rejects as non-increasing. + pub fn set_cumulative(&mut self, key: &str, cumulative: u128) -> Result<(), ChequeStoreError> { + let k = key.to_lowercase(); + let cur = self.payouts.get(&k).copied().unwrap_or(0); + if cumulative < cur { + return Err(ChequeStoreError::Overflow); + } + self.payouts.insert(k, cumulative); + Ok(()) + } +} + +#[cfg(test)] +mod metered_tests { + use super::*; + + const CB: [u8; 20] = [1u8; 20]; + const BEN_A: [u8; 20] = [0xAA; 20]; + const BEN_B: [u8; 20] = [0xBB; 20]; + + /// The deployment that would otherwise deadlock: several lane URLs, one + /// beneficiary. They must share a single running cumulative. + #[test] + fn two_lanes_behind_one_beneficiary_share_a_cumulative() { + let mut s = ChequeStore::new(CB); + let k = relay_key(&BEN_A); + s.set_cumulative(&k, 1000).expect("lane 1 settles"); + // Lane 2, same operator, same beneficiary — must continue from 1000 + // rather than starting over at its own zero. + assert_eq!(s.cumulative(&k), 1000); + s.set_cumulative(&k, 1600).expect("lane 2 settles"); + assert_eq!(s.cumulative(&k), 1600); + } + + #[test] + fn a_cumulative_never_moves_backwards() { + let mut s = ChequeStore::new(CB); + let k = relay_key(&BEN_A); + s.set_cumulative(&k, 500).expect("set"); + s.set_cumulative(&k, 400) + .expect_err("a lower cumulative would be rejected as non-increasing"); + assert_eq!(s.cumulative(&k), 500); + } + + /// Distinct beneficiaries are distinct channels but one balance. + #[test] + fn total_issued_sums_every_payee() { + let mut s = ChequeStore::new(CB); + s.set_cumulative(&relay_key(&BEN_A), 600).expect("a"); + s.set_cumulative(&relay_key(&BEN_B), 300).expect("b"); + // A bee peer drawing on the same chequebook counts too. + s.bump_and_get("abc123", 100).expect("bee peer"); + assert_eq!(s.total_issued(), 1000); + } + + #[test] + fn over_committing_the_balance_is_caught_before_signing() { + let mut s = ChequeStore::new(CB); + s.set_cumulative(&relay_key(&BEN_A), 600).expect("a"); + s.set_cumulative(&relay_key(&BEN_B), 300).expect("b"); + let k = relay_key(&BEN_A); + assert!( + !s.would_exceed_balance(&k, 700, 1000), + "raising A to 700 alongside B's 300 exactly fits" + ); + assert!( + s.would_exceed_balance(&k, 701, 1000), + "one PLUR more does not" + ); + } + + /// Relay and bee keys must not collide: an overlay is 32 bytes of hex + /// and a beneficiary is 20, but the namespace makes it explicit rather + /// than incidental. + #[test] + fn relay_keys_are_namespaced_away_from_bee_overlays() { + let k = relay_key(&BEN_A); + assert!(k.starts_with("relay:")); + let mut s = ChequeStore::new(CB); + s.set_cumulative(&k, 42).expect("relay"); + assert_eq!( + s.cumulative(&hex::encode(BEN_A)), + 0, + "a bare-hex key must not read the relay's entry" + ); + } +} diff --git a/src/client.rs b/src/client.rs index 00d606a..6e8d6a3 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2710,7 +2710,13 @@ where // others — which is exactly what the previous all-or-nothing overlay // collection did. let infos: Vec = - futures::future::join_all(pusher_urls.iter().map(|u| fetch_lane_info(&http, u))).await; + futures::future::join_all(pusher_urls.iter().map(|u| { + // A lane quoting more than this is refused rather than paid. + // Priced off the shipped default so a lane cannot quietly + // charge an order of magnitude more than the design assumes. + fetch_lane_info(&http, u, crate::meter::PRICE_PLUR_PER_KIB * 8) + })) + .await; for (i, (u, info)) in pusher_urls.iter().zip(&infos).enumerate() { info!(target: "hoverfly::upload", "lane {i} {u}: pool={:?} batch_max={:?} inflight_max={:?} budget_gb={:?}", @@ -2807,7 +2813,7 @@ where "hedging {} straggler(s) onto lane {lane}", batch.len()); } tokio::spawn(async move { - post_batch_streaming(&http, url.as_str(), batch_id, lane, &batch, &tx).await; + post_batch_streaming(&http, url.as_str(), batch_id, lane, &batch, &tx, None).await; }); } @@ -3035,6 +3041,10 @@ async fn post_batch_streaming( lane: usize, batch: &[StampedChunk], tx: &tokio::sync::mpsc::UnboundedSender, + // Metered lanes only: the signed admission capability + // (`docs/pusher-incentives.md` §7.2). `None` on an `open` lane, which + // is every production lane today. + challenge: Option<&str>, ) { use crate::pushsched::BatchOutcome; use futures::StreamExt; @@ -3053,7 +3063,11 @@ async fn post_batch_streaming( }); }; - let resp = match http.post(push_url).body(body).send().await { + let mut req = http.post(push_url).body(body); + if let Some(c) = challenge { + req = req.header(crate::metered::CHALLENGE_HEADER, c); + } + let resp = match req.send().await { Ok(r) => r, Err(e) => { warn!(target: "hoverfly::upload", "lane {push_url} POST failed: {e}"); @@ -3064,6 +3078,17 @@ async fn post_batch_streaming( if !resp.status().is_success() { let code = resp.status(); let txt = resp.text().await.unwrap_or_default(); + // A 402 is a bill, not a fault. Reporting it as `Failed` would + // charge lane health for a routine settlement and retire a healthy + // lane after five of them (§12); `PaymentRequired` pauses it + // instead, and `Scheduler::fund_lane` restores it once a cheque + // clears. + if code == reqwest::StatusCode::PAYMENT_REQUIRED { + info!(target: "hoverfly::upload", + "lane {push_url} requires payment: {}", txt.trim()); + finish(BatchOutcome::PaymentRequired, 0); + return; + } warn!(target: "hoverfly::upload", "lane {push_url} rejected batch ({code}): {}", txt.trim()); finish(BatchOutcome::Failed(format!("http {code}")), 0); @@ -3130,7 +3155,11 @@ async fn post_batch_streaming( /// yields `LaneInfo::default()` and gets scheduled on priors rather than /// being excluded. #[cfg(not(target_arch = "wasm32"))] -async fn fetch_lane_info(http: &reqwest::Client, base_url: &str) -> crate::pushsched::LaneInfo { +async fn fetch_lane_info( + http: &reqwest::Client, + base_url: &str, + price_ceiling: u128, +) -> crate::pushsched::LaneInfo { use crate::pushsched::LaneInfo; let url = format!("{}/v1/status", base_url.trim_end_matches('/')); // Generous: free-tier instances cold-start on the first request. @@ -3151,8 +3180,34 @@ async fn fetch_lane_info(http: &reqwest::Client, base_url: &str) -> crate::pushs .and_then(|s| s.as_str()) .and_then(|s| hex::decode(s.trim_start_matches("0x")).ok()) .and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok()); + // The signed price quote (§7.3). Verified, never merely parsed: an + // unsigned or unverifiable price is repudiable in both directions, so a + // quote that does not check out leaves the lane looking unmetered rather + // than looking free. + let (price_plur_per_kib, hard_enforcement) = match v.get("payment") { + Some(pay) if !pay.is_null() => { + // The overlay cross-check is skipped here: it needs the + // network id, which this driver does not carry, and it is the + // *optional* half of verification. The signature and (where a + // caller supplies one) the pin are what establish identity; + // deriving the overlay only turns the lane's own `overlay` + // field from an assertion into a check. A caller that pins + // does the full version. + match crate::payer::PaymentQuote::verify(pay, None, 0, None, price_ceiling) { + Ok(q) => (Some(q.params.price_plur_per_kib), q.hard_enforcement), + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {base_url}: payment quote rejected ({e}); treating as unmetered"); + (None, false) + } + } + } + _ => (None, false), + }; LaneInfo { overlay, + price_plur_per_kib, + hard_enforcement, batch_max: v .get("batch_max") .and_then(|x| x.as_u64()) diff --git a/src/inbound_limit.rs b/src/inbound_limit.rs new file mode 100644 index 0000000..f356a9f --- /dev/null +++ b/src/inbound_limit.rs @@ -0,0 +1,196 @@ +//! Inbound per-key rate limiting — `docs/pusher-incentives.md` §11.6. +//! +//! **`src/ratelimit.rs` cannot be reused for this.** That one is a per-peer +//! *outbound libp2p dial* GCRA pacer that **parks** the caller until its +//! slot comes up; it has no inbound, per-account or HTTP concept. Parking +//! is exactly wrong here — an attacker that can make the relay hold a task +//! per request has turned a rate limiter into a memory amplifier. This one +//! refuses immediately. +//! +//! Used on `/v1/challenge` (per IP — no account exists yet), `/v1/pay` (per +//! account) and `/v1/push` (per account). +//! +//! ## Eviction is fail-closed, and that is the whole design +//! +//! The bucket map is keyed by something the caller influences, so it has to +//! be bounded. But naive eviction *is* the bypass: if an attacker can push +//! its own throttled bucket out of the map, the next request re-creates it +//! with a full budget and the limit never binds. +//! +//! So only buckets that have nothing to lose are evictable — ones refilled +//! to full, which is indistinguishable from never having existed. When the +//! map is at capacity and every bucket is *still throttled*, new keys are +//! **refused** rather than admitted. Under attack the relay gets stricter, +//! not more permissive. + +use std::collections::HashMap; +use std::time::Instant; + +pub struct InboundLimiter { + buckets: HashMap, Bucket>, + cap: usize, + /// Sustained requests per second. + rate: f64, + /// Maximum burst, in requests. + burst: f64, +} + +#[derive(Clone, Copy)] +struct Bucket { + tokens: f64, + last: Instant, +} + +impl Bucket { + fn refill(&mut self, now: Instant, rate: f64, burst: f64) { + let dt = now.saturating_duration_since(self.last).as_secs_f64(); + self.tokens = (self.tokens + dt * rate).min(burst); + self.last = now; + } + + /// Nothing to lose by dropping it: a fresh bucket is identical. + fn is_full(&self, burst: f64) -> bool { + self.tokens >= burst + } +} + +impl InboundLimiter { + pub fn new(rate_per_sec: f64, burst: f64, cap: usize) -> Self { + Self { + buckets: HashMap::new(), + cap: cap.max(1), + rate: rate_per_sec.max(f64::MIN_POSITIVE), + burst: burst.max(1.0), + } + } + + pub fn allow(&mut self, key: &[u8]) -> bool { + self.allow_at(key, Instant::now()) + } + + pub fn allow_at(&mut self, key: &[u8], now: Instant) -> bool { + let (rate, burst) = (self.rate, self.burst); + if let Some(b) = self.buckets.get_mut(key) { + b.refill(now, rate, burst); + if b.tokens < 1.0 { + return false; + } + b.tokens -= 1.0; + return true; + } + if self.buckets.len() >= self.cap { + self.sweep(now); + if self.buckets.len() >= self.cap { + // Every bucket is still throttled and we are full. Admitting + // this one would mean an attacker with enough distinct keys + // can mint budget on demand; refusing costs a stranger one + // request during an active flood. + return false; + } + } + self.buckets.insert( + key.to_vec(), + Bucket { + tokens: burst - 1.0, + last: now, + }, + ); + true + } + + /// Drop only buckets that have refilled to full. + fn sweep(&mut self, now: Instant) { + let (rate, burst) = (self.rate, self.burst); + self.buckets.retain(|_, b| { + b.refill(now, rate, burst); + !b.is_full(burst) + }); + } + + pub fn tracked(&self) -> usize { + self.buckets.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn a_burst_is_allowed_then_the_rate_binds() { + let mut l = InboundLimiter::new(1.0, 5.0, 128); + let t0 = Instant::now(); + for i in 0..5 { + assert!(l.allow_at(b"a", t0), "burst request {i} must pass"); + } + assert!(!l.allow_at(b"a", t0), "the 6th in the same instant must not"); + } + + #[test] + fn tokens_refill_over_time() { + let mut l = InboundLimiter::new(2.0, 2.0, 128); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(l.allow_at(b"a", t0)); + assert!(!l.allow_at(b"a", t0)); + assert!( + l.allow_at(b"a", t0 + Duration::from_millis(500)), + "half a second at 2/s is one token" + ); + } + + #[test] + fn keys_are_independent() { + let mut l = InboundLimiter::new(1.0, 1.0, 128); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(!l.allow_at(b"a", t0)); + assert!(l.allow_at(b"b", t0), "one key's flood must not throttle another"); + } + + /// The bypass this design exists to close: an attacker cycling keys must + /// not be able to evict its own throttled bucket and come back fresh. + #[test] + fn a_throttled_bucket_cannot_be_evicted_by_flooding_new_keys() { + let mut l = InboundLimiter::new(0.001, 1.0, 4); + let t0 = Instant::now(); + assert!(l.allow_at(b"victim", t0)); + assert!(!l.allow_at(b"victim", t0), "victim is now throttled"); + // Flood distinct keys to try to push `victim` out. + for i in 0..200u32 { + l.allow_at(format!("k{i}").as_bytes(), t0); + } + assert!( + !l.allow_at(b"victim", t0), + "the throttled bucket must have survived the flood" + ); + assert!(l.tracked() <= 4, "map stays bounded, got {}", l.tracked()); + } + + /// Fail-closed: when the map is full of throttled buckets, a new key is + /// refused rather than admitted. + #[test] + fn a_full_map_of_throttled_buckets_refuses_new_keys() { + let mut l = InboundLimiter::new(0.001, 1.0, 2); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(l.allow_at(b"b", t0)); + assert!(!l.allow_at(b"c", t0), "must refuse rather than evict a live limit"); + } + + /// …but a bucket that has refilled to full is free to drop, so the map + /// recovers once the flood stops. + #[test] + fn full_buckets_are_reclaimed_so_the_map_recovers() { + let mut l = InboundLimiter::new(10.0, 1.0, 2); + let t0 = Instant::now(); + assert!(l.allow_at(b"a", t0)); + assert!(l.allow_at(b"b", t0)); + let later = t0 + Duration::from_secs(60); + assert!( + l.allow_at(b"c", later), + "once a and b are refilled they carry no state worth keeping" + ); + } +} diff --git a/src/ledger.rs b/src/ledger.rs new file mode 100644 index 0000000..0130014 --- /dev/null +++ b/src/ledger.rs @@ -0,0 +1,556 @@ +//! Relay-side payment ledger — `docs/pusher-incentives.md` §10.2, §11.4. +//! +//! Four coupled per-account quantities, written by N spawned push tasks and +//! read by admission, so they live under one lock: +//! +//! - `owed_plur` — billed and unpaid. +//! - `reserved_plur` — admitted but not yet completed. +//! - `last_cumulative[chequebook]` — the monotonic high-water mark that +//! makes a re-presented cheque worth zero. +//! - the `chequebook → account` binding. +//! +//! **Three of those are persisted and one is not, and the asymmetry is +//! load-bearing.** A reservation belongs to an in-flight POST, and no +//! in-flight POST survives a restart — there is no task left to release it. +//! Restoring `reserved` from disk therefore leaks credit permanently and can +//! brick an account into 402 with no way out, which is exactly the no-exit +//! failure §10.1's invariant exists to prevent. So: +//! +//! > Persist `owed`, `last_cumulative`, the binding and `relay_secret` +//! > atomically. Reconstruct `reserved` as zero at boot. +//! +//! The exposure from zeroing is one body's worth of over-admission right +//! after a restart — cents of egress, against an accounting corruption that +//! never self-heals. +//! +//! Losing `last_cumulative` alone is worse than losing everything: a client +//! re-presents its most recent cheque and is credited the *full cumulative* +//! instead of the delta, repeatably, for free (§11.4). Hence one atomic +//! write covering all of it, never a field at a time. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Refuse a cumulative past this. Total BZZ supply is 10^8 BZZ = 10^24 +/// PLUR, so anything above 10^30 is not a payment — it is an attempt to +/// find an overflow. Bounding it here keeps every downstream figure in +/// `u128` (max ≈ 3.4×10^38) with room to spare. +pub const MAX_CUMULATIVE_PLUR: u128 = 1_000_000_000_000_000_000_000_000_000_000; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum LedgerError { + #[error("cheque cumulative {got} is not greater than the {have} already accepted")] + NotIncreasing { got: u128, have: u128 }, + #[error("chequebook 0x{chequebook} is already bound to a different account")] + ChequebookBound { chequebook: String }, + #[error("cumulative {0} is implausibly large")] + Absurd(u128), + #[error("cheque credits {got} but only {owed} is owed")] + Overpayment { got: u128, owed: u128 }, +} + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("io: {0}")] + Io(String), + #[error("json: {0}")] + Json(#[from] serde_json::Error), +} + +/// Outcome of admitting a request against a credit line. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Admission { + pub reserved_plur: u128, + /// `owed + reserved` after this reservation. + pub outstanding_plur: u128, + pub cap_plur: u128, + /// True when this request pushed the account past its line. Soft mode + /// records it; hard mode releases the reservation and answers 402. + pub over_cap: bool, +} + +#[derive(Debug, Default, Clone)] +struct Account { + owed_plur: u128, + /// Deliberately absent from the on-disk form. See the module docs. + reserved_plur: u128, + last_cumulative: HashMap<[u8; 20], u128>, +} + +impl Account { + fn outstanding(&self) -> u128 { + self.owed_plur.saturating_add(self.reserved_plur) + } +} + +pub struct Ledger { + accounts: HashMap<[u8; 20], Account>, + /// A chequebook belongs to the first account that paid with it and + /// cannot move. Without this, two accounts could share a cumulative and + /// one would ride the other's payments. + binding: HashMap<[u8; 20], [u8; 20]>, + secret: [u8; 32], + path: Option, +} + +// ── On-disk form ───────────────────────────────────────────────────────── +// Hex-keyed and decimal-stringed: JSON has no u128, and PLUR amounts +// comfortably exceed what an f64 can hold exactly. + +#[derive(Serialize, Deserialize)] +struct OnDisk { + version: u32, + secret_hex: String, + accounts: Vec, + binding: Vec<(String, String)>, +} + +#[derive(Serialize, Deserialize)] +struct OnDiskAccount { + account: String, + owed_plur: String, + last_cumulative: Vec<(String, String)>, +} + +impl Ledger { + /// Fresh ledger with a random secret. Used when no `--state-dir` is + /// configured, which metered mode forbids (§5) but open mode allows. + pub fn ephemeral() -> Self { + let mut secret = [0u8; 32]; + // A failure here would mean no entropy source at all; a zero secret + // would make every nonce forgeable, so refuse to run instead. + getrandom::fill(&mut secret).expect("system entropy for the relay secret"); + Self { + accounts: HashMap::new(), + binding: HashMap::new(), + secret, + path: None, + } + } + + pub fn secret(&self) -> &[u8; 32] { + &self.secret + } + + /// Load from disk, or create and immediately persist a new ledger. + /// + /// `relay_secret` is part of the same file: regenerating it at boot + /// invalidates every outstanding challenge, which on a host that sleeps + /// and cold-starts turns each restart into a 403 storm for clients + /// mid-upload (§7.2). + pub fn load_or_create>(path: P) -> Result { + let path = path.as_ref().to_path_buf(); + let text = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let mut fresh = Self::ephemeral(); + fresh.path = Some(path); + fresh.persist()?; + return Ok(fresh); + } + Err(e) => return Err(StoreError::Io(e.to_string())), + }; + let disk: OnDisk = serde_json::from_str(&text)?; + let mut secret = [0u8; 32]; + let raw = hex::decode(disk.secret_hex.trim_start_matches("0x")) + .map_err(|e| StoreError::Io(format!("secret hex: {e}")))?; + if raw.len() != 32 { + return Err(StoreError::Io(format!( + "relay secret must be 32 bytes, got {}", + raw.len() + ))); + } + secret.copy_from_slice(&raw); + + let mut accounts = HashMap::new(); + for a in disk.accounts { + let key = parse_addr(&a.account)?; + let mut last_cumulative = HashMap::new(); + for (cb, v) in a.last_cumulative { + last_cumulative.insert(parse_addr(&cb)?, parse_u128(&v)?); + } + accounts.insert( + key, + Account { + owed_plur: parse_u128(&a.owed_plur)?, + // Never restored. See the module docs. + reserved_plur: 0, + last_cumulative, + }, + ); + } + let mut binding = HashMap::new(); + for (cb, acct) in disk.binding { + binding.insert(parse_addr(&cb)?, parse_addr(&acct)?); + } + Ok(Self { + accounts, + binding, + secret, + path: Some(path), + }) + } + + /// Write the durable half atomically: a temp file in the same directory + /// followed by a rename, so a crash mid-write leaves the previous state + /// rather than a truncated one. Losing `last_cumulative` while keeping + /// `owed` is the specific corruption this prevents. + pub fn persist(&self) -> Result<(), StoreError> { + let Some(path) = &self.path else { + return Ok(()); + }; + let mut accounts: Vec = self + .accounts + .iter() + .filter(|(_, a)| a.owed_plur > 0 || !a.last_cumulative.is_empty()) + .map(|(k, a)| { + let mut last: Vec<(String, String)> = a + .last_cumulative + .iter() + .map(|(cb, v)| (hex::encode(cb), v.to_string())) + .collect(); + last.sort(); + OnDiskAccount { + account: hex::encode(k), + owed_plur: a.owed_plur.to_string(), + last_cumulative: last, + } + }) + .collect(); + accounts.sort_by(|a, b| a.account.cmp(&b.account)); + let mut binding: Vec<(String, String)> = self + .binding + .iter() + .map(|(cb, a)| (hex::encode(cb), hex::encode(a))) + .collect(); + binding.sort(); + + let disk = OnDisk { + version: 1, + secret_hex: hex::encode(self.secret), + accounts, + binding, + }; + let body = serde_json::to_vec_pretty(&disk)?; + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, &body).map_err(|e| StoreError::Io(e.to_string()))?; + std::fs::rename(&tmp, path).map_err(|e| StoreError::Io(e.to_string()))?; + Ok(()) + } + + pub fn owed(&self, account: &[u8; 20]) -> u128 { + self.accounts.get(account).map_or(0, |a| a.owed_plur) + } + + pub fn reserved(&self, account: &[u8; 20]) -> u128 { + self.accounts.get(account).map_or(0, |a| a.reserved_plur) + } + + pub fn outstanding(&self, account: &[u8; 20]) -> u128 { + self.accounts.get(account).map_or(0, Account::outstanding) + } + + pub fn last_cumulative(&self, account: &[u8; 20], chequebook: &[u8; 20]) -> u128 { + self.accounts + .get(account) + .and_then(|a| a.last_cumulative.get(chequebook)) + .copied() + .unwrap_or(0) + } + + /// Number of accounts holding a live reservation — the cardinality + /// §7.2 says to bound, since the map is attacker-influenced. + pub fn live_reservations(&self) -> usize { + self.accounts + .values() + .filter(|a| a.reserved_plur > 0) + .count() + } + + /// Reserve against a credit line, atomically with respect to every + /// other in-flight request for this account. + /// + /// Always reserves, and *reports* whether it went over rather than + /// deciding: soft mode records the overshoot and serves anyway, hard + /// mode releases and answers 402 (§7.1). Doing the arithmetic in one + /// place means the two modes cannot disagree about what "over" means. + pub fn reserve(&mut self, account: [u8; 20], amount: u128, cap: u128) -> Admission { + let a = self.accounts.entry(account).or_default(); + a.reserved_plur = a.reserved_plur.saturating_add(amount); + let outstanding = a.outstanding(); + Admission { + reserved_plur: amount, + outstanding_plur: outstanding, + cap_plur: cap, + over_cap: outstanding > cap, + } + } + + /// Give back an unused reservation, e.g. after a hard-mode 402. + pub fn release(&mut self, account: [u8; 20], amount: u128) { + if let Some(a) = self.accounts.get_mut(&account) { + a.reserved_plur = a.reserved_plur.saturating_sub(amount); + } + } + + /// Turn a reservation into debt for what was actually admitted, and + /// release the remainder. + pub fn commit(&mut self, account: [u8; 20], reserved: u128, billed: u128) { + let a = self.accounts.entry(account).or_default(); + a.reserved_plur = a.reserved_plur.saturating_sub(reserved); + a.owed_plur = a.owed_plur.saturating_add(billed); + } + + /// Accept a cheque and return the amount it newly credits. + /// + /// Monotonicity is what makes a cheque replay-proof *within a live + /// relay* — a re-presented cheque credits zero — and it is why losing + /// this map across a restart is an unbounded free-service loop (§11.4). + pub fn credit( + &mut self, + account: [u8; 20], + chequebook: [u8; 20], + cumulative_plur: u128, + ) -> Result { + if cumulative_plur > MAX_CUMULATIVE_PLUR { + return Err(LedgerError::Absurd(cumulative_plur)); + } + match self.binding.get(&chequebook) { + Some(bound) if *bound != account => { + return Err(LedgerError::ChequebookBound { + chequebook: hex::encode(chequebook), + }); + } + _ => {} + } + let a = self.accounts.entry(account).or_default(); + let have = a.last_cumulative.get(&chequebook).copied().unwrap_or(0); + if cumulative_plur <= have { + return Err(LedgerError::NotIncreasing { + got: cumulative_plur, + have, + }); + } + let delta = cumulative_plur - have; + // Refuse to bank more than is owed. Postpaid means a client should + // never be ahead (§10), and accepting an overpayment would turn the + // relay into a place to park value it cannot return. + if delta > a.owed_plur { + return Err(LedgerError::Overpayment { + got: delta, + owed: a.owed_plur, + }); + } + a.last_cumulative.insert(chequebook, cumulative_plur); + a.owed_plur -= delta; + self.binding.insert(chequebook, account); + Ok(delta) + } +} + +fn parse_addr(s: &str) -> Result<[u8; 20], StoreError> { + let raw = hex::decode(s.trim_start_matches("0x")) + .map_err(|e| StoreError::Io(format!("address hex: {e}")))?; + if raw.len() != 20 { + return Err(StoreError::Io(format!("address must be 20 bytes: {s}"))); + } + let mut out = [0u8; 20]; + out.copy_from_slice(&raw); + Ok(out) +} + +fn parse_u128(s: &str) -> Result { + s.parse() + .map_err(|e| StoreError::Io(format!("u128 parse {s}: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: [u8; 20] = [1u8; 20]; + const B: [u8; 20] = [2u8; 20]; + const CB: [u8; 20] = [9u8; 20]; + + fn tmpdir() -> PathBuf { + let base = std::env::var("CARGO_TARGET_TMPDIR").unwrap_or_else(|_| "/tmp".into()); + let p = PathBuf::from(base).join(format!("ledger-test-{}", std::process::id())); + std::fs::create_dir_all(&p).expect("mkdir"); + p + } + + #[test] + fn a_reservation_shows_up_as_outstanding_and_releases_cleanly() { + let mut l = Ledger::ephemeral(); + let adm = l.reserve(A, 500, 1000); + assert_eq!(adm.outstanding_plur, 500); + assert!(!adm.over_cap); + assert_eq!(l.reserved(&A), 500); + l.release(A, 500); + assert_eq!(l.outstanding(&A), 0); + } + + /// The concurrency case §10.2 is about: N requests each reserve before + /// any of them completes, so the cap must see their sum. + #[test] + fn concurrent_reservations_accumulate_against_one_cap() { + let mut l = Ledger::ephemeral(); + for _ in 0..7 { + assert!(!l.reserve(A, 100, 1000).over_cap); + } + let adm = l.reserve(A, 100, 1000); + assert_eq!(adm.outstanding_plur, 800); + assert!(!adm.over_cap); + let adm = l.reserve(A, 300, 1000); + assert!(adm.over_cap, "1100 > 1000 must report over cap"); + } + + #[test] + fn commit_turns_a_reservation_into_debt_and_frees_the_rest() { + let mut l = Ledger::ephemeral(); + l.reserve(A, 1000, 10_000); + l.commit(A, 1000, 240); + assert_eq!(l.reserved(&A), 0, "the whole reservation is released"); + assert_eq!(l.owed(&A), 240, "only what was admitted is billed"); + } + + #[test] + fn a_cheque_credits_only_the_delta() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + assert_eq!(l.credit(A, CB, 400).expect("first"), 400); + assert_eq!(l.owed(&A), 600); + assert_eq!(l.credit(A, CB, 900).expect("second"), 500); + assert_eq!(l.owed(&A), 100); + } + + /// Replay within a live relay must be worth exactly zero. + #[test] + fn a_re_presented_cheque_credits_nothing() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + l.credit(A, CB, 400).expect("first"); + assert_eq!( + l.credit(A, CB, 400), + Err(LedgerError::NotIncreasing { + got: 400, + have: 400 + }) + ); + assert_eq!(l.owed(&A), 600, "owed must not move"); + } + + #[test] + fn a_chequebook_cannot_move_between_accounts() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + l.credit(A, CB, 100).expect("bind to A"); + l.commit(B, 0, 1000); + assert!(matches!( + l.credit(B, CB, 500), + Err(LedgerError::ChequebookBound { .. }) + )); + } + + #[test] + fn absurd_and_overpaying_cumulatives_are_refused() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 100); + assert!(matches!( + l.credit(A, CB, MAX_CUMULATIVE_PLUR + 1), + Err(LedgerError::Absurd(_)) + )); + assert!(matches!( + l.credit(A, CB, 101), + Err(LedgerError::Overpayment { got: 101, owed: 100 }) + )); + l.credit(A, CB, 100).expect("paying exactly what is owed is fine"); + assert_eq!(l.owed(&A), 0); + } + + /// The asymmetry the module exists to enforce: debt and the cumulative + /// high-water mark survive a restart; a reservation does not. + #[test] + fn a_restart_keeps_debt_and_cumulative_but_drops_reservations() { + let dir = tmpdir(); + let path = dir.join("restart.json"); + let _ = std::fs::remove_file(&path); + + let secret = { + let mut l = Ledger::load_or_create(&path).expect("create"); + l.commit(A, 0, 5000); + l.credit(A, CB, 1200).expect("pay"); + l.reserve(A, 900, 100_000); + l.persist().expect("persist"); + *l.secret() + }; + + let l = Ledger::load_or_create(&path).expect("reload"); + assert_eq!(l.owed(&A), 3800, "debt survives"); + assert_eq!(l.last_cumulative(&A, &CB), 1200, "cumulative survives"); + assert_eq!( + l.reserved(&A), + 0, + "a reservation must NOT survive — no task remains to release it" + ); + assert_eq!( + l.secret(), + &secret, + "a regenerated secret would 403 every live client" + ); + let _ = std::fs::remove_file(&path); + } + + /// §11.4's attack: pay once, consume, wait for a restart, re-present the + /// same cheque. It must still credit zero. + #[test] + fn a_cheque_cannot_be_replayed_across_a_restart() { + let dir = tmpdir(); + let path = dir.join("replay.json"); + let _ = std::fs::remove_file(&path); + { + let mut l = Ledger::load_or_create(&path).expect("create"); + l.commit(A, 0, 5000); + l.credit(A, CB, 1200).expect("pay"); + l.persist().expect("persist"); + } + let mut l = Ledger::load_or_create(&path).expect("reload"); + assert!( + matches!(l.credit(A, CB, 1200), Err(LedgerError::NotIncreasing { .. })), + "re-presenting the same cheque after a restart must credit nothing" + ); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn the_binding_survives_a_restart_too() { + let dir = tmpdir(); + let path = dir.join("binding.json"); + let _ = std::fs::remove_file(&path); + { + let mut l = Ledger::load_or_create(&path).expect("create"); + l.commit(A, 0, 500); + l.credit(A, CB, 100).expect("bind"); + l.persist().expect("persist"); + } + let mut l = Ledger::load_or_create(&path).expect("reload"); + l.commit(B, 0, 500); + assert!( + matches!(l.credit(B, CB, 200), Err(LedgerError::ChequebookBound { .. })), + "the chequebook binding must survive a restart" + ); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn live_reservations_are_countable_for_shedding() { + let mut l = Ledger::ephemeral(); + assert_eq!(l.live_reservations(), 0); + l.reserve(A, 10, 1000); + l.reserve(B, 10, 1000); + assert_eq!(l.live_reservations(), 2); + l.release(A, 10); + assert_eq!(l.live_reservations(), 1); + } +} diff --git a/src/lib.rs b/src/lib.rs index 1362921..8b2130d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,6 +68,24 @@ pub mod daemon; #[cfg(not(target_arch = "wasm32"))] pub mod inbound; +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod challenge; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod inbound_limit; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod ledger; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod meter; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod metered; + +#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +pub mod payer; + #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub mod pusher; diff --git a/src/meter.rs b/src/meter.rs new file mode 100644 index 0000000..ed69421 --- /dev/null +++ b/src/meter.rs @@ -0,0 +1,780 @@ +//! Stage 0 shadow metering — `docs/pusher-incentives.md` §14. +//! +//! Measures what a *metered* relay would have billed, while changing +//! nothing on the wire. No new client-visible behaviour, no refusals, no +//! payment: the relay simply counts, and an operator reads the counts off +//! `/v1/meter`. +//! +//! It answers the two questions Stage 1 is gated on: +//! +//! 1. **Is anyone consuming enough to justify metering?** Per-account bytes +//! admitted, what they would owe at §9.2's candidate price, and how many +//! accounts would ever cross the cashout threshold — below which an +//! account costs more in gas than it yields (§9.3). +//! 2. **Is `credit_ratio = 1000` right?** For every batch actually seen, the +//! credit line §10.3 would have granted it, against the size of a full +//! POST. A batch whose line is under one POST has to split its uploads, +//! which is fine; a population where most batches are in that state means +//! the ratio is wrong. +//! +//! It also measures §9.1's egress multiplier, which the doc currently +//! *estimates* at ×3 racing × 1.15 shallow ≈ 3.45 attempts per chunk. That +//! number sets the whole cost basis and has never been observed. The push +//! path already counts every per-stream attempt +//! (`src/client.rs:5361-5363`), so the multiplier is that total over the +//! frames this module admitted. +//! +//! **Hot path cost is one lock per POST.** A request accumulates into a +//! [`PostTally`] on its own stack and merges once at completion, so N +//! concurrent POSTs contend N times rather than N × `PUSH_BATCH_MAX`. +//! +//! **Known limitation:** state is in-memory, so a restart resets it. That +//! biases "do accounts return?" downward on hosts that sleep, which is +//! exactly the free tier §5 says must not run metered anyway. Read the +//! window length (`window_secs`) before drawing conclusions from repeat +//! rates. + +use serde_json::json; +use std::collections::{HashMap, VecDeque}; +use std::time::Instant; + +// ── Candidate parameters (docs/pusher-incentives.md §9.2, §10.1) ───────── +// +// Constants rather than flags on purpose: Stage 0 exists to decide whether +// these numbers are right, and every figure in the report is derived from +// the same raw observations, so changing one re-derives the report without +// re-collecting anything. + +/// $0.02/GiB at $0.40/BZZ → 5e14 PLUR/GiB ÷ 1 048 576 KiB. +pub const PRICE_PLUR_PER_KIB: u128 = 480_000_000; +/// ~32 MiB — one settlement window. +pub const SETTLE_EVERY_PLUR: u128 = 15_600_000_000_000; +/// ~127 MiB — the global ceiling on a credit line. +pub const MAX_OUTSTANDING_PLUR: u128 = 62_200_000_000_000; +/// 0.25 BZZ ≈ 5 GiB — below this an account never repays its cashout gas. +pub const CASHOUT_THRESHOLD_PLUR: u128 = 2_500_000_000_000_000; +/// Credit line = batch remaining value ÷ this (§10.3). +pub const CREDIT_RATIO: u128 = 1_000; + +const PLUR_PER_BZZ: f64 = 1e16; +const USD_PER_BZZ: f64 = 0.40; + +/// Distinct `(owner, batch)` pairs held before FIFO eviction. Bounded for +/// the same reason the owner cache is (§16.2): the key is attacker-chosen, +/// so an unbounded map is a memory DoS. Evicted rows keep contributing to +/// the totals, only their per-row detail is lost. +const METER_ROW_CAP: usize = 4096; + +/// A shadow account is the batch-owner EOA (§6), and credit is keyed one +/// level finer, on the batch — so the ledger is keyed on both. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +pub struct AccountBatch { + pub owner: [u8; 20], + pub batch: [u8; 32], +} + +#[derive(Clone)] +struct Row { + /// Body bytes attributable to admitted frames, in KiB (§8). + kib_admitted: u64, + /// Of which served from the recent-ack cache, and so billed at zero + /// (§8.2). + kib_dedup: u64, + frames: u64, + dedup_frames: u64, + /// `remainingBalance × 2^depth` observed at resolution (§6). Zero when + /// the batch was never resolved with a value, which cannot currently + /// happen but is not worth panicking over. + remaining_value_plur: u128, + first_seen: Instant, + last_seen: Instant, +} + +impl Row { + fn billable_kib(&self) -> u64 { + self.kib_admitted.saturating_sub(self.kib_dedup) + } + + /// What §10.3 would have granted this batch. + fn credit_plur(&self) -> u128 { + (self.remaining_value_plur / CREDIT_RATIO).min(MAX_OUTSTANDING_PLUR) + } + + fn credit_kib(&self) -> u64 { + (self.credit_plur() / PRICE_PLUR_PER_KIB).min(u64::MAX as u128) as u64 + } +} + +/// One owner's rollup across every batch it pushed under. The account is +/// what settlement and cashout are keyed on (§6, §9.3), while credit is +/// keyed per batch — so the report needs both views of the same rows. +#[derive(Default, Clone, Copy)] +struct OwnerTotals { + kib: u64, + kib_dedup: u64, + frames: u64, + batches: u64, +} + +impl OwnerTotals { + fn owed_plur(&self) -> u128 { + self.kib.saturating_sub(self.kib_dedup) as u128 * PRICE_PLUR_PER_KIB + } +} + +/// Per-request staging buffer. A POST touches at most +/// `PUSH_MAX_BATCH_LOOKUPS` distinct batches, so a linear scan beats a +/// `HashMap` and allocates nothing in the common single-batch case. +#[derive(Default)] +pub struct PostTally { + rows: Vec, +} + +struct TallyRow { + key: AccountBatch, + remaining_value_plur: u128, + bytes_admitted: u64, + bytes_dedup: u64, + frames: u64, + dedup_frames: u64, +} + +impl PostTally { + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + fn slot(&mut self, key: AccountBatch, remaining_value_plur: u128) -> &mut TallyRow { + if let Some(i) = self.rows.iter().position(|r| r.key == key) { + // A later frame may carry a value the first one lacked; never + // let a zero overwrite a real reading. + if self.rows[i].remaining_value_plur == 0 { + self.rows[i].remaining_value_plur = remaining_value_plur; + } + return &mut self.rows[i]; + } + self.rows.push(TallyRow { + key, + remaining_value_plur, + bytes_admitted: 0, + bytes_dedup: 0, + frames: 0, + dedup_frames: 0, + }); + self.rows.last_mut().expect("just pushed") + } + + /// A frame that passed stamp validation and owner resolution. `bytes` is + /// the body it occupied — header plus wire — which is what §8 bills. + pub fn admit(&mut self, key: AccountBatch, remaining_value_plur: u128, bytes: u64) { + let row = self.slot(key, remaining_value_plur); + row.bytes_admitted += bytes; + row.frames += 1; + } + + /// An admitted frame that the recent-ack cache answered. Counted in + /// `admit` as well — this records the portion billed at zero (§8.2). + pub fn dedup(&mut self, key: AccountBatch, remaining_value_plur: u128, bytes: u64) { + let row = self.slot(key, remaining_value_plur); + row.bytes_dedup += bytes; + row.dedup_frames += 1; + } +} + +/// Bounded shadow ledger over `(owner, batch)`. +pub struct Meter { + rows: HashMap, + order: VecDeque, + cap: usize, + started: Instant, + /// Totals that survive eviction, so the headline figures stay honest + /// even once detail rows are dropped. + evicted_rows: u64, + evicted_kib: u64, + evicted_kib_dedup: u64, + evicted_frames: u64, +} + +impl Default for Meter { + fn default() -> Self { + Self::new(METER_ROW_CAP) + } +} + +impl Meter { + pub fn new(cap: usize) -> Self { + Self { + rows: HashMap::new(), + order: VecDeque::new(), + cap: cap.max(1), + started: Instant::now(), + evicted_rows: 0, + evicted_kib: 0, + evicted_kib_dedup: 0, + evicted_frames: 0, + } + } + + /// Fold one request's tally in. Bytes become KiB here, once per + /// `(POST, batch)` — which is how a metered relay would round, since it + /// bills a whole body against one `Content-Length` (§8). + pub fn merge(&mut self, tally: PostTally) { + let now = Instant::now(); + for t in tally.rows { + let kib = t.bytes_admitted.div_ceil(1024); + let kib_dedup = t.bytes_dedup.div_ceil(1024).min(kib); + match self.rows.get_mut(&t.key) { + Some(row) => { + row.kib_admitted += kib; + row.kib_dedup += kib_dedup; + row.frames += t.frames; + row.dedup_frames += t.dedup_frames; + if t.remaining_value_plur > 0 { + // Latest reading wins: the line decays as the batch + // is spent down, and the decay is the point (§10.3). + row.remaining_value_plur = t.remaining_value_plur; + } + row.last_seen = now; + } + None => { + self.rows.insert( + t.key, + Row { + kib_admitted: kib, + kib_dedup, + frames: t.frames, + dedup_frames: t.dedup_frames, + remaining_value_plur: t.remaining_value_plur, + first_seen: now, + last_seen: now, + }, + ); + self.order.push_back(t.key); + } + } + } + while self.order.len() > self.cap { + let Some(old) = self.order.pop_front() else { + break; + }; + if let Some(row) = self.rows.remove(&old) { + self.evicted_rows += 1; + self.evicted_kib += row.kib_admitted; + self.evicted_kib_dedup += row.kib_dedup; + self.evicted_frames += row.frames; + } + } + } + + fn window_secs(&self) -> u64 { + self.started.elapsed().as_secs() + } + + fn by_owner(&self) -> HashMap<[u8; 20], OwnerTotals> { + let mut out: HashMap<[u8; 20], OwnerTotals> = HashMap::new(); + for (k, r) in &self.rows { + let e = out.entry(k.owner).or_default(); + e.kib += r.kib_admitted; + e.kib_dedup += r.kib_dedup; + e.frames += r.frames; + e.batches += 1; + } + out + } + + /// Headline figures. Names no account, so it is safe on the public + /// `/v1/status` — total volume is already published there as + /// `bytes_pushed`. + /// + /// `full_post_kib` is the body of a maximal POST, and `attempts` the sum + /// of the per-stream push outcome counters (§9.1). + pub fn summary(&self, full_post_kib: u64, attempts: u64) -> serde_json::Value { + let live_kib: u64 = self.rows.values().map(|r| r.kib_admitted).sum(); + let live_dedup: u64 = self.rows.values().map(|r| r.kib_dedup).sum(); + let live_frames: u64 = self.rows.values().map(|r| r.frames).sum(); + let kib = live_kib + self.evicted_kib; + let dedup = live_kib.min(live_dedup) + self.evicted_kib_dedup; + let frames = live_frames + self.evicted_frames; + let billable = kib.saturating_sub(dedup); + let owed = billable as u128 * PRICE_PLUR_PER_KIB; + + let owners = self.by_owner(); + let would_settle = owners + .values() + .filter(|t| t.owed_plur() >= SETTLE_EVERY_PLUR) + .count(); + let would_cash = owners + .values() + .filter(|t| t.owed_plur() >= CASHOUT_THRESHOLD_PLUR) + .count(); + + json!({ + "window_secs": self.window_secs(), + "price_plur_per_kib": PRICE_PLUR_PER_KIB.to_string(), + "accounts": owners.len(), + "batches": self.rows.len(), + "evicted_batches": self.evicted_rows, + "kib_admitted": kib, + "kib_dedup": dedup, + "frames_admitted": frames, + "owed_plur": owed.to_string(), + "owed_usd": plur_to_usd(owed), + // The two questions §9.3 turns on: how many accounts reach one + // settlement at all, and how many ever repay their cashout gas. + "accounts_reaching_settlement": would_settle, + "accounts_reaching_cashout": would_cash, + "egress": egress(frames, billable, attempts), + "credit": self.credit_summary(full_post_kib), + }) + } + + /// §10.3's calibration: what credit line every observed batch would get. + fn credit_summary(&self, full_post_kib: u64) -> serde_json::Value { + let mut lines: Vec = self + .rows + .values() + .filter(|r| r.remaining_value_plur > 0) + .map(Row::credit_kib) + .collect(); + if lines.is_empty() { + return json!({"batches_priced": 0}); + } + lines.sort_unstable(); + let below_post = lines.iter().filter(|&&c| c < full_post_kib).count(); + let capped = self + .rows + .values() + .filter(|r| r.remaining_value_plur > 0) + .filter(|r| r.remaining_value_plur / CREDIT_RATIO < MAX_OUTSTANDING_PLUR) + .count(); + json!({ + "batches_priced": lines.len(), + "credit_ratio": CREDIT_RATIO, + "full_post_kib": full_post_kib, + // A batch here must split its uploads across smaller POSTs + // (§7.2). Fine individually; a high fraction means the ratio is + // mis-set. + "batches_below_one_full_post": below_post, + // Below the global ceiling, i.e. the per-batch line binds. + "batches_capped_below_ceiling": capped, + "credit_kib_p10": pct(&lines, 10), + "credit_kib_p50": pct(&lines, 50), + "credit_kib_p90": pct(&lines, 90), + }) + } + + /// Per-account and per-batch detail. **Operator-only** — this is a + /// volume oracle over on-chain-enumerable batch owners, which is exactly + /// why §7 authenticates `/v1/account`. + pub fn detail(&self, full_post_kib: u64, attempts: u64, top: usize) -> serde_json::Value { + let mut owners: Vec<([u8; 20], OwnerTotals)> = self.by_owner().into_iter().collect(); + owners.sort_by(|a, b| b.1.kib.cmp(&a.1.kib).then(a.0.cmp(&b.0))); + let accounts: Vec = owners + .iter() + .take(top) + .map(|(owner, t)| { + let owed = t.owed_plur(); + json!({ + "account": format!("0x{}", hex::encode(owner)), + "batches": t.batches, + "kib_admitted": t.kib, + "kib_dedup": t.kib_dedup, + "frames": t.frames, + "owed_plur": owed.to_string(), + "owed_usd": plur_to_usd(owed), + "settlements": (owed / SETTLE_EVERY_PLUR) as u64, + "reaches_cashout": owed >= CASHOUT_THRESHOLD_PLUR, + }) + }) + .collect(); + + let mut rows: Vec<(&AccountBatch, &Row)> = self.rows.iter().collect(); + rows.sort_by(|a, b| b.1.kib_admitted.cmp(&a.1.kib_admitted).then(a.0.cmp(b.0))); + let batches: Vec = rows + .iter() + .take(top) + .map(|(k, r)| { + let credit_kib = r.credit_kib(); + json!({ + "account": format!("0x{}", hex::encode(k.owner)), + "batch": format!("0x{}", hex::encode(k.batch)), + "kib_admitted": r.kib_admitted, + "kib_dedup": r.kib_dedup, + "frames": r.frames, + "dedup_frames": r.dedup_frames, + "remaining_value_plur": r.remaining_value_plur.to_string(), + "credit_plur": r.credit_plur().to_string(), + "credit_kib": credit_kib, + "fits_full_post": credit_kib >= full_post_kib, + "billable_kib": r.billable_kib(), + "age_secs": r.first_seen.elapsed().as_secs(), + "idle_secs": r.last_seen.elapsed().as_secs(), + }) + }) + .collect(); + + json!({ + "summary": self.summary(full_post_kib, attempts), + "accounts": accounts, + "batches": batches, + "truncated_to": top, + }) + } +} + +/// §9.1's cost basis, observed rather than estimated. The doc's model is +/// ×3 peer race × 1.15 shallow retries ≈ 3.45 stream attempts per chunk; a +/// materially different number moves the price in §9.2. +fn egress(frames: u64, billable_kib: u64, attempts: u64) -> serde_json::Value { + if frames == 0 { + return json!({"frames": 0, "stream_attempts": attempts}); + } + let per = attempts as f64 / frames as f64; + let mut out = json!({ + "frames": frames, + "stream_attempts": attempts, + // The measured number. Everything below is derived from it. + "attempts_per_frame": round3(per), + "modelled_attempts_per_frame": 3.45, + }); + if billable_kib > 0 { + // Each Delivery is ~4.4 KiB on the wire for a 4 KiB chunk (§9.1: + // addr + stamp + span/data, plus ~5 % protobuf/yamux/noise/TCP). + // Denominated in *billable* payload, since a dedup hit generates no + // attempts and would otherwise dilute the ratio. The doc models 3.7. + let ratio = attempts as f64 * 4.4 / billable_kib as f64; + out["egress_ratio_estimate"] = json!(round3(ratio)); + out["modelled_egress_ratio"] = json!(3.7); + } + out +} + +fn pct(sorted: &[u64], p: usize) -> u64 { + if sorted.is_empty() { + return 0; + } + let i = (sorted.len() * p / 100).min(sorted.len() - 1); + sorted[i] +} + +fn plur_to_usd(plur: u128) -> f64 { + round6(plur as f64 / PLUR_PER_BZZ * USD_PER_BZZ) +} + +fn round3(v: f64) -> f64 { + (v * 1000.0).round() / 1000.0 +} + +fn round6(v: f64) -> f64 { + (v * 1e6).round() / 1e6 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(owner: u8, batch: u8) -> AccountBatch { + AccountBatch { + owner: [owner; 20], + batch: [batch; 32], + } + } + + /// One full frame: 147 B header + 4104 B wire (`src/pushframe.rs`). + const FRAME: u64 = 4251; + + #[test] + fn bytes_round_to_kib_once_per_post_not_once_per_frame() { + // Per-frame rounding would charge 5 KiB for a 4.15 KiB frame — a + // 20 % over-count that would have made every figure in the report + // wrong in the relay's favour. + let mut m = Meter::default(); + let mut t = PostTally::default(); + for _ in 0..100 { + t.admit(key(1, 1), 0, FRAME); + } + m.merge(t); + let want = (FRAME * 100).div_ceil(1024); + assert_eq!(want, 416, "100 frames is 415.14 KiB, ceil 416"); + let s = m.summary(2126, 0); + assert_eq!(s["kib_admitted"], want); + } + + #[test] + fn dedup_is_recorded_but_not_double_counted_as_volume() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + // Ten frames admitted, three of them cache hits: dedup is a subset + // of admitted, never an addition to it (§8.2). + for _ in 0..10 { + t.admit(key(1, 1), 0, FRAME); + } + for _ in 0..3 { + t.dedup(key(1, 1), 0, FRAME); + } + m.merge(t); + let s = m.summary(2126, 0); + assert_eq!(s["kib_admitted"], (FRAME * 10).div_ceil(1024)); + assert_eq!(s["kib_dedup"], (FRAME * 3).div_ceil(1024)); + let billable = s["kib_admitted"].as_u64().expect("u64") + - s["kib_dedup"].as_u64().expect("u64"); + assert_eq!( + s["owed_plur"].as_str().expect("string"), + (billable as u128 * PRICE_PLUR_PER_KIB).to_string() + ); + } + + #[test] + fn credit_line_follows_batch_value_and_caps_at_the_ceiling() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + // 0.01 BZZ = 1e14 PLUR → 1e11 credit → ~208 KiB (§10.3's worked + // example). Must not saturate the global ceiling. + t.admit(key(1, 1), 100_000_000_000_000, FRAME); + // 100 BZZ is far past the ceiling, so the line clamps. + t.admit(key(2, 2), 1_000_000_000_000_000_000, FRAME); + m.merge(t); + let d = m.detail(2126, 0, 10); + let by_batch: HashMap = d["batches"] + .as_array() + .expect("array") + .iter() + .map(|b| (b["account"].as_str().expect("acct").to_string(), b.clone())) + .collect(); + let dust = &by_batch[&format!("0x{}", hex::encode([1u8; 20]))]; + assert_eq!(dust["credit_kib"], 208, "0.01 BZZ batch earns ~208 KiB"); + assert_eq!(dust["fits_full_post"], false, "dust must split its POSTs"); + let rich = &by_batch[&format!("0x{}", hex::encode([2u8; 20]))]; + assert_eq!( + rich["credit_plur"].as_str().expect("string"), + MAX_OUTSTANDING_PLUR.to_string(), + "a rich batch clamps to the global ceiling" + ); + assert_eq!(rich["fits_full_post"], true); + } + + #[test] + fn eviction_is_bounded_and_totals_survive_it() { + let mut m = Meter::new(4); + for i in 0..64u8 { + let mut t = PostTally::default(); + t.admit(key(i, i), 0, 1024); + m.merge(t); + } + assert_eq!(m.rows.len(), 4, "rows stay at cap"); + assert_eq!(m.order.len(), 4, "order stays at cap"); + // The whole point of carrying evicted totals: the headline volume + // must not silently shrink as detail rows are dropped. + let s = m.summary(2126, 0); + assert_eq!(s["kib_admitted"], 64, "all 64 KiB still counted"); + assert_eq!(s["evicted_batches"], 60); + } + + #[test] + fn a_later_reading_updates_the_batch_value_but_zero_never_clobbers_it() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + t.admit(key(1, 1), 0, FRAME); // resolved without a value + t.admit(key(1, 1), 100_000_000_000_000, FRAME); // then with one + m.merge(t); + let mut t2 = PostTally::default(); + t2.admit(key(1, 1), 0, FRAME); // a valueless frame must not erase it + m.merge(t2); + let d = m.detail(2126, 0, 10); + assert_eq!( + d["batches"][0]["remaining_value_plur"] + .as_str() + .expect("string"), + "100000000000000" + ); + } + + #[test] + fn egress_multiplier_is_attempts_over_frames() { + let mut m = Meter::default(); + let mut t = PostTally::default(); + for _ in 0..100 { + t.admit(key(1, 1), 0, FRAME); + } + m.merge(t); + // 345 stream attempts for 100 chunks is the doc's modelled 3.45. + let s = m.summary(2126, 345); + assert_eq!(s["egress"]["attempts_per_frame"], 3.45); + // …which lands within a whisker of §9.1's modelled 3.7 GiB of real + // egress per GiB of payload. If a live relay reports something far + // from this, §9.2's price is wrong. + let ratio = s["egress"]["egress_ratio_estimate"].as_f64().expect("f64"); + assert!( + (3.6..3.8).contains(&ratio), + "modelled attempts should reproduce the modelled ratio, got {ratio}" + ); + } + + #[test] + fn cashout_and_settlement_thresholds_classify_accounts() { + let mut m = Meter::default(); + // 5 GiB is the cashout threshold (§9.3); 32 MiB is one settlement. + let five_gib_kib = 5 * 1024 * 1024; + let mut t = PostTally::default(); + t.admit(key(9, 9), 0, five_gib_kib * 1024); + t.admit(key(3, 3), 0, 40 * 1024 * 1024); // ~40 MiB, one settlement + t.admit(key(4, 4), 0, 1024 * 1024); // 1 MiB, neither + m.merge(t); + let s = m.summary(2126, 0); + assert_eq!(s["accounts_reaching_cashout"], 1); + assert_eq!(s["accounts_reaching_settlement"], 2); + } +} + +/// ~8 MiB — the dust floor. Exists only to bound RPC cost per unit of +/// value (§11.6 lists up to 4 `eth_call`s per cheque), *not* to cover +/// cashout gas, which cumulative cheques amortize separately (§8.3). +pub const MIN_CHEQUE_PLUR: u128 = 3_900_000_000_000; + +/// Metered-mode parameters, quoted in `/v1/status` and enforced at +/// admission. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Params { + pub price_plur_per_kib: u128, + pub min_cheque_plur: u128, + pub settle_every_plur: u128, + pub max_outstanding_plur: u128, + pub credit_ratio: u128, +} + +impl Default for Params { + fn default() -> Self { + Self { + price_plur_per_kib: PRICE_PLUR_PER_KIB, + min_cheque_plur: MIN_CHEQUE_PLUR, + settle_every_plur: SETTLE_EVERY_PLUR, + max_outstanding_plur: MAX_OUTSTANDING_PLUR, + credit_ratio: CREDIT_RATIO, + } + } +} + +impl Params { + /// §10.1's invariant, checked at startup so a misconfigured relay + /// refuses to boot rather than bricking every account it serves. + /// + /// Violating it is not a tuning mistake, it is a deadlock: an account + /// accrues, crosses `settle_every`, signs a cheque, has it **rejected as + /// dust**, keeps accruing, hits its cap, and the only cheque that would + /// clear the 402 is larger than what it owes — which the no-prepayment + /// rule forbids. There is no exit. + pub fn validate(&self) -> Result<(), String> { + if self.price_plur_per_kib == 0 { + return Err("price_plur_per_kib must be non-zero".into()); + } + if self.credit_ratio == 0 { + return Err("credit_ratio must be non-zero".into()); + } + if self.min_cheque_plur > self.settle_every_plur { + return Err(format!( + "min_cheque_plur ({}) exceeds settle_every_plur ({}): every account \ + would accrue past settlement and have its cheque refused as dust, \ + with no cheque able to clear the resulting 402 (§10.1)", + self.min_cheque_plur, self.settle_every_plur + )); + } + if self.settle_every_plur >= self.max_outstanding_plur { + return Err(format!( + "settle_every_plur ({}) must be below max_outstanding_plur ({}): \ + otherwise an account hits its cap before it is ever asked to pay (§10.1)", + self.settle_every_plur, self.max_outstanding_plur + )); + } + Ok(()) + } + + /// §10.3: scale the credit line to the batch's on-chain value rather + /// than asserting a constant, so the Sybil margin is `credit_ratio` by + /// construction and independent of batch size. + pub fn credit_line(&self, remaining_value_plur: u128) -> u128 { + (remaining_value_plur / self.credit_ratio).min(self.max_outstanding_plur) + } + + /// What a body of `bytes` costs. Rounds up to KiB **once**, per §8 — + /// rounding per frame would over-count a 4 251-byte frame by 20 %. + pub fn price_bytes(&self, bytes: u64) -> u128 { + u128::from(bytes.div_ceil(1024)) * self.price_plur_per_kib + } +} + +#[cfg(test)] +mod param_tests { + use super::*; + + #[test] + fn the_shipped_defaults_satisfy_the_invariant() { + Params::default().validate().expect("defaults must be valid"); + } + + /// The exact misconfiguration an early draft published: a dust floor 87× + /// larger than the settlement window. + #[test] + fn a_dust_floor_above_the_settlement_window_is_refused_at_startup() { + let p = Params { + min_cheque_plur: SETTLE_EVERY_PLUR * 87, + ..Params::default() + }; + let e = p.validate().expect_err("must refuse to boot"); + assert!(e.contains("no exit") || e.contains("dust"), "got: {e}"); + } + + #[test] + fn a_cap_at_or_below_the_settlement_window_is_refused() { + let p = Params { + max_outstanding_plur: SETTLE_EVERY_PLUR, + ..Params::default() + }; + p.validate().expect_err("cap must exceed the settlement window"); + } + + #[test] + fn zero_price_or_ratio_is_refused() { + Params { + price_plur_per_kib: 0, + ..Params::default() + } + .validate() + .expect_err("zero price"); + Params { + credit_ratio: 0, + ..Params::default() + } + .validate() + .expect_err("zero ratio would divide by zero"); + } + + /// §10.3's worked example, and the §7.2 consequence: a dust batch gets a + /// usable line that is nonetheless too small for one full POST. + #[test] + fn a_dust_batch_gets_a_small_but_usable_line() { + let p = Params::default(); + let line = p.credit_line(100_000_000_000_000); // 0.01 BZZ + assert_eq!(line / p.price_plur_per_kib, 208, "~208 KiB of credit"); + assert!(line < p.max_outstanding_plur, "must not reach the ceiling"); + } + + #[test] + fn a_rich_batch_clamps_to_the_global_ceiling() { + let p = Params::default(); + assert_eq!( + p.credit_line(u128::MAX / 2), + p.max_outstanding_plur, + "the ceiling binds however rich the batch" + ); + } + + #[test] + fn a_body_is_priced_by_rounding_up_once() { + let p = Params::default(); + assert_eq!(p.price_bytes(1), p.price_plur_per_kib, "a partial KiB is a KiB"); + assert_eq!(p.price_bytes(1024), p.price_plur_per_kib); + assert_eq!(p.price_bytes(1025), 2 * p.price_plur_per_kib); + // One full frame, priced once rather than per-frame. + assert_eq!(p.price_bytes(4251), 5 * p.price_plur_per_kib); + } +} diff --git a/src/metered.rs b/src/metered.rs new file mode 100644 index 0000000..23df442 --- /dev/null +++ b/src/metered.rs @@ -0,0 +1,694 @@ +//! Metered relay mode — `docs/pusher-incentives.md` Stage 1. +//! +//! Holds the state the metered endpoints share and implements their logic, +//! so `src/pusher.rs` stays a router. Everything here defends the **relay** +//! against the **client** (§2); nothing here tries to prove to a client that +//! the relay did its work. +//! +//! Stage 1 runs in **soft mode**: the relay meters, reports, and accepts +//! cheques, but never refuses a push. Hard mode (402 enforcement) is +//! Stage 2 and flips one flag — the arithmetic that decides "over cap" is +//! already computed here so the two modes cannot disagree about it. + +use crate::challenge::{ChallengeError, ChallengeFields}; +use crate::inbound_limit::InboundLimiter; +use crate::ledger::{Ledger, LedgerError}; +use crate::meter::Params; +use alloy_primitives::Address; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// Header carrying the capability plus the client's proof it holds the +/// account key. One custom header, so the CORS preflight allow-list grows +/// by exactly one entry (§7.2's browser blocker). +pub const CHALLENGE_HEADER: &str = "x-hoverfly-challenge"; + +/// Cap on the header itself. The fields are fixed-width apart from +/// `origin`, so anything larger is not a challenge. +const MAX_CHALLENGE_HEADER: usize = 2048; + +/// Accounts allowed to hold a live reservation at once (§7.2). The map is +/// attacker-influenced — one entry per batch in standing — so it is capped +/// and sheds beyond. +const MAX_LIVE_RESERVATIONS: usize = 4096; + +/// Chequebook-deployment answers cached per address. `true` never changes, +/// and `false` is exactly what an attacker replays, so both are cached +/// (§11.6). +const DEPLOYED_CACHE_CAP: usize = 4096; +const DEPLOYED_OK_TTL: Duration = Duration::from_secs(86_400); +const DEPLOYED_BAD_TTL: Duration = Duration::from_secs(600); + +#[derive(Debug, Clone)] +pub struct MeterConfig { + /// Configured hostnames. **Never derived from a request header** — see + /// `challenge::verify`. + pub origins: Vec, + pub beneficiary: [u8; 20], + pub chain_id: u64, + pub factory: Address, + pub params: Params, + /// Stage 2. False = soft mode: record the overshoot, serve anyway. + pub hard_mode: bool, +} + +pub struct Metered { + pub cfg: MeterConfig, + pub ledger: Mutex, + /// Per-IP: `/v1/challenge` runs before any account exists. + challenge_limit: Mutex, + /// Per-account: `/v1/pay` and `/v1/push`. + account_limit: Mutex, + deployed: Mutex, +} + +impl Metered { + pub fn new(cfg: MeterConfig, ledger: Ledger) -> Self { + Self { + cfg, + ledger: Mutex::new(ledger), + // A client needs one challenge per POST batch, so a handful per + // second sustained is generous; the burst covers a pipelined + // upload opening several lanes at once. + challenge_limit: Mutex::new(InboundLimiter::new(5.0, 40.0, 8192)), + account_limit: Mutex::new(InboundLimiter::new(20.0, 120.0, 8192)), + deployed: Mutex::new(DeployedCache::new(DEPLOYED_CACHE_CAP)), + } + } + + pub fn allow_challenge(&self, ip: &str) -> bool { + self.challenge_limit + .lock() + .expect("challenge limiter poisoned") + .allow(ip.as_bytes()) + } + + pub fn allow_account(&self, account: &[u8; 20]) -> bool { + self.account_limit + .lock() + .expect("account limiter poisoned") + .allow(account) + } + + /// Issue a capability for a batch in good standing. + /// + /// Standing is resolved by the caller (it owns the RPC path and its + /// cache); this turns it into a credit line and a MAC. The result is + /// what makes `/v1/push` admission chain-free. + pub fn issue( + &self, + account: [u8; 20], + batch: [u8; 32], + remaining_value_plur: u128, + origin: &str, + now: u64, + ) -> Result { + let cap = self.cfg.params.credit_line(remaining_value_plur); + let fields = ChallengeFields { + account, + batch, + origin: origin.to_string(), + expiry_unix: now + crate::challenge::CHALLENGE_TTL_SECS, + cap_plur: cap, + }; + let secret = *self.ledger.lock().expect("ledger poisoned").secret(); + let nonce = crate::challenge::nonce(&secret, &fields)?; + Ok(IssuedChallenge { fields, nonce }) + } + + /// Verify a presented challenge header end to end. + /// + /// Two independent proofs, and both are required: + /// 1. our MAC over the nonce — this relay issued this capability; + /// 2. the client's EIP-712 signature — the caller holds the account key + /// *and* signed for this origin (§11.1). + /// + /// A valid MAC alone would let anyone who observed a challenge use it, + /// and a valid signature alone would let a challenge issued by another + /// relay be presented here. + pub fn verify_header(&self, raw: &str, now: u64) -> Result { + if raw.len() > MAX_CHALLENGE_HEADER { + return Err(format!("challenge header too large: {} bytes", raw.len())); + } + let presented = PresentedChallenge::decode(raw)?; + let secret = *self.ledger.lock().expect("ledger poisoned").secret(); + crate::challenge::verify( + &secret, + &presented.fields, + &presented.nonce, + now, + &self.cfg.origins, + ) + .map_err(|e| e.to_string())?; + + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(presented.nonce), + origin: presented.fields.origin.clone(), + account: Address::from(presented.fields.account), + batchId: alloy_primitives::B256::from(presented.fields.batch), + expiry: alloy_primitives::U256::from(presented.fields.expiry_unix), + }; + let signer = crate::signer::recover_push_challenge(&sol, self.cfg.chain_id, &presented.sig) + .map_err(|e| format!("challenge signature: {e}"))?; + if signer != presented.fields.account { + return Err(format!( + "challenge signed by 0x{} but claims account 0x{}", + hex::encode(signer), + hex::encode(presented.fields.account) + )); + } + Ok(VerifiedChallenge { + account: presented.fields.account, + batch: presented.fields.batch, + cap_plur: presented.fields.cap_plur, + }) + } + + /// Is this address a chequebook our canonical factory deployed? + /// Cached both ways — an uncached miss is a one-RPC-per-request + /// amplifier on an endpoint an attacker reaches for free (§11.6). + pub async fn is_deployed(&self, rpc_url: &str, chequebook: [u8; 20]) -> Result { + if let Some(hit) = self + .deployed + .lock() + .expect("deployed cache poisoned") + .get(&chequebook) + { + return Ok(hit); + } + let ok = crate::batch::is_deployed_chequebook( + rpc_url, + self.cfg.factory, + Address::from(chequebook), + ) + .await + .map_err(|e| format!("factory lookup: {e}"))?; + self.deployed + .lock() + .expect("deployed cache poisoned") + .insert(chequebook, ok); + Ok(ok) + } + + /// Reserve for a request whose body is `content_length` bytes. + /// + /// The reservation and the eventual bill are the *same* quantity + /// computed the same way (§7.2), so there is no estimate to be wrong + /// about and no flat over-reserve to lock small batches out. + pub fn reserve_for_body( + &self, + account: [u8; 20], + content_length: u64, + cap: u128, + ) -> crate::ledger::Admission { + let amount = self.cfg.params.price_bytes(content_length); + let mut l = self.ledger.lock().expect("ledger poisoned"); + l.reserve(account, amount, cap) + } + + pub fn shed_reservations(&self) -> bool { + self.ledger.lock().expect("ledger poisoned").live_reservations() >= MAX_LIVE_RESERVATIONS + } + + /// Apply a cheque. Every free check runs before this is called; this is + /// the ledger half only. + pub fn credit( + &self, + account: [u8; 20], + chequebook: [u8; 20], + cumulative: u128, + ) -> Result { + let mut l = self.ledger.lock().expect("ledger poisoned"); + let accepted = l.credit(account, chequebook, cumulative)?; + // Persist immediately: the window between accepting a cheque and + // durably recording its cumulative is exactly §11.4's replay hole. + if let Err(e) = l.persist() { + tracing::error!("ledger persist after credit failed: {e}"); + } + Ok(accepted) + } +} + +pub struct IssuedChallenge { + pub fields: ChallengeFields, + pub nonce: [u8; 32], +} + +impl IssuedChallenge { + pub fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "nonce": format!("0x{}", hex::encode(self.nonce)), + "account": format!("0x{}", hex::encode(self.fields.account)), + "batch": format!("0x{}", hex::encode(self.fields.batch)), + "origin": self.fields.origin, + "expiry": self.fields.expiry_unix, + "max_outstanding_plur": self.fields.cap_plur.to_string(), + "expires_ms": self.fields.expiry_unix.saturating_mul(1000), + }) + } +} + +#[derive(Debug, Clone)] +pub struct VerifiedChallenge { + pub account: [u8; 20], + pub batch: [u8; 32], + pub cap_plur: u128, +} + +/// What the client sends back: the capability it was issued plus its +/// signature over the same fields. +struct PresentedChallenge { + fields: ChallengeFields, + nonce: [u8; 32], + sig: [u8; 65], +} + +impl PresentedChallenge { + /// `base64(json)` in one header, so the CORS allow-list grows by one. + fn decode(raw: &str) -> Result { + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(raw.trim()) + .map_err(|e| format!("challenge header base64: {e}"))?; + let v: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| format!("challenge header json: {e}"))?; + let get = |k: &str| -> Result { + v.get(k) + .and_then(|x| x.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| format!("challenge header: missing {k}")) + }; + let fixed = |k: &str, n: usize| -> Result, String> { + let raw = hex::decode(get(k)?.trim_start_matches("0x")) + .map_err(|e| format!("challenge {k} hex: {e}"))?; + if raw.len() != n { + return Err(format!("challenge {k} must be {n} bytes, got {}", raw.len())); + } + Ok(raw) + }; + let mut account = [0u8; 20]; + account.copy_from_slice(&fixed("account", 20)?); + let mut batch = [0u8; 32]; + batch.copy_from_slice(&fixed("batch", 32)?); + let mut nonce = [0u8; 32]; + nonce.copy_from_slice(&fixed("nonce", 32)?); + let mut sig = [0u8; 65]; + sig.copy_from_slice(&fixed("sig", 65)?); + let expiry_unix = v + .get("expiry") + .and_then(|x| x.as_u64()) + .ok_or("challenge header: missing expiry")?; + let cap_plur: u128 = get("max_outstanding_plur")? + .parse() + .map_err(|e| format!("challenge cap: {e}"))?; + Ok(Self { + fields: ChallengeFields { + account, + batch, + origin: get("origin")?, + expiry_unix, + cap_plur, + }, + nonce, + sig, + }) + } +} + +/// Encode a challenge plus signature into the header value. Client side, +/// and used by the tests to drive the relay path end to end. +pub fn encode_challenge_header( + issued: &IssuedChallenge, + sig: &[u8; 65], +) -> String { + use base64::Engine; + let body = serde_json::json!({ + "nonce": format!("0x{}", hex::encode(issued.nonce)), + "account": format!("0x{}", hex::encode(issued.fields.account)), + "batch": format!("0x{}", hex::encode(issued.fields.batch)), + "origin": issued.fields.origin, + "expiry": issued.fields.expiry_unix, + "max_outstanding_plur": issued.fields.cap_plur.to_string(), + "sig": format!("0x{}", hex::encode(sig)), + }); + base64::engine::general_purpose::STANDARD.encode(body.to_string()) +} + +struct DeployedCache { + map: HashMap<[u8; 20], (bool, Instant)>, + order: std::collections::VecDeque<[u8; 20]>, + cap: usize, +} + +impl DeployedCache { + fn new(cap: usize) -> Self { + Self { + map: HashMap::new(), + order: std::collections::VecDeque::new(), + cap: cap.max(1), + } + } + + fn get(&self, k: &[u8; 20]) -> Option { + let (v, at) = self.map.get(k)?; + let ttl = if *v { DEPLOYED_OK_TTL } else { DEPLOYED_BAD_TTL }; + (at.elapsed() < ttl).then_some(*v) + } + + fn insert(&mut self, k: [u8; 20], v: bool) { + if self.map.insert(k, (v, Instant::now())).is_none() { + self.order.push_back(k); + } + while self.order.len() > self.cap { + if let Some(old) = self.order.pop_front() { + self.map.remove(&old); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::signer::SwarmSigner; + + const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + + fn signer() -> SwarmSigner { + SwarmSigner::from_hex_with_nonce(KEY, &format!("0x{}", hex::encode([0u8; 32])), 1) + .expect("key") + } + + fn metered() -> Metered { + Metered::new( + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: Address::ZERO, + params: Params::default(), + hard_mode: false, + }, + Ledger::ephemeral(), + ) + } + + /// Issue → sign → present → verify, the whole admission path. + fn round_trip(m: &Metered, now: u64) -> Result { + let s = signer(); + let account = *s.eth_address(); + let issued = m + .issue(account, [5u8; 32], 6_200_000_000_000_000_000, "relay-a.example", now) + .expect("issue"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(account), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + m.verify_header(&header, now) + } + + #[test] + fn a_challenge_we_issued_and_the_client_signed_is_admitted() { + let m = metered(); + let v = round_trip(&m, 1000).expect("must admit"); + assert_eq!(v.account, *signer().eth_address()); + assert_eq!(v.batch, [5u8; 32]); + assert_eq!( + v.cap_plur, + Params::default().max_outstanding_plur, + "a rich batch clamps to the ceiling" + ); + } + + /// A capability alone is not enough — the caller must prove it holds the + /// account key, or a harvested header would be usable by anyone. + #[test] + fn a_challenge_signed_by_the_wrong_key_is_refused() { + let m = metered(); + let victim = *signer().eth_address(); + let issued = m + .issue(victim, [5u8; 32], 1_000_000_000_000_000, "relay-a.example", 1000) + .expect("issue"); + let attacker = SwarmSigner::from_hex_with_nonce( + "0x1111111111111111111111111111111111111111111111111111111111111111", + &format!("0x{}", hex::encode([0u8; 32])), + 1, + ) + .expect("key"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(victim), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = attacker.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = m.verify_header(&header, 1000).expect_err("must refuse"); + assert!(e.contains("claims account"), "got: {e}"); + } + + /// §11.1: a signature gathered at relay A must be useless at relay B. + #[test] + fn a_challenge_for_another_relay_is_refused() { + let a = metered(); + let mut b_cfg = a.cfg.clone(); + b_cfg.origins = vec!["relay-b.example".into()]; + let b = Metered::new(b_cfg, Ledger::ephemeral()); + + let s = signer(); + let issued = a + .issue(*s.eth_address(), [5u8; 32], 1_000_000_000_000_000, "relay-a.example", 1000) + .expect("issue"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = b.verify_header(&header, 1000).expect_err("must refuse"); + assert!(e.contains("origin"), "got: {e}"); + } + + #[test] + fn an_expired_challenge_is_refused() { + let m = metered(); + let e = round_trip(&m, 1000).map(|_| ()).and(Ok(())); + assert!(e.is_ok()); + // Same issue instant, far-future presentation. + let s = signer(); + let issued = m + .issue(*s.eth_address(), [5u8; 32], 1_000_000_000_000_000, "relay-a.example", 1000) + .expect("issue"); + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = m + .verify_header(&header, 1000 + crate::challenge::CHALLENGE_TTL_SECS + 1) + .expect_err("must refuse"); + assert!(e.contains("expired"), "got: {e}"); + } + + /// The cap is inside the MAC, so a client cannot present a dust batch's + /// id alongside a rich batch's credit line. + #[test] + fn an_inflated_cap_is_refused() { + let m = metered(); + let s = signer(); + let mut issued = m + .issue(*s.eth_address(), [5u8; 32], 100_000_000_000_000, "relay-a.example", 1000) + .expect("issue"); + let honest_cap = issued.fields.cap_plur; + issued.fields.cap_plur = honest_cap * 1_000_000; + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(issued.nonce), + origin: issued.fields.origin.clone(), + account: Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from(issued.fields.batch), + expiry: alloy_primitives::U256::from(issued.fields.expiry_unix), + }; + let sig = s.sign_push_challenge(&sol, 100).expect("sign"); + let header = encode_challenge_header(&issued, &sig); + let e = m.verify_header(&header, 1000).expect_err("must refuse"); + assert!(e.contains("not ours"), "got: {e}"); + } + + #[test] + fn malformed_headers_are_refused_not_panicked_on() { + let m = metered(); + for raw in ["", "!!!!", "eyJ9", "e30="] { + m.verify_header(raw, 1000).expect_err("must refuse"); + } + m.verify_header(&"A".repeat(MAX_CHALLENGE_HEADER + 1), 1000) + .expect_err("oversized"); + } + + #[test] + fn the_reservation_is_the_price_of_the_declared_body() { + let m = metered(); + let p = Params::default(); + let adm = m.reserve_for_body([1u8; 20], 4251, p.max_outstanding_plur); + assert_eq!(adm.reserved_plur, p.price_bytes(4251)); + assert!(!adm.over_cap); + } + + /// §7.2's small-batch case: a one-frame POST from a dust batch fits, + /// where an earlier design's flat `PUSH_BATCH_MAX × price` reserve would + /// have 402'd it. + #[test] + fn a_dust_batch_can_afford_a_small_post() { + let m = metered(); + let cap = Params::default().credit_line(100_000_000_000_000); + let adm = m.reserve_for_body([1u8; 20], 4251, cap); + assert!(!adm.over_cap, "a single frame must fit a dust batch's line"); + let m2 = metered(); + let big = m2.reserve_for_body([1u8; 20], 512 * 4251, cap); + assert!(big.over_cap, "but a full 512-frame POST does not"); + } +} + +#[cfg(test)] +mod lifecycle_tests { + //! The full soft-mode money path over the real ledger: admit → bill → + //! settle → replay. These exercise `Metered` + `Ledger` together, which + //! is where the accounting can silently drift. + use super::*; + use crate::meter::Params; + + fn m() -> Metered { + Metered::new( + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: Address::ZERO, + params: Params::default(), + hard_mode: false, + }, + Ledger::ephemeral(), + ) + } + + const ACCT: [u8; 20] = [1u8; 20]; + const CB: [u8; 20] = [9u8; 20]; + + /// One POST: reserve on the declared body, bill what was admitted, + /// release the rest. The reservation must not survive as phantom debt. + #[test] + fn a_request_reserves_then_commits_only_what_it_admitted() { + let m = m(); + let p = Params::default(); + let cap = p.max_outstanding_plur; + let adm = m.reserve_for_body(ACCT, 512 * 4251, cap); + assert_eq!(m.ledger.lock().unwrap().reserved(&ACCT), adm.reserved_plur); + + // Only 100 frames actually got admitted. + let billed = p.price_bytes(100 * 4251); + m.ledger.lock().unwrap().commit(ACCT, adm.reserved_plur, billed); + let l = m.ledger.lock().unwrap(); + assert_eq!(l.reserved(&ACCT), 0, "the reservation is fully released"); + assert_eq!(l.owed(&ACCT), billed, "only admitted bytes are billed"); + assert!(billed < adm.reserved_plur, "and it is less than was reserved"); + } + + /// A POST that admits nothing must still hand its reservation back — + /// leaking it ratchets the account toward a 402 it can never clear. + #[test] + fn a_request_that_admits_nothing_leaks_no_credit() { + let m = m(); + let adm = m.reserve_for_body(ACCT, 512 * 4251, Params::default().max_outstanding_plur); + m.ledger.lock().unwrap().commit(ACCT, adm.reserved_plur, 0); + let l = m.ledger.lock().unwrap(); + assert_eq!(l.outstanding(&ACCT), 0, "nothing owed, nothing reserved"); + } + + /// Settle, then push again: the cumulative carries forward, so the + /// second cheque credits only the new debt. + #[test] + fn settlement_across_two_uploads_uses_one_growing_cumulative() { + let m = m(); + let p = Params::default(); + let first = p.price_bytes(40 * 1024 * 1024); + m.ledger.lock().unwrap().commit(ACCT, 0, first); + + let accepted = m.credit(ACCT, CB, first).expect("first cheque"); + assert_eq!(accepted, first); + assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); + + let second = p.price_bytes(32 * 1024 * 1024); + m.ledger.lock().unwrap().commit(ACCT, 0, second); + // A cumulative cheque: the *total*, not the delta. + let accepted = m.credit(ACCT, CB, first + second).expect("second cheque"); + assert_eq!(accepted, second, "only the new debt is credited"); + assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); + } + + /// Soft mode reports the overshoot but still admits — that is the whole + /// point of Stage 1 shipping soft. + #[test] + fn soft_mode_reports_an_overshoot_without_refusing() { + let m = m(); + let cap = Params::default().credit_line(100_000_000_000_000); // dust + let adm = m.reserve_for_body(ACCT, 512 * 4251, cap); + assert!(adm.over_cap, "a full POST exceeds a dust batch's line"); + assert!(!m.cfg.hard_mode, "Stage 1 ships soft"); + assert!( + m.ledger.lock().unwrap().reserved(&ACCT) > 0, + "soft mode still reserves, so the measurement is real" + ); + } + + /// The same request under hard mode must leave no trace once refused. + #[test] + fn hard_mode_releases_the_reservation_it_refuses() { + let mut cfg = m().cfg.clone(); + cfg.hard_mode = true; + let m = Metered::new(cfg, Ledger::ephemeral()); + let cap = Params::default().credit_line(100_000_000_000_000); + let adm = m.reserve_for_body(ACCT, 512 * 4251, cap); + assert!(adm.over_cap); + m.ledger.lock().unwrap().release(ACCT, adm.reserved_plur); + assert_eq!( + m.ledger.lock().unwrap().outstanding(&ACCT), + 0, + "a refused request must not leave phantom debt behind" + ); + } + + /// §10.1's invariant in motion: a client 402'd at the cap can always + /// clear it with a cheque for exactly what it owes. + #[test] + fn an_account_at_its_cap_can_always_pay_its_way_out() { + let m = m(); + let p = Params::default(); + let cap = p.max_outstanding_plur; + // Accrue right up to the ceiling. + m.ledger.lock().unwrap().commit(ACCT, 0, cap); + let owed = m.ledger.lock().unwrap().owed(&ACCT); + assert!( + owed >= p.min_cheque_plur, + "what is owed must clear the dust floor, or there is no exit" + ); + m.credit(ACCT, CB, owed).expect("a cheque for exactly what is owed"); + assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); + assert!( + !m.reserve_for_body(ACCT, 4251, cap).over_cap, + "and the account can push again" + ); + } +} diff --git a/src/payer.rs b/src/payer.rs new file mode 100644 index 0000000..7c87ac4 --- /dev/null +++ b/src/payer.rs @@ -0,0 +1,681 @@ +//! Client-side payment for metered lanes — `docs/pusher-incentives.md` +//! Stage 1, client half. +//! +//! Four jobs, in the order a client meets them: +//! +//! 1. **Pin the lane.** Parse and *verify* the signed quote from +//! `/v1/status`, checking it against the identity in config rather than +//! trusting what the lane says about itself (§7.3). +//! 2. **Get admitted.** Fetch a challenge, sign it, and carry the header on +//! every `/v1/push` and `/v1/pay`. +//! 3. **Size the POST.** The challenge returns the credit line; a body that +//! would exceed it is split rather than sent and refused (§7.2). +//! 4. **Settle.** Track what is owed by the same arithmetic the relay uses, +//! and issue a cumulative cheque when it crosses `settle_every`. +//! +//! The client computes its bill from **bytes it sent**, not from anything +//! the relay reports. That is the property §8 is built on, and it is why +//! there is nothing here that verifies the relay's work: a disagreement is +//! arithmetic, visible immediately, and settled by not paying. + +use crate::meter::Params; + +/// A lane's signed `payment` block, after verification. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PaymentQuote { + pub beneficiary: [u8; 20], + /// Recovered from `sig`. **This is what a client pins**, not the + /// overlay — see [`PaymentQuote::verify`]. + pub node_eth_address: [u8; 20], + pub overlay_nonce: [u8; 32], + pub origin: String, + pub chain_id: u64, + pub params: Params, + /// True when the relay enforces 402. Soft-mode lanes bill but serve. + pub hard_enforcement: bool, +} + +/// What a client pins in config for a lane it is willing to pay. +/// +/// `PUSHER_URLS` is already a hardcoded list, so carrying two more fields +/// per entry costs nothing — and reading the beneficiary from `/v1/status` +/// at runtime instead would mean paying whoever answers the URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LanePin { + pub node_eth_address: [u8; 20], + pub beneficiary: [u8; 20], +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum QuoteError { + #[error("quote field {0} missing or malformed")] + Field(&'static str), + #[error("quote signature does not verify: {0}")] + Signature(String), + #[error("quote signed by 0x{got} but this lane is pinned to 0x{want}")] + WrongSigner { got: String, want: String }, + #[error("quote beneficiary 0x{got} is not the pinned 0x{want}")] + WrongBeneficiary { got: String, want: String }, + #[error("advertised overlay does not derive from the signed identity")] + OverlayMismatch, + #[error("quote parameters are unusable: {0}")] + BadParams(String), + #[error("lane price {got} exceeds the client's ceiling {ceiling}")] + TooExpensive { got: u128, ceiling: u128 }, +} + +impl PaymentQuote { + /// Parse and verify a `/v1/status` `payment` block. + /// + /// `advertised_overlay` is the lane's own `overlay` field. Checking that + /// it derives from the *signed* identity is what makes the overlay + /// trustworthy at all: an overlay is + /// `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`, so a signature alone + /// yields the eth address while the nonce is neither transmitted nor + /// derivable — which is why "pin `(url, overlay)`" was never + /// implementable and the pin is on the address. + pub fn verify( + payment: &serde_json::Value, + advertised_overlay: Option<&[u8; 32]>, + network_id: u64, + pin: Option<&LanePin>, + price_ceiling_plur_per_kib: u128, + ) -> Result { + let sig_hex = payment + .get("sig") + .and_then(|s| s.as_str()) + .ok_or(QuoteError::Field("sig"))?; + let sig = hex::decode(sig_hex.trim_start_matches("0x")) + .map_err(|e| QuoteError::Signature(e.to_string()))?; + + // The relay signs the block *without* `sig`, and `serde_json`'s map + // is a `BTreeMap`, so re-serializing after removing that one field + // reproduces the signed bytes exactly. + let mut unsigned = payment.clone(); + unsigned + .as_object_mut() + .ok_or(QuoteError::Field("payment"))? + .remove("sig"); + let payload = unsigned.to_string(); + let node_eth_address = + crate::signer::recover_eth_address_from_eip191(payload.as_bytes(), &sig) + .map_err(|e| QuoteError::Signature(e.to_string()))?; + + let addr = |k: &'static str| -> Result<[u8; 20], QuoteError> { + let s = payment.get(k).and_then(|x| x.as_str()).ok_or(QuoteError::Field(k))?; + let raw = hex::decode(s.trim_start_matches("0x")).map_err(|_| QuoteError::Field(k))?; + <[u8; 20]>::try_from(raw.as_slice()).map_err(|_| QuoteError::Field(k)) + }; + let plur = |k: &'static str| -> Result { + payment + .get(k) + .and_then(|x| x.as_str()) + .and_then(|s| s.parse().ok()) + .ok_or(QuoteError::Field(k)) + }; + + let beneficiary = addr("beneficiary")?; + let claimed_node = addr("node_eth_address")?; + if claimed_node != node_eth_address { + return Err(QuoteError::WrongSigner { + got: hex::encode(node_eth_address), + want: hex::encode(claimed_node), + }); + } + let overlay_nonce = { + let s = payment + .get("overlay_nonce") + .and_then(|x| x.as_str()) + .ok_or(QuoteError::Field("overlay_nonce"))?; + let raw = + hex::decode(s.trim_start_matches("0x")).map_err(|_| QuoteError::Field("overlay_nonce"))?; + <[u8; 32]>::try_from(raw.as_slice()).map_err(|_| QuoteError::Field("overlay_nonce"))? + }; + + // Pinning is the actual root of trust (§2): the lane URL over HTTPS + // plus an identity the client already knew. + if let Some(pin) = pin { + if pin.node_eth_address != node_eth_address { + return Err(QuoteError::WrongSigner { + got: hex::encode(node_eth_address), + want: hex::encode(pin.node_eth_address), + }); + } + if pin.beneficiary != beneficiary { + return Err(QuoteError::WrongBeneficiary { + got: hex::encode(beneficiary), + want: hex::encode(pin.beneficiary), + }); + } + } + + if let Some(overlay) = advertised_overlay { + let derived = crate::signer::derive_overlay(&node_eth_address, network_id, &overlay_nonce); + if &derived != overlay { + return Err(QuoteError::OverlayMismatch); + } + } + + let params = Params { + price_plur_per_kib: plur("price_plur_per_kib")?, + min_cheque_plur: plur("min_cheque_plur")?, + settle_every_plur: plur("settle_every_plur")?, + max_outstanding_plur: plur("max_outstanding_plur")?, + credit_ratio: payment + .get("credit_ratio") + .and_then(|x| x.as_u64()) + .map(u128::from) + .ok_or(QuoteError::Field("credit_ratio"))?, + }; + // A lane whose parameters violate §10.1's invariant would brick this + // client, so refuse it here rather than discovering it at the first + // 402 with no cheque able to clear it. + params.validate().map_err(QuoteError::BadParams)?; + if params.price_plur_per_kib > price_ceiling_plur_per_kib { + return Err(QuoteError::TooExpensive { + got: params.price_plur_per_kib, + ceiling: price_ceiling_plur_per_kib, + }); + } + + Ok(Self { + beneficiary, + node_eth_address, + overlay_nonce, + origin: payment + .get("origin") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(), + chain_id: payment + .get("chain_id") + .and_then(|x| x.as_u64()) + .ok_or(QuoteError::Field("chain_id"))?, + params, + hard_enforcement: payment.get("enforcement").and_then(|x| x.as_str()) == Some("hard"), + }) + } +} + +/// A challenge the relay issued, ready to sign. +#[derive(Debug, Clone)] +pub struct OfferedChallenge { + pub nonce: [u8; 32], + pub account: [u8; 20], + pub batch: [u8; 32], + pub origin: String, + pub expiry_unix: u64, + pub cap_plur: u128, +} + +impl OfferedChallenge { + pub fn parse(v: &serde_json::Value) -> Result { + let fixed = |k: &str, n: usize| -> Result, String> { + let s = v + .get(k) + .and_then(|x| x.as_str()) + .ok_or_else(|| format!("challenge: missing {k}"))?; + let raw = hex::decode(s.trim_start_matches("0x")) + .map_err(|e| format!("challenge {k}: {e}"))?; + if raw.len() != n { + return Err(format!("challenge {k}: want {n} bytes, got {}", raw.len())); + } + Ok(raw) + }; + let mut nonce = [0u8; 32]; + nonce.copy_from_slice(&fixed("nonce", 32)?); + let mut account = [0u8; 20]; + account.copy_from_slice(&fixed("account", 20)?); + let mut batch = [0u8; 32]; + batch.copy_from_slice(&fixed("batch", 32)?); + Ok(Self { + nonce, + account, + batch, + origin: v + .get("origin") + .and_then(|x| x.as_str()) + .ok_or("challenge: missing origin")? + .to_string(), + expiry_unix: v + .get("expiry") + .and_then(|x| x.as_u64()) + .ok_or("challenge: missing expiry")?, + cap_plur: v + .get("max_outstanding_plur") + .and_then(|x| x.as_str()) + .and_then(|s| s.parse().ok()) + .ok_or("challenge: missing max_outstanding_plur")?, + }) + } + + /// Sign it and produce the header value. + /// + /// Signing binds `origin`, which is what makes this header useless at + /// any other relay even if it is observed in flight (§11.1). + pub fn sign( + &self, + signer: &crate::signer::SwarmSigner, + chain_id: u64, + ) -> Result { + let sol = crate::signer::PushChallenge { + nonce: alloy_primitives::B256::from(self.nonce), + origin: self.origin.clone(), + account: alloy_primitives::Address::from(self.account), + batchId: alloy_primitives::B256::from(self.batch), + expiry: alloy_primitives::U256::from(self.expiry_unix), + }; + let sig = signer + .sign_push_challenge(&sol, chain_id) + .map_err(|e| e.to_string())?; + let issued = crate::metered::IssuedChallenge { + fields: crate::challenge::ChallengeFields { + account: self.account, + batch: self.batch, + origin: self.origin.clone(), + expiry_unix: self.expiry_unix, + cap_plur: self.cap_plur, + }, + nonce: self.nonce, + }; + Ok(crate::metered::encode_challenge_header(&issued, &sig)) + } + + /// Re-fetch before this, rather than racing the expiry with a POST in + /// flight. A challenge is cheap; a mid-upload 401 is not. + pub fn stale_after(&self) -> u64 { + self.expiry_unix.saturating_sub(30) + } +} + +/// Per-lane running total, tracked by the client from bytes it sent. +#[derive(Debug, Clone)] +pub struct LaneAccount { + pub params: Params, + pub beneficiary: [u8; 20], + /// Billed and not yet covered by a cheque. + owed_plur: u128, + /// Total already promised to this beneficiary. Cheques are cumulative, + /// so this only grows. + cumulative_plur: u128, +} + +impl LaneAccount { + pub fn new(params: Params, beneficiary: [u8; 20]) -> Self { + Self { + params, + beneficiary, + owed_plur: 0, + cumulative_plur: 0, + } + } + + /// Restore the cumulative from the on-disk store, so a second CLI run + /// does not issue a cheque the relay rejects as non-increasing. + pub fn with_cumulative(mut self, cumulative_plur: u128) -> Self { + self.cumulative_plur = cumulative_plur; + self + } + + pub fn owed(&self) -> u128 { + self.owed_plur + } + + pub fn cumulative(&self) -> u128 { + self.cumulative_plur + } + + /// Record a POST body we sent. Same arithmetic as the relay's (§8), so + /// the two sides agree without exchanging anything. + pub fn record_sent(&mut self, body_bytes: u64) { + self.owed_plur = self + .owed_plur + .saturating_add(self.params.price_bytes(body_bytes)); + } + + /// A dedup hit costs nothing, so give it back when the ack says so + /// (§8.2). The relay's claim only ever lowers the bill, so believing it + /// is safe. + pub fn refund_dedup(&mut self, body_bytes: u64) { + self.owed_plur = self + .owed_plur + .saturating_sub(self.params.price_bytes(body_bytes)); + } + + pub fn should_settle(&self) -> bool { + self.owed_plur >= self.params.settle_every_plur + } + + /// The cumulative for the next cheque, or `None` when what is owed is + /// still under the lane's dust floor and would be refused. + pub fn next_cumulative(&self) -> Option { + if self.owed_plur < self.params.min_cheque_plur { + return None; + } + Some(self.cumulative_plur.saturating_add(self.owed_plur)) + } + + /// Call once a cheque for `cumulative` has been accepted. + pub fn settled(&mut self, cumulative: u128) { + let credited = cumulative.saturating_sub(self.cumulative_plur); + self.cumulative_plur = cumulative; + self.owed_plur = self.owed_plur.saturating_sub(credited); + } + + /// Largest body this lane will admit right now, in bytes. + /// + /// The client sizes its POST to fit rather than discovering the ceiling + /// as a 402 — which matters most for exactly the small batches §10.3 + /// exists to keep, whose whole credit line is under one full POST. + pub fn max_body_bytes(&self, cap_plur: u128) -> u64 { + let headroom = cap_plur.saturating_sub(self.owed_plur); + let kib = headroom / self.params.price_plur_per_kib.max(1); + (kib.saturating_mul(1024)).min(u64::MAX as u128) as u64 + } +} + +/// Aggregate exposure across every beneficiary drawn on one chequebook. +/// +/// Cumulative payouts are per `(chequebook, beneficiary)`, so N lanes are N +/// independent claims on **one** balance. Without this a cheque to the +/// second lane silently exceeds it and bounces — and §11.3's Sybil case is +/// exactly one operator presenting several beneficiaries. +#[derive(Debug, Default, Clone)] +pub struct TotalIssued { + per_beneficiary: std::collections::BTreeMap<[u8; 20], u128>, +} + +impl TotalIssued { + pub fn total(&self) -> u128 { + self.per_beneficiary.values().copied().sum() + } + + pub fn issued_to(&self, beneficiary: &[u8; 20]) -> u128 { + self.per_beneficiary.get(beneficiary).copied().unwrap_or(0) + } + + /// Would raising this beneficiary's cumulative to `cumulative` push the + /// total past the chequebook's balance? + pub fn would_exceed(&self, beneficiary: &[u8; 20], cumulative: u128, balance: u128) -> bool { + let others = self.total() - self.issued_to(beneficiary); + others.saturating_add(cumulative) > balance + } + + pub fn record(&mut self, beneficiary: [u8; 20], cumulative: u128) { + let e = self.per_beneficiary.entry(beneficiary).or_insert(0); + // Cumulatives only grow; a lower value is a stale report, not a + // refund. + if cumulative > *e { + *e = cumulative; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::signer::SwarmSigner; + + const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + const NONCE: [u8; 32] = [0u8; 32]; + + fn node() -> SwarmSigner { + SwarmSigner::from_hex_with_nonce(KEY, &format!("0x{}", hex::encode(NONCE)), 1).expect("key") + } + + /// Build a quote exactly the way the relay does, so the test exercises + /// the real signed bytes rather than a hand-rolled approximation. + fn quote_json(beneficiary: [u8; 20]) -> serde_json::Value { + let n = node(); + let p = Params::default(); + let mut body = serde_json::json!({ + "mode": "metered", + "enforcement": "soft", + "beneficiary": format!("0x{}", hex::encode(beneficiary)), + "node_eth_address": format!("0x{}", hex::encode(n.eth_address())), + "overlay_nonce": format!("0x{}", hex::encode(NONCE)), + "origin": "relay-a.example", + "chain_id": 100, + "factory": format!("0x{}", hex::encode([0u8; 20])), + "price_plur_per_kib": p.price_plur_per_kib.to_string(), + "min_cheque_plur": p.min_cheque_plur.to_string(), + "settle_every_plur": p.settle_every_plur.to_string(), + "max_outstanding_plur": p.max_outstanding_plur.to_string(), + "credit_ratio": p.credit_ratio as u64, + "challenge_ttl_secs": 300, + }); + let sig = n.sign_eip191(body.to_string().as_bytes()).expect("sign"); + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + body + } + + fn ceiling() -> u128 { + Params::default().price_plur_per_kib * 4 + } + + #[test] + fn a_signed_quote_verifies_and_yields_the_signing_identity() { + let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()) + .expect("must verify"); + assert_eq!(q.node_eth_address, *node().eth_address()); + assert_eq!(q.beneficiary, [3u8; 20]); + assert_eq!(q.params, Params::default()); + assert!(!q.hard_enforcement); + } + + /// The reason the pin is on the address and the nonce is published: + /// the client can now check the lane's overlay claim instead of taking + /// it on faith. + #[test] + fn the_advertised_overlay_must_derive_from_the_signed_identity() { + let good = crate::signer::derive_overlay(node().eth_address(), 1, &NONCE); + PaymentQuote::verify("e_json([3u8; 20]), Some(&good), 1, None, ceiling()) + .expect("a derivable overlay verifies"); + assert_eq!( + PaymentQuote::verify("e_json([3u8; 20]), Some(&[9u8; 32]), 1, None, ceiling()), + Err(QuoteError::OverlayMismatch) + ); + } + + /// Tampering with any signed field must break the signature — otherwise + /// a lane could serve one price and bill another. + #[test] + fn tampering_with_the_quote_breaks_it() { + for field in ["price_plur_per_kib", "beneficiary", "origin", "chain_id"] { + let mut q = quote_json([3u8; 20]); + q[field] = match field { + "price_plur_per_kib" => serde_json::json!("1"), + "beneficiary" => serde_json::json!(format!("0x{}", hex::encode([0xAAu8; 20]))), + "origin" => serde_json::json!("evil.example"), + _ => serde_json::json!(1u64), + }; + assert!( + PaymentQuote::verify(&q, None, 1, None, ceiling()).is_err(), + "tampering with {field} must be caught" + ); + } + } + + /// The root of trust: an identity the client already knew, not one the + /// lane asserts about itself. + #[test] + fn a_quote_from_an_unpinned_identity_is_refused() { + let pin = LanePin { + node_eth_address: [0xEE; 20], + beneficiary: [3u8; 20], + }; + let e = PaymentQuote::verify("e_json([3u8; 20]), None, 1, Some(&pin), ceiling()) + .expect_err("must refuse"); + assert!(matches!(e, QuoteError::WrongSigner { .. }), "got {e:?}"); + } + + /// §11.3: a correctly-signed relay advertising someone else's + /// beneficiary must not be paid. + #[test] + fn a_quote_with_an_unpinned_beneficiary_is_refused() { + let pin = LanePin { + node_eth_address: *node().eth_address(), + beneficiary: [3u8; 20], + }; + let e = PaymentQuote::verify("e_json([0xBB; 20]), None, 1, Some(&pin), ceiling()) + .expect_err("must refuse"); + assert!(matches!(e, QuoteError::WrongBeneficiary { .. }), "got {e:?}"); + } + + #[test] + fn an_overpriced_or_bricking_lane_is_refused() { + let e = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, 1) + .expect_err("price ceiling"); + assert!(matches!(e, QuoteError::TooExpensive { .. }), "got {e:?}"); + + // A lane whose dust floor exceeds its settlement window would brick + // this client with no cheque able to clear the 402. + let n = node(); + let mut body = quote_json([3u8; 20]); + body["min_cheque_plur"] = + serde_json::json!((Params::default().settle_every_plur * 87).to_string()); + let mut unsigned = body.clone(); + unsigned.as_object_mut().unwrap().remove("sig"); + let sig = n.sign_eip191(unsigned.to_string().as_bytes()).expect("sign"); + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + let e = PaymentQuote::verify(&body, None, 1, None, ceiling()).expect_err("bricking lane"); + assert!(matches!(e, QuoteError::BadParams(_)), "got {e:?}"); + } + + #[test] + fn a_challenge_round_trips_into_a_header_the_relay_accepts() { + use crate::ledger::Ledger; + use crate::metered::{MeterConfig, Metered}; + let acct_signer = node(); + let account = *acct_signer.eth_address(); + let m = Metered::new( + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: alloy_primitives::Address::ZERO, + params: Params::default(), + hard_mode: false, + }, + Ledger::ephemeral(), + ); + let issued = m + .issue(account, [7u8; 32], 6_200_000_000_000_000_000, "relay-a.example", 1000) + .expect("issue"); + // Straight through the wire form the relay actually serves. + let offered = OfferedChallenge::parse(&issued.to_json()).expect("parse"); + let header = offered.sign(&acct_signer, 100).expect("sign"); + let v = m.verify_header(&header, 1000).expect("relay must accept"); + assert_eq!(v.account, account); + assert_eq!(v.batch, [7u8; 32]); + } + + #[test] + fn owed_tracks_bytes_sent_and_a_cheque_clears_it() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(32 * 1024 * 1024); + assert_eq!(a.owed(), p.price_bytes(32 * 1024 * 1024)); + assert!(a.should_settle(), "32 MiB crosses the settlement window"); + let c = a.next_cumulative().expect("above the dust floor"); + a.settled(c); + assert_eq!(a.owed(), 0); + assert_eq!(a.cumulative(), c); + } + + /// Cheques are cumulative, so a second upload adds to the same running + /// total rather than starting over — which is what a relay's + /// monotonicity check requires. + #[test] + fn a_second_upload_grows_the_same_cumulative() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(40 * 1024 * 1024); + let first = a.next_cumulative().expect("cheque"); + a.settled(first); + a.record_sent(40 * 1024 * 1024); + let second = a.next_cumulative().expect("cheque"); + assert!(second > first, "cumulative must increase: {second} > {first}"); + assert_eq!(second - first, p.price_bytes(40 * 1024 * 1024)); + } + + #[test] + fn a_cumulative_restored_from_disk_keeps_increasing() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]).with_cumulative(5_000_000_000_000_000); + a.record_sent(40 * 1024 * 1024); + let c = a.next_cumulative().expect("cheque"); + assert!( + c > 5_000_000_000_000_000, + "a fresh run must not re-issue below what a previous run already sent" + ); + } + + #[test] + fn dust_owings_do_not_produce_a_cheque_the_relay_would_refuse() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(1024); + assert!(!a.should_settle()); + assert_eq!(a.next_cumulative(), None, "below the lane's dust floor"); + } + + #[test] + fn dedup_hits_are_refunded() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(100 * 4251); + let before = a.owed(); + a.refund_dedup(10 * 4251); + assert!(a.owed() < before); + } + + /// §7.2: size the POST to the credit line instead of discovering the + /// ceiling as a 402. A dust batch gets a small but usable body. + #[test] + fn post_size_is_bounded_by_the_credit_line() { + let p = Params::default(); + let a = LaneAccount::new(p, [3u8; 20]); + let dust_cap = p.credit_line(100_000_000_000_000); + let max = a.max_body_bytes(dust_cap); + assert_eq!(max, 208 * 1024, "~208 KiB, matching the credit line"); + assert!(max >= 4251, "and still enough for at least one frame"); + // A rich batch is bounded by the global ceiling instead. + let rich = a.max_body_bytes(p.max_outstanding_plur); + assert!(rich > 512 * 4251, "a full POST fits comfortably"); + } + + #[test] + fn headroom_shrinks_as_debt_accrues() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let cap = p.max_outstanding_plur; + let before = a.max_body_bytes(cap); + a.record_sent(64 * 1024 * 1024); + assert!(a.max_body_bytes(cap) < before, "unpaid debt eats the line"); + } + + /// N lanes are N claims on one balance. The client must see the sum, or + /// the second cheque bounces. + #[test] + fn total_issued_aggregates_across_beneficiaries() { + let mut t = TotalIssued::default(); + t.record([1u8; 20], 600); + t.record([2u8; 20], 300); + assert_eq!(t.total(), 900); + assert!(!t.would_exceed(&[1u8; 20], 700, 1000), "700 + 300 fits"); + assert!(t.would_exceed(&[1u8; 20], 800, 1000), "800 + 300 does not"); + assert!( + !t.would_exceed(&[3u8; 20], 100, 1000), + "a new beneficiary is checked against the others' total" + ); + } + + #[test] + fn a_stale_cumulative_report_never_lowers_the_total() { + let mut t = TotalIssued::default(); + t.record([1u8; 20], 600); + t.record([1u8; 20], 100); + assert_eq!(t.total(), 600, "cumulatives only grow"); + } +} diff --git a/src/protocols/pushsync.rs b/src/protocols/pushsync.rs index 9936b3e..05aa9b3 100644 --- a/src/protocols/pushsync.rs +++ b/src/protocols/pushsync.rs @@ -121,6 +121,31 @@ where if !receipt.err.is_empty() { return Err(PushsyncError::Peer(receipt.err)); } + // The receipt must be *for the chunk we pushed*. Two things go wrong + // without this check, and both are reachable by any peer in the pool + // (membership comes from hive gossip and the seed list, so it is not a + // trusted set): + // + // 1. `address` is copied straight off the wire, and callers build a + // `[u8; 32]` from it with `copy_from_slice` (`src/client.rs:5380`, + // `:5398`), which panics on any other length. A 0- or 33-byte + // address is a remote panic that unwinds the push task. + // 2. Nothing else compares it to what we sent. A peer could accept the + // Delivery, store nothing, and sign a receipt for some *other* + // address deep inside its own neighborhood: `is_shallow` and the + // `po` computation both read `r.address`, so the forged receipt + // looks like a perfect deep delivery and the chunk is acked `ok` + // having never been stored. + // + // Checking it here means every `PushsyncReceipt` in the codebase + // carries exactly the 32-byte address that was pushed. + if receipt.address != address[..] { + return Err(PushsyncError::Peer(format!( + "receipt address mismatch: pushed {}, receipt {}", + hex::encode(address), + hex::encode(&receipt.address), + ))); + } Ok(PushsyncReceipt { address: receipt.address, signature: receipt.signature, @@ -128,3 +153,110 @@ where storage_radius: receipt.storage_radius, }) } + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message; + use std::pin::Pin; + use std::task::{Context, Poll}; + + /// Stream whose read side replays a canned peer response and whose + /// write side is discarded — enough to drive `push` to the receipt. + struct Canned { + read: Vec, + pos: usize, + } + + impl futures::AsyncRead for Canned { + fn poll_read( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let n = (self.read.len() - self.pos).min(buf.len()); + buf[..n].copy_from_slice(&self.read[self.pos..self.pos + n]); + self.pos += n; + Poll::Ready(Ok(n)) + } + } + + impl futures::AsyncWrite for Canned { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Poll::Ready(Ok(buf.len())) + } + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + /// Response headers followed by `receipt`, both length-delimited. + fn peer_saying(receipt: pb::Receipt) -> Canned { + let mut read = Vec::new(); + hdr::Headers { headers: vec![] } + .encode_length_delimited(&mut read) + .expect("encode headers"); + receipt + .encode_length_delimited(&mut read) + .expect("encode receipt"); + Canned { read, pos: 0 } + } + + fn receipt_for(address: Vec) -> pb::Receipt { + pb::Receipt { + address, + signature: vec![7u8; 65], + nonce: vec![9u8; 32], + err: String::new(), + storage_radius: 8, + } + } + + fn push_against(receipt: pb::Receipt) -> Result { + let pushed = [1u8; 32]; + let mut stream = peer_saying(receipt); + tokio_test::block_on(push(&mut stream, &pushed, &[0u8; 16], &[0u8; 113])) + } + + #[test] + fn receipt_for_the_pushed_address_is_accepted() { + let r = push_against(receipt_for(vec![1u8; 32])).expect("should accept"); + assert_eq!(r.address, vec![1u8; 32]); + assert_eq!(r.storage_radius, 8); + } + + /// A peer can otherwise sign a receipt for a *different* address deep in + /// its own neighborhood, having stored nothing: `is_shallow` and the `po` + /// computation both read the receipt's address, so it would look like a + /// perfect delivery. + #[test] + fn receipt_for_a_different_address_is_rejected() { + let err = push_against(receipt_for(vec![2u8; 32])).expect_err("should reject"); + assert!( + matches!(&err, PushsyncError::Peer(m) if m.contains("address mismatch")), + "unexpected error: {err}" + ); + } + + /// Callers build `[u8; 32]` from this with `copy_from_slice`, which + /// panics on any other length — so a malformed address must be rejected + /// here rather than unwinding the push task. + #[test] + fn missized_addresses_are_rejected_not_panicked_on() { + for bad in [vec![], vec![1u8; 31], vec![1u8; 33], vec![1u8; 64]] { + let n = bad.len(); + let err = push_against(receipt_for(bad)).expect_err("missized address must be rejected"); + assert!( + matches!(&err, PushsyncError::Peer(m) if m.contains("address mismatch")), + "unexpected error for {n}-byte address: {err}" + ); + } + } +} diff --git a/src/protocols/swap.rs b/src/protocols/swap.rs index 905df88..ec56b92 100644 --- a/src/protocols/swap.rs +++ b/src/protocols/swap.rs @@ -179,3 +179,236 @@ where write_message(stream, &msg).await?; Ok(()) } + +// ────────────────────────────────────────────────────────────────────── +// Inbound cheques (metered relay — docs/pusher-incentives.md Stage 1) +// ────────────────────────────────────────────────────────────────────── + +/// A cheque as it arrives at `POST /v1/pay`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SignedCheque { + pub chequebook: [u8; 20], + pub beneficiary: [u8; 20], + pub cumulative_payout: U256, + pub signature: [u8; 65], +} + +/// Largest body we will even look at. A cheque is ~250 bytes; anything +/// bigger is not one, and scanning it is work an unauthenticated caller +/// should not be able to buy (§11.6). +pub const MAX_CHEQUE_JSON: usize = 4096; + +/// Decode the JSON `encode_signed_cheque_json` produces — and that bee's +/// `json.Marshal(SignedCheque)` produces, which is the same bytes. +/// +/// `CumulativePayout` is extracted from the raw input rather than through +/// `serde_json`, for the same reason the encoder is hand-written: Go emits +/// `*big.Int` as a **bare JSON number**, and `serde_json` without +/// `arbitrary_precision` silently widens anything past `u64` into `f64`. +/// A cheque for 10^20 PLUR would round to a different number and still +/// parse, so the relay would credit an amount the signature does not cover +/// and the recovered issuer would be garbage. Losing precision here is not +/// a rounding bug, it is a money bug. +pub fn decode_signed_cheque_json(bytes: &[u8]) -> Result { + if bytes.len() > MAX_CHEQUE_JSON { + return Err(SwapError::Json(format!( + "cheque body too large: {} bytes (max {MAX_CHEQUE_JSON})", + bytes.len() + ))); + } + let v: serde_json::Value = + serde_json::from_slice(bytes).map_err(|e| SwapError::Json(e.to_string()))?; + + let chequebook = json_address(&v, "Chequebook")?; + let beneficiary = json_address(&v, "Beneficiary")?; + let cumulative_payout = extract_cumulative(bytes)?; + + let sig_b64 = v + .get("Signature") + .and_then(|s| s.as_str()) + .ok_or_else(|| SwapError::Json("missing Signature".into()))?; + let sig_bytes = { + use base64::Engine; + base64::engine::general_purpose::STANDARD + .decode(sig_b64) + .map_err(|e| SwapError::Json(format!("Signature base64: {e}")))? + }; + // Canonical form is checked *here*, at the only place every cheque + // passes through, rather than left to each caller. A high-`s` or + // `v ∈ {0,1}` cheque recovers fine off-chain and reverts at cashout + // (§11.6), so accepting one means giving away service for a signature + // that can never be redeemed. + crate::signer::check_canonical_signature(&sig_bytes) + .map_err(|e| SwapError::Json(e.to_string()))?; + let mut signature = [0u8; 65]; + signature.copy_from_slice(&sig_bytes); + + Ok(SignedCheque { + chequebook, + beneficiary, + cumulative_payout, + signature, + }) +} + +fn json_address(v: &serde_json::Value, field: &str) -> Result<[u8; 20], SwapError> { + let s = v + .get(field) + .and_then(|x| x.as_str()) + .ok_or_else(|| SwapError::Json(format!("missing {field}")))?; + let hex_str = s.trim_start_matches("0x").trim_start_matches("0X"); + let raw = + hex::decode(hex_str).map_err(|e| SwapError::Json(format!("{field} not hex: {e}")))?; + if raw.len() != 20 { + return Err(SwapError::Json(format!( + "{field} must be 20 bytes, got {}", + raw.len() + ))); + } + let mut out = [0u8; 20]; + out.copy_from_slice(&raw); + Ok(out) +} + +/// Pull `CumulativePayout`'s digits straight out of the input. +/// +/// Rejects a repeated key rather than picking one: duplicate keys are legal +/// JSON and `serde_json` keeps the last, so if the scan and the parser +/// disagreed about which one counts, an attacker could show the relay one +/// amount and the signature check another. +fn extract_cumulative(bytes: &[u8]) -> Result { + const KEY: &[u8] = b"\"CumulativePayout\""; + let mut hits = bytes + .windows(KEY.len()) + .enumerate() + .filter(|(_, w)| *w == KEY) + .map(|(i, _)| i); + let at = hits + .next() + .ok_or_else(|| SwapError::Json("missing CumulativePayout".into()))?; + if hits.next().is_some() { + return Err(SwapError::Json("duplicate CumulativePayout key".into())); + } + let mut i = at + KEY.len(); + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if i >= bytes.len() || bytes[i] != b':' { + return Err(SwapError::Json("CumulativePayout is not a field".into())); + } + i += 1; + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + let start = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if start == i { + return Err(SwapError::Json( + "CumulativePayout must be a bare JSON integer, as Go emits it".into(), + )); + } + let digits = std::str::from_utf8(&bytes[start..i]) + .map_err(|e| SwapError::Json(format!("CumulativePayout utf8: {e}")))?; + U256::from_str_radix(digits, 10) + .map_err(|e| SwapError::Json(format!("CumulativePayout overflows u256: {e}"))) +} + +#[cfg(test)] +mod cheque_decode_tests { + use super::*; + + fn sample(cumulative: U256) -> Vec { + encode_signed_cheque_json(&[0x11; 20], &[0x22; 20], cumulative, &canonical_sig()) + } + + /// `v = 27` and a low `s`, so it survives the canonical check. + fn canonical_sig() -> [u8; 65] { + let mut s = [0x01u8; 65]; + s[64] = 27; + s + } + + #[test] + fn round_trips_our_own_encoder() { + let amount = U256::from(1_234_567u64); + let got = decode_signed_cheque_json(&sample(amount)).expect("decode"); + assert_eq!(got.chequebook, [0x11; 20]); + assert_eq!(got.beneficiary, [0x22; 20]); + assert_eq!(got.cumulative_payout, amount); + assert_eq!(got.signature, canonical_sig()); + } + + /// The reason the decoder is hand-written. `serde_json` without + /// `arbitrary_precision` widens this to `f64` and hands back a + /// *different number* — which would have the relay credit an amount the + /// signature never covered. + #[test] + fn a_payout_past_u64_survives_exactly() { + // 2^80 + 1: needs u256, and is not representable in f64. + let amount = U256::from(1u64) << 80 | U256::from(1u64); + let body = sample(amount); + let got = decode_signed_cheque_json(&body).expect("decode"); + assert_eq!(got.cumulative_payout, amount); + + let lossy = serde_json::from_slice::(&body) + .expect("parses") + .get("CumulativePayout") + .and_then(|n| n.as_u64()); + assert!( + lossy.is_none(), + "if serde ever parses this exactly, the hand-rolled scan can go" + ); + } + + #[test] + fn a_payout_at_the_u256_ceiling_decodes() { + let amount = U256::MAX; + let got = decode_signed_cheque_json(&sample(amount)).expect("decode"); + assert_eq!(got.cumulative_payout, amount); + } + + #[test] + fn a_payout_past_the_u256_ceiling_is_rejected() { + let mut body = String::from_utf8(sample(U256::from(1u64))).expect("utf8"); + body = body.replace( + "\"CumulativePayout\":1,", + &format!("\"CumulativePayout\":{},", "9".repeat(78)), + ); + decode_signed_cheque_json(body.as_bytes()).expect_err("must not wrap around"); + } + + /// Duplicate keys are legal JSON and serde keeps the last. If the scan + /// picked the first, the relay would credit one amount while verifying + /// a signature over another. + #[test] + fn a_duplicated_payout_key_is_rejected_not_guessed() { + let body = br#"{"Chequebook":"0x1111111111111111111111111111111111111111","Beneficiary":"0x2222222222222222222222222222222222222222","CumulativePayout":1,"CumulativePayout":999999,"Signature":"AQ=="}"#; + let e = decode_signed_cheque_json(body).expect_err("ambiguous"); + assert!(format!("{e}").contains("duplicate"), "got: {e}"); + } + + #[test] + fn non_canonical_signatures_are_rejected_at_the_boundary() { + let mut bad = canonical_sig(); + bad[64] = 1; + let body = encode_signed_cheque_json(&[0x11; 20], &[0x22; 20], U256::from(5u64), &bad); + let e = decode_signed_cheque_json(&body).expect_err("uncashable cheque"); + assert!(format!("{e}").contains("non-canonical"), "got: {e}"); + } + + #[test] + fn malformed_bodies_are_rejected_not_panicked_on() { + for body in [ + &b"{}"[..], + b"not json", + b"[]", + br#"{"Chequebook":"0x11","Beneficiary":"0x22","CumulativePayout":1,"Signature":"AQ=="}"#, + br#"{"Chequebook":"0x1111111111111111111111111111111111111111","Beneficiary":"0x2222222222222222222222222222222222222222","CumulativePayout":"1","Signature":"AQ=="}"#, + ] { + decode_signed_cheque_json(body).expect_err("must reject"); + } + decode_signed_cheque_json(&vec![b'x'; MAX_CHEQUE_JSON + 1]).expect_err("oversized"); + } +} diff --git a/src/pusher.rs b/src/pusher.rs index c0a78e9..c074777 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -92,6 +92,34 @@ const PUSH_POOL_TARGET_DEFAULT: usize = 128; const PUSH_POOL_TARGET_MAX: usize = 512; /// Per-chunk retry budget on the push path. const PUSH_MAX_RETRIES: usize = 20; +/// Default cap on concurrent inbound connections +/// (`HOVERFLY_PUSH_MAX_CONNS`). A `/v1/push` in flight holds its collected +/// body plus decoded frames — a couple of MiB at `PUSH_BATCH_MAX` — so an +/// uncapped accept loop is an uncapped memory commitment. +const PUSH_MAX_CONNS_DEFAULT: usize = 256; +/// Ceiling on the accept-loop retry backoff. +const ACCEPT_BACKOFF_MAX_MS: u64 = 1000; +/// How long a connection may take to send its request headers. Only the +/// header read — response streaming is unbounded by design. +const HEADER_READ_TIMEOUT_SECS: u64 = 30; +/// How long `/v1/push` may take to receive its (already size-capped) body. +/// Generous for ~2 MiB on a slow mobile link, but finite: the read happens +/// before any work is spawned, so an unbounded one is free to hold. +const PUSH_BODY_READ_TIMEOUT_SECS: u64 = 120; +/// Bound on the `batch_id → owner` cache. Batch ids are enumerable +/// on-chain (`BatchCreated`), so an unbounded map is a remote memory sink. +const OWNER_CACHE_CAP: usize = 4096; +/// How long a successful batch resolution stays cached. Bounded so a batch +/// that expires while cached stops being served indefinitely. +const OWNER_OK_TTL_SECS: u64 = 1800; +/// How long a *definitive* rejection (absent on-chain, or expired) stays +/// cached. This is what stops a flood of bogus batch ids from turning one +/// unauthenticated POST into one RPC round trip per frame. +const OWNER_BAD_TTL_SECS: u64 = 300; +/// Distinct batch resolutions that may reach the chain in a single POST. +/// Honest clients push one or a few batches per request; this bounds the +/// RPC amplification of a request that names 512 different ones. +const PUSH_MAX_BATCH_LOOKUPS: usize = 8; /// Recently-acked address cache: enough to cover several in-flight /// batches across every lane a client might hedge between. const RECENT_ACK_CAP: usize = 8192; @@ -125,6 +153,30 @@ pub struct PusherOpts { /// HOVERFLY_PUSHER_IDENTITY. `None` = reuse the stamp key. pub node_identity: Option, pub transport: TransportConfig, + /// Metered mode (`docs/pusher-incentives.md` Stage 1). `None` = `open`, + /// today's unmetered behaviour, which the production lanes keep running. + pub meter: Option, +} + +/// Everything `--meter` needs. Validated at startup: a relay that cannot +/// state its own origin, or whose parameters violate §10.1's invariant, +/// refuses to boot rather than serving a broken meter. +#[derive(Debug, Clone)] +pub struct MeterOpts { + /// `--origin`, one or more. **Required**, and never derived from a + /// request header (§11.1). + pub origins: Vec, + /// EOA that must appear as `Cheque.beneficiary`. The relay holds the + /// address only — never the key (§6). + pub beneficiary: [u8; 20], + /// Settlement chain. Pins the EIP-712 domain and the factory. + pub chain_id: u64, + pub params: crate::meter::Params, + /// Where the ledger and relay secret live. Required: metered mode + /// without durable state is an unbounded free-service loop (§11.4). + pub state_dir: PathBuf, + /// Stage 2. False = soft mode: meter and report, never refuse. + pub hard_mode: bool, } struct State { @@ -143,9 +195,82 @@ struct State { /// /v1/push requests (filled lazily on first push). `None` transport /// means the node key was unresolvable; /v1/push then 503s. push: Option, - /// `batch_id(hex) → on-chain owner`, so repeated pushes for one batch - /// cost a single RPC. - owner_cache: std::sync::Mutex>, + /// `batch_id(hex) → resolution`, so repeated pushes for one batch cost a + /// single RPC. Caches rejections too — see [`OwnerCache`]. + owner_cache: std::sync::Mutex, + /// Stage 0 shadow metering (`src/meter.rs`, incentives §14): counts what + /// a metered relay *would* have billed, bills nothing, and changes + /// nothing on the wire. Merged once per request, never per frame. + meter: std::sync::Mutex, + /// Stage 1 metered mode. `None` in `open` mode, which is every + /// production lane today. + metered: Option, +} + +/// Outcome of resolving a batch id on-chain. +#[derive(Clone)] +enum OwnerLookup { + /// Batch exists, is funded, and is owned by this address. Carries the + /// batch's total remaining value in PLUR (`remainingBalance × 2^depth`), + /// which costs nothing extra — `resolve_owner` already reads both halves + /// to check for expiry — and is what Stage 0 shadow metering prices a + /// credit line from (`src/meter.rs`, incentives §10.3). + Owner([u8; 20], u128), + /// Batch is *definitively* unusable: absent on-chain, or out of + /// balance. Carries the reason so the ack is unchanged. + Rejected(String), +} + +/// Bounded, TTL'd `batch_id → outcome` cache. +/// +/// Caching only *successes* was a live amplification bug. `stamp::validate` +/// checks that a signature recovers to a non-zero address and nothing else, +/// so any random key over any attacker-chosen batch id reaches the chain +/// read. With no negative entry, every frame naming a bogus batch re-issued +/// the RPC, so one unauthenticated POST of `PUSH_BATCH_MAX` frames became +/// that many serial `eth_call`s. Rejections are cached for a shorter TTL +/// than successes, so a batch that is topped up recovers quickly. +/// +/// Only definitive on-chain answers are cached. A transport error is *not* +/// cached — an RPC blip must not blacklist a live batch. +struct OwnerCache { + map: HashMap, + order: std::collections::VecDeque, + cap: usize, +} + +impl OwnerCache { + fn new(cap: usize) -> Self { + Self { + map: HashMap::new(), + order: std::collections::VecDeque::new(), + cap, + } + } + + fn get(&self, batch_id_hex: &str) -> Option { + let (entry, when) = self.map.get(batch_id_hex)?; + let ttl = match entry { + OwnerLookup::Owner(..) => OWNER_OK_TTL_SECS, + OwnerLookup::Rejected(_) => OWNER_BAD_TTL_SECS, + }; + (when.elapsed() < std::time::Duration::from_secs(ttl)).then(|| entry.clone()) + } + + fn insert(&mut self, batch_id_hex: &str, entry: OwnerLookup) { + if self + .map + .insert(batch_id_hex.to_string(), (entry, Instant::now())) + .is_none() + { + self.order.push_back(batch_id_hex.to_string()); + } + while self.order.len() > self.cap { + if let Some(old) = self.order.pop_front() { + self.map.remove(&old); + } + } + } } struct PushState { @@ -160,6 +285,10 @@ struct PushState { /// maintenance loop so the (sync) status handler never has to touch /// the async pool mutex. pool_live: AtomicUsize, + /// The node-identity key. Kept alongside the transport (which owns its + /// own clone) so the metered quote can be signed without rebuilding it + /// — it signs prices, never payments, and is not spendable (§6). + signer: crate::signer::SwarmSigner, /// This node's Kademlia overlay (node eth address + nonce). Published /// in `/v1/status` so a multi-lane client can route each chunk to the /// relay whose overlay is nearest the chunk's destination neighborhood @@ -171,21 +300,31 @@ struct PushState { /// The client turns `budget_remaining_gb` into a scheduling weight. budget_gb: Option, bytes_pushed: AtomicU64, - /// Addresses acked recently, so a duplicate frame (client hedging a - /// straggler across two lanes) is answered from cache instead of - /// paying a second real push. docs/pusher-design.md §7 "ChunkCache". + /// (Address, batch) pairs acked recently, so a duplicate frame (client + /// hedging a straggler across two lanes) is answered from cache + /// instead of paying a second real push. Keyed by batch so a dedup + /// hit can't substitute another uploader's stamp. + /// docs/pusher-design.md §7 "ChunkCache". recent: std::sync::Mutex, } -/// Bounded, TTL'd set of recently-acked chunk addresses. +/// Bounded, TTL'd set of recently-acked (chunk address, batch). +/// +/// Keyed on the batch too: a chunk address is content-derived, so it is +/// not unique across batch owners. Under a bare-address key a dedup hit +/// acks a frame `ok` while silently discarding the submitted stamp — one +/// uploader's dust batch could shadow another uploader's long-lived +/// batch for the TTL, and the victim's chunk is then garbage-collected +/// when the shadowing batch expires (docs/pusher-incentives.md §15). It +/// also fires spuriously between honest users uploading identical bytes. /// /// Insert order doubles as eviction order (a chunk's ack time only moves -/// forward), so a `VecDeque` of `(addr, when)` plus a `HashMap` index is -/// enough — no LRU bookkeeping, since re-acking an address doesn't need -/// to extend its life. +/// forward), so a `VecDeque` of `((addr, batch), when)` plus a `HashMap` +/// index is enough — no LRU bookkeeping, since re-acking an entry +/// doesn't need to extend its life. struct RecentAcks { - seen: HashMap<[u8; 32], Instant>, - order: std::collections::VecDeque<[u8; 32]>, + seen: HashMap<([u8; 32], [u8; 32]), Instant>, + order: std::collections::VecDeque<([u8; 32], [u8; 32])>, cap: usize, ttl: std::time::Duration, } @@ -200,13 +339,16 @@ impl RecentAcks { } } - fn contains(&self, addr: &[u8; 32]) -> bool { - self.seen.get(addr).is_some_and(|t| t.elapsed() < self.ttl) + fn contains(&self, addr: &[u8; 32], batch_id: [u8; 32]) -> bool { + self.seen + .get(&(*addr, batch_id)) + .is_some_and(|t| t.elapsed() < self.ttl) } - fn insert(&mut self, addr: [u8; 32]) { - if self.seen.insert(addr, Instant::now()).is_none() { - self.order.push_back(addr); + fn insert(&mut self, addr: [u8; 32], batch_id: [u8; 32]) { + let key = (addr, batch_id); + if self.seen.insert(key, Instant::now()).is_none() { + self.order.push_back(key); } while self.order.len() > self.cap { if let Some(old) = self.order.pop_front() { @@ -238,12 +380,23 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { warn!("push node identity unresolvable; /v1/push will 503 (probe/status still work)"); } + // Metered mode is validated *before* the listener binds. Every failure + // here is one that would otherwise be discovered by a paying client: + // a parameter set that bricks accounts (§10.1), an origin the relay + // cannot state (§11.1), a chain with no vetted factory (§6), or state + // it cannot persist (§11.4). Refuse to boot instead. + let metered = match build_metered(&opts) { + Ok(m) => m, + Err(e) => return Err(format!("--meter: {e}").into()), + }; + let listener = tokio::net::TcpListener::bind(opts.listen).await?; info!( - "pusher listening on http://{} (probe {}; push {}; {} known peers from {})", + "pusher listening on http://{} (probe {}; push {}; mode {}; {} known peers from {})", opts.listen, if opts.probe_enabled { "ON" } else { "off" }, if push.is_some() { "ON" } else { "off" }, + if metered.is_some() { "metered" } else { "open" }, peers_known, opts.peerlist.display(), ); @@ -255,7 +408,9 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { peers_known: AtomicUsize::new(peers_known), batch_cache: std::sync::Mutex::new(HashMap::new()), push, - owner_cache: std::sync::Mutex::new(HashMap::new()), + owner_cache: std::sync::Mutex::new(OwnerCache::new(OWNER_CACHE_CAP)), + meter: std::sync::Mutex::new(crate::meter::Meter::default()), + metered, }); // Background warm-pool maintenance: fill on startup and keep the pool @@ -266,26 +421,78 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { tokio::spawn(async move { push_maintenance(s).await }); } + // Bound concurrent connections. The permit is held for the connection's + // lifetime, so once `max_conns` are in flight the loop stops accepting and + // the OS backlog supplies the backpressure. Without this the accept loop + // spawned per connection with no cap at all, and (see the timer below) + // held those connections open forever. + let max_conns = std::env::var("HOVERFLY_PUSH_MAX_CONNS") + .ok() + .and_then(|s| s.parse::().ok()) + .map(|n| n.clamp(8, 4096)) + .unwrap_or(PUSH_MAX_CONNS_DEFAULT); + info!("pusher connection cap = {max_conns} (HOVERFLY_PUSH_MAX_CONNS to override)"); + let conns = Arc::new(tokio::sync::Semaphore::new(max_conns)); + let mut accept_backoff_ms = 0u64; + loop { - let (stream, _remote) = listener.accept().await?; + let Ok(permit) = conns.clone().acquire_owned().await else { + break Ok(()); // semaphore closed — shutting down + }; + let (stream, remote) = match listener.accept().await { + Ok(v) => { + accept_backoff_ms = 0; + v + } + Err(e) => { + // Per-connection accept errors (EMFILE, ENFILE, ECONNABORTED) + // are exactly what a connection burst produces against a + // process that also holds a warm libp2p pool. Propagating + // them with `?` killed the whole relay; back off instead. + let wait = accept_backoff_ms.max(10); + warn!("accept error: {e} — retrying in {wait}ms"); + tokio::time::sleep(std::time::Duration::from_millis(wait)).await; + accept_backoff_ms = (wait * 2).min(ACCEPT_BACKOFF_MAX_MS); + continue; + } + }; let io = hyper_util::rt::TokioIo::new(stream); let state = state.clone(); + // The peer address from the accept loop, never a client-supplied + // header: `/v1/challenge` rate-limits per IP, and a limiter keyed on + // anything the caller controls limits nothing. + let peer_ip = remote.ip().to_string(); tokio::spawn(async move { + // Held until the connection finishes, so the cap above is real. + let _permit = permit; let svc = service_fn(move |req| { let state = state.clone(); - async move { Ok::<_, Infallible>(handle(state, req).await) } + let peer_ip = peer_ip.clone(); + async move { Ok::<_, Infallible>(handle(state, req, &peer_ip).await) } }); - // Streamed probe responses outlive any sane header timeout; - // hyper's defaults are fine, errors here are just client - // disconnects. + // A timer MUST be installed or hyper's `header_read_timeout` + // default is silently inert: with `Time::Empty` the check logs + // "timeout has default, but no timer set" and returns `None`, so + // the previous builder had no header timeout whatsoever and a + // client could hold a connection open indefinitely by dribbling + // request headers. This bounds only the *request header* read — + // streamed probe and push responses are unaffected. let _ = hyper::server::conn::http1::Builder::new() + .timer(hyper_util::rt::TokioTimer::new()) + .header_read_timeout(std::time::Duration::from_secs( + HEADER_READ_TIMEOUT_SECS, + )) .serve_connection(io, svc) .await; }); } } -async fn handle(state: Arc, req: Request) -> Response { +async fn handle( + state: Arc, + req: Request, + peer: &str, +) -> Response { // Browsers push cross-origin (a dApp on some origin → this relay), with a // custom content-type that triggers a CORS preflight. Answer OPTIONS and // tag every response with permissive CORS headers — the relay serves no @@ -296,10 +503,19 @@ async fn handle(state: Arc, req: Request) -> Respo } let mut resp = match (req.method(), req.uri().path()) { (&Method::GET, "/v1/status") => status_response(&state), + (&Method::GET, "/v1/meter") => meter_response(&state, req.headers()), + (&Method::GET, "/v1/account") => account_response(&state, req.headers()), + (&Method::GET, "/v1/challenge") => { + // Per-IP, because no account exists yet. `peer` comes from the + // accept loop, never from a client-supplied header. + challenge_response(state, req.uri().query(), peer).await + } + (&Method::POST, "/v1/pay") => pay_response(state, req).await, (&Method::POST, "/v1/probe") => probe_response(state, req.uri().query()), (&Method::POST, "/v1/tcpcheck") => tcpcheck_response(state, req.uri().query()), (&Method::POST, "/v1/push") => push_response(state, req).await, - (_, "/v1/probe") | (_, "/v1/status") | (_, "/v1/tcpcheck") | (_, "/v1/push") => { + (_, "/v1/probe") | (_, "/v1/status") | (_, "/v1/tcpcheck") | (_, "/v1/push") + | (_, "/v1/meter") | (_, "/v1/challenge") | (_, "/v1/pay") | (_, "/v1/account") => { json_line_response(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") } _ => json_line_response(StatusCode::NOT_FOUND, "not found"), @@ -308,6 +524,513 @@ async fn handle(state: Arc, req: Request) -> Respo resp } +/// What admission decided, carried into the push task so completion can +/// convert the reservation into debt. `None` in open mode. +pub struct Admitted { + account: [u8; 20], + batch: [u8; 32], + reserved_plur: u128, +} + +/// Metered admission for `/v1/push` (§7.2). +/// +/// Runs before the body is read, and reads **no chain state** — the +/// challenge already carries the credit line, resolved once when it was +/// issued. That is what keeps up to 512 ecrecovers and an RPC round trip +/// off the front of every request. +fn admit_metered( + state: &Arc, + req: &Request, +) -> Result, Response> { + let Some(m) = state.metered.as_ref() else { + return Ok(None); + }; + let raw = req + .headers() + .get(crate::metered::CHALLENGE_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + let verified = m + .verify_header(raw, crate::challenge::now_unix()) + .map_err(|e| json_line_response(StatusCode::UNAUTHORIZED, &e))?; + if !m.allow_account(&verified.account) { + return Err(json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down")); + } + // The reservation ledger is attacker-influenced (one entry per batch in + // standing), so shed rather than grow without bound (§7.2). + if m.shed_reservations() { + return Err(json_line_response( + StatusCode::SERVICE_UNAVAILABLE, + "too many accounts with live reservations", + )); + } + // Bound the reservation by the *declared* body. Same quantity, same + // arithmetic as the eventual bill (§8), so there is no estimate to be + // wrong about — and a one-frame POST reserves one frame's worth rather + // than a flat PUSH_BATCH_MAX, which is what keeps small batches usable. + let declared = req + .headers() + .get(hyper::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + let Some(declared) = declared else { + return Err(json_line_response( + StatusCode::LENGTH_REQUIRED, + "metered mode requires Content-Length so the reservation can be bounded", + )); + }; + if declared > PUSH_MAX_BODY as u64 { + return Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body exceeds limit", + )); + } + let adm = m.reserve_for_body(verified.account, declared, verified.cap_plur); + if adm.over_cap { + if m.cfg.hard_mode { + // Hard mode: give the reservation back, then refuse. Keeping it + // would leak credit on every refusal. + m.ledger + .lock() + .expect("ledger poisoned") + .release(verified.account, adm.reserved_plur); + return Err(json_response( + StatusCode::PAYMENT_REQUIRED, + &serde_json::json!({ + "error": "payment required", + "outstanding_plur": adm.outstanding_plur.to_string(), + "max_outstanding_plur": adm.cap_plur.to_string(), + "settle_every_plur": m.cfg.params.settle_every_plur.to_string(), + }), + )); + } + // Soft mode: record and serve anyway. This is the instrument Stage 0 + // could not provide — how often a real client *would* have been + // 402'd, measured against live traffic before anyone is refused. + tracing::info!( + account = %hex::encode(verified.account), + outstanding_plur = %adm.outstanding_plur, + cap_plur = %adm.cap_plur, + "soft-mode overshoot: this request would 402 under hard mode" + ); + } + Ok(Some(Admitted { + account: verified.account, + batch: verified.batch, + reserved_plur: adm.reserved_plur, + })) +} + +/// Validate `--meter` and build the metered state, or `None` for `open`. +/// +/// Every check here is one a paying client would otherwise discover the +/// hard way, so all of them are fatal rather than warnings. +fn build_metered(opts: &PusherOpts) -> Result, String> { + let Some(m) = &opts.meter else { + return Ok(None); + }; + m.params.validate()?; + if m.origins.is_empty() || m.origins.iter().any(|o| o.trim().is_empty()) { + return Err( + "--origin is required and must be non-empty: the relay compares a challenge's \ + origin against configuration, never against the Host header, and a relay that \ + cannot state its own hostname cannot close the cross-relay replay (§11.1)" + .into(), + ); + } + let factory = crate::batch::swap_factory_for_chain(m.chain_id).ok_or_else(|| { + format!( + "no vetted SimpleSwapFactory for chain {}: a factory address must never come \ + from the wire, so metered mode cannot run here (§6)", + m.chain_id + ) + })?; + if m.beneficiary == [0u8; 20] { + return Err("--beneficiary must be set: it is the EOA cheques are made out to".into()); + } + std::fs::create_dir_all(&m.state_dir) + .map_err(|e| format!("--state-dir {}: {e}", m.state_dir.display()))?; + let ledger = crate::ledger::Ledger::load_or_create(m.state_dir.join("ledger.json")) + .map_err(|e| { + format!( + "ledger at {}: {e} — metered mode requires durable state, because losing \ + last_cumulative turns one signature into unlimited free service (§11.4)", + m.state_dir.display() + ) + })?; + info!( + "metered mode: origin(s) {} beneficiary 0x{} chain {} price {} PLUR/KiB ({} mode)", + m.origins.join(","), + hex::encode(m.beneficiary), + m.chain_id, + m.params.price_plur_per_kib, + if m.hard_mode { "hard" } else { "soft" }, + ); + Ok(Some(crate::metered::Metered::new( + crate::metered::MeterConfig { + origins: m.origins.clone(), + beneficiary: m.beneficiary, + chain_id: m.chain_id, + factory, + params: m.params, + hard_mode: m.hard_mode, + }, + ledger, + ))) +} + +/// `GET /v1/challenge?account=&batch=` — issue a capability (§7.2). +/// +/// This is where the chain reads happen: standing is resolved once, priced +/// into a credit line, and baked into the nonce, so `/v1/push` admission +/// touches no chain state at all. Amplification is bounded by *distinct +/// batch ids* rather than request count, because the owner cache answers +/// repeats — plus a per-IP limit on top. +async fn challenge_response( + state: Arc, + query: Option<&str>, + peer_ip: &str, +) -> Response { + let Some(m) = state.metered.as_ref() else { + return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); + }; + if !m.allow_challenge(peer_ip) { + return json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down"); + } + let q = parse_query(query); + let (Some(account_hex), Some(batch_hex)) = (q.get("account"), q.get("batch")) else { + return json_line_response(StatusCode::BAD_REQUEST, "need account= and batch="); + }; + let account = match parse_hex_array::<20>(account_hex) { + Ok(a) => a, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &format!("account: {e}")), + }; + let batch = match parse_hex_array::<32>(batch_hex) { + Ok(b) => b, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &format!("batch: {e}")), + }; + let batch_id_hex = hex::encode(batch); + let mut budget = 1usize; + let (owner, remaining_value) = match resolve_owner(&state, &batch_id_hex, &mut budget).await { + Ok(v) => v, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e), + }; + // No nonce for a batch this account does not own. Without this, + // admission would grant a reservation to any EOA that can sign, and + // free identities could occupy ledger entries without owning anything. + if owner != account { + return json_line_response( + StatusCode::FORBIDDEN, + "account is not the on-chain owner of that batch", + ); + } + let origin = m.cfg.origins.first().cloned().unwrap_or_default(); + match m.issue( + account, + batch, + remaining_value, + &origin, + crate::challenge::now_unix(), + ) { + Ok(issued) => json_response(StatusCode::OK, &issued.to_json()), + Err(e) => json_line_response(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()), + } +} + +/// `POST /v1/pay` — accept a cheque (§11.6). +/// +/// Ordered cheapest-first, but the *bound* comes from the two free refusals +/// at the top: a challenge is required, and an account with no debt cannot +/// spend a single `eth_call`. Without those, every "free" check passes for +/// a cheque an attacker synthesizes at zero cost and each garbage POST buys +/// one `deployedContracts` call. +async fn pay_response(state: Arc, req: Request) -> Response { + let Some(m) = state.metered.as_ref() else { + return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); + }; + let header = req + .headers() + .get(crate::metered::CHALLENGE_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string(); + let verified = match m.verify_header(&header, crate::challenge::now_unix()) { + Ok(v) => v, + Err(e) => return json_line_response(StatusCode::UNAUTHORIZED, &e), + }; + if !m.allow_account(&verified.account) { + return json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down"); + } + // No debt, no cheque — before parsing anything. Postpaid means an honest + // client always has debt by the time it settles, so this costs nothing + // legitimate and makes the endpoint useless to anyone who has not first + // done billable work. + let owed = m.ledger.lock().expect("ledger poisoned").owed(&verified.account); + if owed == 0 { + return json_line_response(StatusCode::BAD_REQUEST, "nothing owed on this account"); + } + + let body = match read_body_limited(req, crate::protocols::swap::MAX_CHEQUE_JSON).await { + Ok(b) => b, + Err(resp) => return resp, + }; + let cheque = match crate::protocols::swap::decode_signed_cheque_json(&body) { + Ok(c) => c, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), + }; + if cheque.beneficiary != m.cfg.beneficiary { + return json_line_response(StatusCode::BAD_REQUEST, "cheque is not made out to us"); + } + let cumulative: u128 = match u128::try_from(cheque.cumulative_payout) { + Ok(v) if v <= crate::ledger::MAX_CUMULATIVE_PLUR => v, + _ => return json_line_response(StatusCode::BAD_REQUEST, "cumulative payout is implausible"), + }; + let have = m + .ledger + .lock() + .expect("ledger poisoned") + .last_cumulative(&verified.account, &cheque.chequebook); + if cumulative <= have { + return json_line_response( + StatusCode::BAD_REQUEST, + &format!("cheque cumulative {cumulative} does not exceed the {have} already accepted"), + ); + } + if cumulative - have < m.cfg.params.min_cheque_plur { + return json_line_response( + StatusCode::BAD_REQUEST, + &format!( + "cheque credits {} but the dust floor is {}", + cumulative - have, + m.cfg.params.min_cheque_plur + ), + ); + } + // Every free check has passed; only now does this cost RPC. + match m.is_deployed(&state.opts.rpc_url, cheque.chequebook).await { + Ok(true) => {} + Ok(false) => { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook was not deployed by the canonical factory", + ); + } + Err(e) => return json_line_response(StatusCode::BAD_GATEWAY, &e), + } + let issuer_ok = crate::signer::recover_cheque_issuer( + &cheque.chequebook, + &cheque.beneficiary, + cheque.cumulative_payout, + m.cfg.chain_id, + &cheque.signature, + ); + let recovered = match issuer_ok { + Ok(a) => a, + Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), + }; + let cb_state = match crate::batch::read_chequebook_state( + &state.opts.rpc_url, + alloy_primitives::Address::from(cheque.chequebook), + alloy_primitives::Address::from(m.cfg.beneficiary), + ) + .await + { + Ok(s) => s, + Err(e) => return json_line_response(StatusCode::BAD_GATEWAY, &format!("chequebook: {e}")), + }; + if cb_state.bounced { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook has bounced a cheque before and is refused", + ); + } + if cb_state.issuer.into_array() != recovered { + return json_line_response(StatusCode::BAD_REQUEST, "cheque was not signed by the issuer"); + } + if cb_state.issuer.into_array() != verified.account { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook issuer is not the account that owns this batch", + ); + } + // The funding check, against `liquidBalanceFor(us)` rather than + // `balance()` — the latter counts other beneficiaries' hard deposits as + // our coverage, which is unsound (§11.2). + let paid_out = u128::try_from(cb_state.paid_out_to_us).unwrap_or(u128::MAX); + let liquid = u128::try_from(cb_state.liquid_for_us).unwrap_or(u128::MAX); + if liquid < cumulative.saturating_sub(paid_out) { + return json_line_response( + StatusCode::BAD_REQUEST, + "chequebook cannot cover this cheque", + ); + } + match m.credit(verified.account, cheque.chequebook, cumulative) { + Ok(accepted) => { + let l = m.ledger.lock().expect("ledger poisoned"); + json_response( + StatusCode::OK, + &serde_json::json!({ + "accepted_plur": accepted.to_string(), + "cumulative": cumulative.to_string(), + "owed_plur": l.owed(&verified.account).to_string(), + "outstanding_plur": l.outstanding(&verified.account).to_string(), + }), + ) + } + Err(e) => json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), + } +} + +/// `GET /v1/account` — the client's own ledger row (§7). +/// +/// Authenticated with the challenge header: unauthenticated it is a +/// per-identity volume oracle over on-chain-enumerable batch owners, and a +/// targeting oracle for tipping a victim into 402 at a chosen moment. +fn account_response(state: &State, headers: &hyper::HeaderMap) -> Response { + let Some(m) = state.metered.as_ref() else { + return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); + }; + let raw = headers + .get(crate::metered::CHALLENGE_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + let verified = match m.verify_header(raw, crate::challenge::now_unix()) { + Ok(v) => v, + Err(e) => return json_line_response(StatusCode::UNAUTHORIZED, &e), + }; + let l = m.ledger.lock().expect("ledger poisoned"); + json_response( + StatusCode::OK, + &serde_json::json!({ + "account": format!("0x{}", hex::encode(verified.account)), + "owed_plur": l.owed(&verified.account).to_string(), + "reserved_plur": l.reserved(&verified.account).to_string(), + "outstanding_plur": l.outstanding(&verified.account).to_string(), + "max_outstanding_plur": verified.cap_plur.to_string(), + "settle_every_plur": m.cfg.params.settle_every_plur.to_string(), + "min_cheque_plur": m.cfg.params.min_cheque_plur.to_string(), + }), + ) +} + +/// Read a request body with a hard cap, so an oversized one costs nothing. +async fn read_body_limited( + req: Request, + max: usize, +) -> Result> { + use http_body_util::BodyExt; + use hyper::body::Body as _; + use std::time::Duration; + if let Some(len) = req.body().size_hint().upper() + && len > max as u64 + { + return Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body too large", + )); + } + match tokio::time::timeout( + Duration::from_secs(HEADER_READ_TIMEOUT_SECS), + req.into_body().collect(), + ) + .await + { + Ok(Ok(c)) => { + let b = c.to_bytes(); + if b.len() > max { + return Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body too large", + )); + } + Ok(b) + } + Ok(Err(e)) => Err(json_line_response( + StatusCode::BAD_REQUEST, + &format!("body: {e}"), + )), + Err(_) => Err(json_line_response(StatusCode::REQUEST_TIMEOUT, "body read timed out")), + } +} + +fn parse_hex_array(s: &str) -> Result<[u8; N], String> { + let raw = hex::decode(s.trim_start_matches("0x")).map_err(|e| e.to_string())?; + if raw.len() != N { + return Err(format!("must be {N} bytes, got {}", raw.len())); + } + let mut out = [0u8; N]; + out.copy_from_slice(&raw); + Ok(out) +} + +/// Body of a maximal POST, for the credit-line comparison in `src/meter.rs`: +/// a batch whose line is under this has to split its uploads across smaller +/// requests (incentives §7.2). +fn full_post_kib() -> u64 { + (PUSH_BATCH_MAX * pushframe::MAX_FRAME_LEN).div_ceil(1024) as u64 +} + +/// Total per-stream push attempts since boot. Incentives §9.1's egress +/// multiplier is this over frames admitted — the counters are bumped per +/// stream rather than per chunk (`src/client.rs:5361-5363`), so losing +/// racers and shallow retries are both included, which is exactly the cost +/// the relay actually pays. +fn stream_attempts() -> u64 { + use crate::transport::diag; + use std::sync::atomic::Ordering; + diag::PUSH_OUTCOME_OK.load(Ordering::Relaxed) + + diag::PUSH_OUTCOME_SHALLOW.load(Ordering::Relaxed) + + diag::PUSH_OUTCOME_OVERDRAFT.load(Ordering::Relaxed) + + diag::PUSH_OUTCOME_ERROR.load(Ordering::Relaxed) +} + +/// `GET /v1/meter` — Stage 0 shadow-metering detail (incentives §14). +/// +/// **Open by default; set `HOVERFLY_PUSH_METER_TOKEN` to require a bearer +/// token.** Stage 0's rows are derived from state that is already public: +/// batch owners and their balances are on-chain and enumerable from +/// `BatchCreated`, and the stamp on every relayed chunk names its batch, +/// which retrieval hands back (incentives §2). All this endpoint adds is +/// relay attribution and timing, so gating it by default buys little and +/// costs a lot — an instrument nobody reads answers no questions, and +/// deciding whether to meter at all is the only reason Stage 0 exists. +/// +/// That flips at **Stage 1**, and the reason is worth recording because it +/// is not the obvious one. Once 402s are live, `/v1/account` exposes an +/// account's *outstanding* balance, which lets a reader time a stamp replay +/// (§11.1) to tip a victim over its cap mid-upload. That is an active +/// attack enabler rather than a privacy leak, and it is the real reason +/// incentives §7 authenticates that endpoint. +fn meter_response(state: &State, headers: &hyper::HeaderMap) -> Response { + if let Ok(want) = std::env::var("HOVERFLY_PUSH_METER_TOKEN") { + let got = headers + .get(hyper::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .unwrap_or_default(); + // Compare the *contents* in constant time — a byte-wise early exit + // would let the token be ground out one character at a time. Length + // is compared directly and so is observable, which is fine: it is a + // static property of the operator's config, not something an + // attacker can narrow down to a value. + let ok = got.len() == want.len() + && got + .bytes() + .zip(want.bytes()) + .fold(0u8, |acc, (a, b)| acc | (a ^ b)) + == 0; + if !ok { + return json_line_response(StatusCode::UNAUTHORIZED, "unauthorized"); + } + } + let body = state + .meter + .lock() + .expect("meter poisoned") + .detail(full_post_kib(), stream_attempts(), 100); + json_response(StatusCode::OK, &body) +} + /// Add permissive CORS headers to a response. fn add_cors(h: &mut hyper::HeaderMap) { use hyper::header::HeaderValue; @@ -333,7 +1056,7 @@ fn cors_preflight() -> Response { ); h.insert( "access-control-allow-headers", - HeaderValue::from_static("content-type"), + HeaderValue::from_static("content-type, x-hoverfly-challenge"), ); h.insert("access-control-max-age", HeaderValue::from_static("86400")); resp @@ -377,6 +1100,20 @@ fn status_response(state: &State) -> Response { // log-only, so a deployed relay can be inspected without shell // access to it. "diag": diag::summary(), + // Stage 0 shadow metering (incentives §14): what a metered relay + // *would* have billed. Nothing is charged and no client behaviour + // changes. Aggregates only — per-account detail is behind + // /v1/meter, since it names identities (see `meter_response`). + // Metered-mode quote (§7.3), signed with the node-identity key so a + // price is not repudiable in either direction. Absent in open mode. + "payment": payment_quote(state), + "meter": state.push.as_ref().map(|_| { + state + .meter + .lock() + .expect("meter poisoned") + .summary(full_post_kib(), stream_attempts()) + }), }); json_response(StatusCode::OK, &body) } @@ -592,6 +1329,7 @@ fn build_push_state(opts: &PusherOpts) -> Option { }; let keypair = crate::inbound::libp2p_keypair_from_identity(&node_signer); let overlay = *node_signer.overlay(); + let quote_signer = node_signer.clone(); let snapshot = crate::protocols::status::StatusSnapshot::default(); let transport = Transport::new_with_keypair(node_signer, opts.transport.clone(), keypair) .with_status_snapshot(snapshot); @@ -619,6 +1357,7 @@ fn build_push_state(opts: &PusherOpts) -> Option { pool: tokio::sync::Mutex::new(None), pool_target, pool_live: AtomicUsize::new(0), + signer: quote_signer, overlay, budget_gb, bytes_pushed: AtomicU64::new(0), @@ -646,15 +1385,36 @@ async fn push_response( "push disabled (no node identity resolvable)", ); } - // Bounded body read — a whole batch, not a stream. - let bytes = match Limited::new(req.into_body(), PUSH_MAX_BODY).collect().await { - Ok(c) => c.to_bytes(), - Err(_) => { + // Metered admission, entirely before the body is read (§7.2). Nothing + // here touches the chain: the challenge already carries the credit line, + // which is the point of issuing it. + let admitted = match admit_metered(&state, &req) { + Ok(a) => a, + Err(resp) => return resp, + }; + // Bounded body read — a whole batch, not a stream. Bounded in *time* as + // well as size: the size limit alone let a client dribble a body forever + // and hold the connection (and, under metering, its admission + // reservation) for free. + let read = tokio::time::timeout( + std::time::Duration::from_secs(PUSH_BODY_READ_TIMEOUT_SECS), + Limited::new(req.into_body(), PUSH_MAX_BODY).collect(), + ) + .await; + let bytes = match read { + Ok(Ok(c)) => c.to_bytes(), + Ok(Err(_)) => { return json_line_response( StatusCode::PAYLOAD_TOO_LARGE, "body exceeds limit or read error", ); } + Err(_) => { + return json_line_response( + StatusCode::REQUEST_TIMEOUT, + "body read timed out", + ); + } }; let chunks = match pushframe::decode_batch(&bytes, PUSH_BATCH_MAX) { Ok(c) => c, @@ -673,7 +1433,7 @@ async fn push_response( // sessions from it (maintain=false), so no per-push dial burst. let (tx, rx) = futures::channel::mpsc::unbounded::, Infallible>>(); tokio::spawn(async move { - run_push(state, chunks, tx).await; + run_push(state, chunks, tx, admitted).await; }); Response::builder() @@ -689,6 +1449,7 @@ async fn run_push( state: Arc, chunks: Vec, tx: futures::channel::mpsc::UnboundedSender, Infallible>>, + admitted: Option, ) { let push = state.push.as_ref().expect("push state present"); let mut dedup_hits = 0usize; @@ -727,8 +1488,21 @@ async fn run_push( // recovers to. All chunks in one upload share a batch; verify the // batch once, then check each chunk's recovered signer against it. let mut accepted: Vec = Vec::with_capacity(chunks.len()); - let mut batch_owner: Option<[u8; 20]> = None; + let mut batch_owner: Option<([u8; 20], u128)> = None; let mut batch_hex: Option = None; + // Stage 0 shadow metering, accumulated on this task's stack and merged + // once below. Per-frame locking would serialize concurrent POSTs on a + // counter nobody is billed from. + let mut tally = crate::meter::PostTally::default(); + // Body bytes that will actually be billed: admitted minus dedup hits, + // which did no push work and so cost nothing (§8.2). + let mut billable_bytes: u64 = 0; + // addr -> owning batch, for the recent-ack cache: dedup must be + // scoped per (addr, batch) so one uploader's frame can't be acked + // "ok" under another uploader's stamp (§15). + let mut batch_of: HashMap<[u8; 32], [u8; 32]> = HashMap::with_capacity(chunks.len()); + // Distinct on-chain batch resolutions this request may perform. + let mut rpc_budget = PUSH_MAX_BATCH_LOOKUPS; for chunk in chunks { let vs = match crate::stamp::validate(&chunk.addr, &chunk.stamp) { @@ -743,7 +1517,7 @@ async fn run_push( let owner = if batch_hex.as_deref() == Some(bid.as_str()) { batch_owner } else { - match resolve_owner(&state, &bid).await { + match resolve_owner(&state, &bid, &mut rpc_budget).await { Ok(o) => { batch_hex = Some(bid.clone()); batch_owner = Some(o); @@ -756,18 +1530,51 @@ async fn run_push( } }; match owner { - Some(o) if o == vs.signer => { + Some((o, batch_value)) if o == vs.signer => { // Duplicate suppression: a client hedging a straggler // sends the same frame to two lanes on purpose. Answering // from the recent-ack cache makes the loser of that race // free instead of a second real push through the pool. + // Scoped per (addr, batch) — a bare address would let the + // hit discard the submitted stamp (§15). + let mut batch_id = [0u8; 32]; + batch_id.copy_from_slice(vs.batch_id); + // One batch per request under metering (§6). Standing, the + // credit line and the reservation are all properties of a + // *batch*, so a POST that mixes them lets one good-standing + // frame carry 511 others from an account that is over its + // cap (§11.8). Rejected rather than billed to whoever it + // names. + if let Some(adm) = &admitted + && batch_id != adm.batch + { + ack( + &chunk.addr, + "err", + Some("frame batch does not match the challenge (one batch per request)"), + ); + continue; + } + // Body bytes this frame occupied — header plus wire — which + // is the unit incentives §8 bills. Recorded for every + // admitted frame, dedup hits included, because the relay + // received those bytes either way. + let key = crate::meter::AccountBatch { + owner: o, + batch: batch_id, + }; + let frame_bytes = (pushframe::HEADER_LEN + chunk.wire.len()) as u64; + tally.admit(key, batch_value, frame_bytes); + billable_bytes += frame_bytes; let dup = push .recent .lock() .expect("recent-ack cache poisoned") - .contains(&chunk.addr); + .contains(&chunk.addr, batch_id); if dup { dedup_hits += 1; + tally.dedup(key, batch_value, frame_bytes); + billable_bytes = billable_bytes.saturating_sub(frame_bytes); ack_ok( &chunk.addr, crate::client::PushInfo { @@ -778,10 +1585,11 @@ async fn run_push( }, ); } else { + batch_of.insert(chunk.addr, batch_id); accepted.push(chunk); } } - Some(o) => ack( + Some((o, _)) => ack( &chunk.addr, "err", Some(&format!( @@ -794,6 +1602,32 @@ async fn run_push( } } + // Turn the reservation into debt for what was actually admitted, and + // release the rest (§10.2). Runs before the early return below, so a + // POST that admitted nothing still gives its reservation back — leaking + // it would ratchet the account toward a 402 it can never clear. + if let (Some(adm), Some(m)) = (&admitted, state.metered.as_ref()) { + let billed = m.cfg.params.price_bytes(billable_bytes); + let mut l = m.ledger.lock().expect("ledger poisoned"); + l.commit(adm.account, adm.reserved_plur, billed); + if let Err(e) = l.persist() { + // `owed` is written at batch completion, so a failed persist + // forfeits at most this batch — the safe direction (§10.2). + tracing::error!("ledger persist after commit failed: {e}"); + } + } + + // One lock for the whole request. Runs before the early return below so + // an all-dedup POST is still measured — those are exactly the requests + // §8.2 bills at zero, and their share is a number Stage 0 wants. + if !tally.is_empty() { + state + .meter + .lock() + .expect("meter poisoned") + .merge(std::mem::take(&mut tally)); + } + if accepted.is_empty() { send_line(&serde_json::json!({ "done": {"pushed": 0, "rejected": dedup_hits == 0, "dedup": dedup_hits} @@ -880,10 +1714,15 @@ async fn run_push( for (a, ok) in &seen { if *ok { pushed += 1; + // A successful chunk belongs to exactly the batch it was + // admitted under; cache it under (addr, batch) so dedup stays + // scoped. `batch_of` is only missing a key if the chunk was + // admitted on a path that never mapped it — none currently. + let bid = batch_of.get(a).copied().unwrap_or([0u8; 32]); push.recent .lock() .expect("recent-ack cache poisoned") - .insert(*a); + .insert(*a, bid); } } if total > 0 { @@ -919,43 +1758,93 @@ async fn run_push( /// is "the batch is alive" (docs/pusher-design.md §5), so a batch whose /// `remainingBalance` has drained to zero is rejected — bee nodes would /// refuse its stamps anyway, and pushing them just burns relay egress. -/// The aliveness read happens once per batch (the cache never expires); -/// a batch that dies *while cached* only wastes its own push attempts — -/// bees reject the stamps downstream — and a pusher restart re-checks. -async fn resolve_owner(state: &State, batch_id_hex: &str) -> Result<[u8; 20], String> { - if let Some(o) = state +/// The aliveness read happens once per batch and is cached for +/// `OWNER_OK_TTL_SECS`; a batch that dies *while cached* only wastes its +/// own push attempts — bees reject the stamps downstream. +/// `rpc_budget` bounds how many *distinct* batch ids one request may push +/// through to the chain. A cache hit never spends it; only a genuine miss +/// does. Without it, negative caching alone still leaves a single POST +/// naming `PUSH_BATCH_MAX` different bogus batch ids able to issue that +/// many serial `eth_call`s, since every one of them is a first miss. +async fn resolve_owner( + state: &State, + batch_id_hex: &str, + rpc_budget: &mut usize, +) -> Result<([u8; 20], u128), String> { + if let Some(hit) = state .owner_cache .lock() .expect("owner cache poisoned") .get(batch_id_hex) { - return Ok(*o); + return match hit { + OwnerLookup::Owner(o, value) => Ok((o, value)), + OwnerLookup::Rejected(why) => Err(why), + }; + } + if *rpc_budget == 0 { + return Err(format!( + "batch {batch_id_hex}: too many distinct batches in one request \ + (limit {PUSH_MAX_BATCH_LOOKUPS}); split them across requests" + )); } + *rpc_budget -= 1; + + // Only *definitive* on-chain answers are cached. A transport error must + // not blacklist a live batch for OWNER_BAD_TTL_SECS, so those propagate + // uncached. let stamp_addr: alloy_primitives::Address = crate::batch::MAINNET_POSTAGE_STAMP .parse() .expect("hardcoded valid"); let info = crate::batch::read_batch(&state.opts.rpc_url, stamp_addr, batch_id_hex) .await .map_err(|e| format!("batch owner RPC: {e}"))?; + let reject = |state: &State, why: String| -> String { + state + .owner_cache + .lock() + .expect("owner cache poisoned") + .insert(batch_id_hex, OwnerLookup::Rejected(why.clone())); + why + }; if info.not_found { - return Err(format!("batch {batch_id_hex} not found on-chain")); + return Err(reject( + state, + format!("batch {batch_id_hex} not found on-chain"), + )); } let remaining = crate::batch::read_remaining_balance(&state.opts.rpc_url, stamp_addr, batch_id_hex) .await .map_err(|e| format!("batch balance RPC: {e}"))?; if remaining.is_zero() { - return Err(format!( - "batch {batch_id_hex} has expired (zero remaining balance) — bees would reject every stamp" + return Err(reject( + state, + format!( + "batch {batch_id_hex} has expired (zero remaining balance) — bees would reject every stamp" + ), )); } + // Total value still funded on this batch: `remainingBalance` is PLUR per + // chunk, and a batch of depth `d` covers 2^d chunks (incentives §6). + // Saturating rather than wrapping — a nonsense depth from a malformed + // read must not produce a small number that looks like a real answer. + let depth_factor = 1u128 + .checked_shl(u32::from(info.depth)) + .unwrap_or(u128::MAX); + let remaining_value_plur = u128::try_from(remaining) + .unwrap_or(u128::MAX) + .saturating_mul(depth_factor); let owner = info.owner.into_array(); state .owner_cache .lock() .expect("owner cache poisoned") - .insert(batch_id_hex.to_string(), owner); - Ok(owner) + .insert( + batch_id_hex, + OwnerLookup::Owner(owner, remaining_value_plur), + ); + Ok((owner, remaining_value_plur)) } /// Return the warm pool, filling/topping it up to `push.pool_target`. @@ -1409,3 +2298,130 @@ fn json_response(status: StatusCode, body: &serde_json::Value) -> Response Response { json_response(status, &serde_json::json!({"error": message})) } + +#[cfg(test)] +mod owner_cache_tests { + use super::*; + + fn owner_of(c: &OwnerCache, k: &str) -> Option<[u8; 20]> { + match c.get(k) { + Some(OwnerLookup::Owner(o, _)) => Some(o), + _ => None, + } + } + + fn value_of(c: &OwnerCache, k: &str) -> Option { + match c.get(k) { + Some(OwnerLookup::Owner(_, v)) => Some(v), + _ => None, + } + } + + #[test] + fn rejections_are_cached_so_a_bogus_batch_is_not_re_resolved() { + let mut c = OwnerCache::new(16); + assert!(c.get("deadbeef").is_none(), "cold cache must miss"); + c.insert("deadbeef", OwnerLookup::Rejected("not found".into())); + // The point of the fix: a second frame naming the same bogus batch + // finds a cached rejection instead of issuing another eth_call. + assert!( + matches!(c.get("deadbeef"), Some(OwnerLookup::Rejected(w)) if w == "not found"), + "rejection must be cached" + ); + } + + /// The batch's remaining value rides along on the cached success so + /// Stage 0 can price a credit line without a second `eth_call` + /// (`src/meter.rs`). A cache that dropped it would silently make every + /// batch look unpriced. + #[test] + fn the_cached_success_carries_the_batch_value() { + let mut c = OwnerCache::new(16); + c.insert("b0", OwnerLookup::Owner([7u8; 20], 100_000_000_000_000)); + assert_eq!(owner_of(&c, "b0"), Some([7u8; 20])); + assert_eq!(value_of(&c, "b0"), Some(100_000_000_000_000)); + } + + #[test] + fn eviction_is_bounded_by_cap() { + let mut c = OwnerCache::new(4); + for i in 0..64 { + c.insert(&format!("batch{i}"), OwnerLookup::Owner([i as u8; 20], 0)); + } + assert_eq!(c.map.len(), 4, "map must stay at cap"); + assert_eq!(c.order.len(), 4, "order must stay at cap"); + assert!(owner_of(&c, "batch0").is_none(), "oldest evicted"); + assert_eq!(owner_of(&c, "batch63"), Some([63u8; 20]), "newest retained"); + } + + #[test] + fn reinserting_a_key_does_not_grow_the_order_queue() { + let mut c = OwnerCache::new(8); + for _ in 0..32 { + c.insert("same", OwnerLookup::Owner([1u8; 20], 0)); + } + assert_eq!(c.order.len(), 1, "one order slot per distinct key"); + assert_eq!(owner_of(&c, "same"), Some([1u8; 20])); + } + + #[test] + fn entries_expire() { + // Zero-length TTLs are not reachable through the constants, so drive + // expiry by backdating the insert instant directly. + let mut c = OwnerCache::new(8); + c.insert("stale", OwnerLookup::Owner([2u8; 20], 0)); + let aged = Instant::now() + .checked_sub(std::time::Duration::from_secs(OWNER_OK_TTL_SECS + 1)) + .expect("clock supports backdating"); + c.map.get_mut("stale").expect("present").1 = aged; + assert!(c.get("stale").is_none(), "expired entry must not be served"); + } +} + +/// The signed `payment` block for `/v1/status` (incentives §7.3). +/// +/// An unsigned price is repudiable in both directions: the relay can serve +/// `P` and bill `10P`, the client can claim it saw `P/10`, and +/// reconciliation can detect the mismatch but never attribute it. +/// +/// It carries `node_eth_address` and `overlay_nonce` because +/// "pin `(url, overlay)`" is not implementable — an overlay is +/// `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`, so verifying a signature +/// yields the *eth address* while the nonce is neither transmitted nor +/// derivable. With both present a client can recompute the overlay and +/// check it against what the relay advertises, and pin the triple +/// `(url, node_eth_address, beneficiary)`. +fn payment_quote(state: &State) -> Option { + let m = state.metered.as_ref()?; + let push = state.push.as_ref()?; + let p = &m.cfg.params; + let mut body = serde_json::json!({ + "mode": "metered", + "enforcement": if m.cfg.hard_mode { "hard" } else { "soft" }, + "beneficiary": format!("0x{}", hex::encode(m.cfg.beneficiary)), + "node_eth_address": format!("0x{}", hex::encode(push.signer.eth_address())), + "overlay_nonce": format!("0x{}", hex::encode(state.opts.nonce)), + "origin": m.cfg.origins.first().cloned().unwrap_or_default(), + "chain_id": m.cfg.chain_id, + "factory": format!("0x{}", hex::encode(m.cfg.factory)), + "price_plur_per_kib": p.price_plur_per_kib.to_string(), + "min_cheque_plur": p.min_cheque_plur.to_string(), + "settle_every_plur": p.settle_every_plur.to_string(), + "max_outstanding_plur": p.max_outstanding_plur.to_string(), + "credit_ratio": p.credit_ratio, + "challenge_ttl_secs": crate::challenge::CHALLENGE_TTL_SECS, + }); + // Sign the canonical serialization of the block itself, so what the + // client verifies is exactly what it read. + let payload = body.to_string(); + match push.signer.sign_eip191(payload.as_bytes()) { + Ok(sig) => { + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + Some(body) + } + Err(e) => { + tracing::error!("cannot sign payment quote: {e}"); + None + } + } +} diff --git a/src/pushframe.rs b/src/pushframe.rs index 06e74e1..e100433 100644 --- a/src/pushframe.rs +++ b/src/pushframe.rs @@ -22,7 +22,11 @@ pub const STAMP_LEN: usize = 113; /// Max wire length: span(8) + max chunk data(4096). pub const MAX_WIRE_LEN: usize = 4104; /// Fixed frame header before the variable wire: addr + stamp + len(u16). -const HEADER_LEN: usize = 32 + STAMP_LEN + 2; +/// Public because it is the smallest a frame can be, which is what bounds +/// how many frames a body of a given `Content-Length` can hold — the basis +/// of the metered reservation (`docs/pusher-incentives.md` §7.2) and of +/// Stage 0's per-frame byte accounting. +pub const HEADER_LEN: usize = 32 + STAMP_LEN + 2; /// Upper bound on a single encoded frame. pub const MAX_FRAME_LEN: usize = HEADER_LEN + MAX_WIRE_LEN; diff --git a/src/pushsched.rs b/src/pushsched.rs index 4d7ae9d..dd13609 100644 --- a/src/pushsched.rs +++ b/src/pushsched.rs @@ -83,6 +83,10 @@ pub enum LaneHealth { /// 35 s to first byte on a sleeping instance), and a cold lane must not /// be able to swallow a full-size batch before proving it is awake. Warming, + /// Over its credit line, or the client has no way to pay it. Ineligible + /// but **not terminal** — unlike `Retired`, which is permanent for the + /// run. Cleared by [`Scheduler::fund_lane`] once a cheque is accepted. + Unfunded, /// Serving normally. Live, /// Failing; not eligible until `until_ms`. @@ -109,6 +113,14 @@ pub struct LaneInfo { pub budget_remaining_gb: Option, /// Live warm sessions; the strongest available prior on throughput. pub pool_live: Option, + /// Price in PLUR per KiB of body, from a *verified* signed quote + /// (`docs/pusher-incentives.md` §7.3). `None` on an `open` lane, and + /// also `None` when the quote failed verification — an unverifiable + /// quote is treated as "not metered" rather than "free", so the lane is + /// simply not paid and not scheduled for payment. + pub price_plur_per_kib: Option, + /// True when the lane enforces 402 rather than metering softly. + pub hard_enforcement: bool, } /// Tunables. Defaults are the shipping configuration. @@ -229,6 +241,8 @@ impl Lane { match self.health { LaneHealth::Live | LaneHealth::Warming => true, LaneHealth::Backoff { until_ms } => now_ms >= until_ms, + // Ineligible until paid, but recoverable within this run. + LaneHealth::Unfunded => false, LaneHealth::Retired => false, } } @@ -307,6 +321,15 @@ pub enum BatchOutcome { Answered, /// Transport error, non-2xx, or an unparseable body. Failed(String), + /// `402 Payment Required` — the account is over its credit line on this + /// lane (`docs/pusher-incentives.md` §12). + /// + /// **Not a failure.** Mapping it to `Failed` would charge lane health + /// for a *routine settlement*: five of them retire a perfectly healthy + /// lane mid-upload, which is the opposite of what should happen when a + /// relay says "pay me". The lane is paused until a cheque clears and its + /// streak is left untouched. + PaymentRequired, } /// Why a run couldn't finish. @@ -333,6 +356,7 @@ pub enum LaneHealthKind { Warming, Live, Backoff, + Unfunded, Retired, } @@ -726,6 +750,35 @@ impl Scheduler { .retain(|&k| self.chunks[k].phase == ChunkPhase::Pending); } + /// Bring a lane back after a cheque has been accepted. + /// + /// The counterpart to `PaymentRequired`: because that outcome left + /// `fail_streak` and `backoff_exp` alone, settling restores the lane to + /// exactly the health it had before it ran out of credit, rather than + /// making it re-warm. + pub fn fund_lane(&mut self, lane: usize) { + if let Some(l) = self.lanes.get_mut(lane) + && l.health == LaneHealth::Unfunded + { + l.health = if l.bytes_total > 0 { + LaneHealth::Live + } else { + LaneHealth::Warming + }; + } + } + + /// Lanes currently paused for payment, so the driver knows which ones a + /// cheque would unblock. + pub fn unfunded_lanes(&self) -> Vec { + self.lanes + .iter() + .enumerate() + .filter(|(_, l)| l.health == LaneHealth::Unfunded) + .map(|(i, _)| i) + .collect() + } + /// Record the HTTP-level result of a dispatch. Must be called exactly /// once per [`Assignment`], after all of that batch's acks. pub fn on_batch_result(&mut self, batch: u64, outcome: BatchOutcome, now_ms: u64) { @@ -746,6 +799,13 @@ impl Scheduler { l.backoff_exp = 0; l.health = LaneHealth::Live; } + // A 402 says the relay is willing and the client is behind on + // payment. Pause the lane without touching `fail_streak` or + // `backoff_exp`, so settling restores it instantly and a long + // upload is not punished for crossing a settlement window. + BatchOutcome::PaymentRequired => { + self.lanes[lane].health = LaneHealth::Unfunded; + } _ => { let l = &mut self.lanes[lane]; l.fail_streak += 1; @@ -962,6 +1022,7 @@ impl Scheduler { LaneHealth::Warming => LaneHealthKind::Warming, LaneHealth::Live => LaneHealthKind::Live, LaneHealth::Backoff { .. } => LaneHealthKind::Backoff, + LaneHealth::Unfunded => LaneHealthKind::Unfunded, LaneHealth::Retired => LaneHealthKind::Retired, }), }) diff --git a/src/pushsched/tests.rs b/src/pushsched/tests.rs index 7c9bb76..7796445 100644 --- a/src/pushsched/tests.rs +++ b/src/pushsched/tests.rs @@ -643,3 +643,101 @@ fn all_acked_policy_ignores_groups() { assert_eq!(sched.acked(), 10, "AllAcked must not stop at the threshold"); assert_eq!(sched.skipped(), 0); } + +// ── Metered lanes (docs/pusher-incentives.md §12) ──────────────────────── + +fn two_lanes() -> Scheduler { + let infos = vec![LaneInfo::default(), LaneInfo::default()]; + let mut s = Scheduler::new(infos, Config::default()); + s.admit(addrs(64, 7)); + s +} + +/// Drive one batch and hand back `(batch, lane)`. The scheduler chooses the +/// lane by weight, so tests follow its choice rather than dictating one. +fn dispatch_one(s: &mut Scheduler, now_ms: u64) -> (u64, usize) { + let a = s + .next(now_ms) + .expect("a lane with pending work must produce an assignment"); + (a.batch, a.lane) +} + +/// The failure §12 exists to prevent: five routine settlements retiring a +/// perfectly healthy lane mid-upload. +#[test] +fn repeated_402s_never_retire_a_lane() { + let mut s = two_lanes(); + let mut now = 0u64; + for _ in 0..20 { + let (b, lane) = dispatch_one(&mut s, now); + s.on_batch_result(b, BatchOutcome::PaymentRequired, now); + // Settling is what un-pauses it; without that the lane stays out. + s.fund_lane(lane); + now += 1000; + } + for (i, st) in s.lane_stats().iter().enumerate() { + assert_ne!( + st.health.expect("health"), + LaneHealthKind::Retired, + "lane {i} kept asking to be paid and must never be retired" + ); + } +} + +/// A 402 must not touch the failure streak — otherwise a lane that has +/// been asking for payment is one transport error away from backoff. +#[test] +fn a_402_does_not_charge_lane_health() { + let mut s = two_lanes(); + let mut victim = None; + for i in 0..10 { + let (b, lane) = dispatch_one(&mut s, 100 + i * 10); + // Charge every 402 to one lane, so any accumulation would show. + if victim.is_none() { + victim = Some(lane); + } + s.on_batch_result(b, BatchOutcome::PaymentRequired, 100 + i * 10); + s.fund_lane(lane); + } + // A single real failure now must still be just one failure, not the + // straw that tips an already-charged streak into backoff. + let (b, lane) = dispatch_one(&mut s, 500); + s.on_batch_result(b, BatchOutcome::Failed("boom".into()), 500); + let health = s.lane_stats()[lane].health.expect("health"); + assert!( + matches!(health, LaneHealthKind::Live | LaneHealthKind::Warming), + "one failure after many 402s must not have compounded: {health:?}" + ); +} + +/// `Unfunded` is ineligible but recoverable — unlike `Retired`, which is +/// permanent for the run. +#[test] +fn an_unfunded_lane_is_paused_then_restored_by_paying() { + let mut s = two_lanes(); + let (b, lane) = dispatch_one(&mut s, 0); + s.on_batch_result(b, BatchOutcome::PaymentRequired, 0); + + assert_eq!(s.unfunded_lanes(), vec![lane], "the driver must see what to pay"); + for _ in 0..8 { + assert_ne!( + s.next(20).map(|a| a.lane), + Some(lane), + "an unfunded lane must not be dispatched to" + ); + } + s.fund_lane(lane); + assert!(s.unfunded_lanes().is_empty(), "paying clears the pause"); + assert!(s.next(30).is_some(), "and the run continues"); +} + +/// Work must not be stranded on a paused lane: chunks go back to pending +/// and another lane picks them up while the cheque is in flight. +#[test] +fn work_on_an_unfunded_lane_fails_over_rather_than_stalling() { + let mut s = two_lanes(); + let (b, lane) = dispatch_one(&mut s, 0); + s.on_batch_result(b, BatchOutcome::PaymentRequired, 0); + let other = s.next(10).expect("work must not strand on a paused lane"); + assert_ne!(other.lane, lane, "it fails over to the funded lane"); +} diff --git a/src/signer.rs b/src/signer.rs index d80c595..2e43b8e 100644 --- a/src/signer.rs +++ b/src/signer.rs @@ -453,6 +453,16 @@ pub fn recover_eth_address_from_handshake_v15( recover_eth_from_eip191(&payload, signature) } +/// Public wrapper over [`recover_eth_from_eip191`], for verifying the +/// relay's signed price quote client-side (`docs/pusher-incentives.md` +/// §7.3). An unsigned price is repudiable in both directions. +pub fn recover_eth_address_from_eip191( + payload: &[u8], + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + recover_eth_from_eip191(payload, signature) +} + /// Recover the 20-byte Ethereum address from a 65-byte (r || s || v) /// EIP-191 signature over `payload`. `v` is expected in Ethereum form /// (27 or 28); k256 normalises to 0/1 internally. @@ -498,3 +508,315 @@ fn recover_eth_from_eip191(payload: &[u8], signature: &[u8]) -> Result<[u8; 20], addr.copy_from_slice(&hash[12..]); Ok(addr) } + +// ────────────────────────────────────────────────────────────────────── +// Metered relay (docs/pusher-incentives.md Stage 1) +// ────────────────────────────────────────────────────────────────────── + +/// secp256k1n ÷ 2. `ERC20SimpleSwap.recoverEIP712` calls `ECDSA.recover` +/// from `@openzeppelin/contracts/cryptography/ECDSA.sol` at `^3.4.1`, which +/// requires `uint256(s) <= this` and `v ∈ {27, 28}` — so a signature outside +/// that range is *off-chain valid and on-chain uncashable*. See incentives +/// §11.6; rejecting it at the boundary is what stops a client buying service +/// with a cheque that can never be redeemed. +const SECP256K1N_HALF: [u8; 32] = [ + 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x5D, 0x57, 0x6E, 0x73, 0x57, 0xA4, 0x50, 0x1D, 0xDF, 0xE9, 0x2F, 0x46, 0x68, 0x1B, 0x20, 0xA0, +]; + +/// Reject a signature the deployed chequebook would refuse to honour. +/// +/// This is a *validity* check, not a malleability nicety. `alloy` recovers +/// high-`s` and `v ∈ {0,1}` happily, so without this the relay accepts +/// cheques it can never cash. +pub fn check_canonical_signature(sig: &[u8]) -> Result<(), SignerError> { + if sig.len() != 65 { + return Err(SignerError::Alloy(format!( + "signature must be 65 bytes, got {}", + sig.len() + ))); + } + if sig[64] != 27 && sig[64] != 28 { + return Err(SignerError::Alloy(format!( + "non-canonical v={}: ERC20SimpleSwap requires 27 or 28, so this \ + cheque would revert at cashout", + sig[64] + ))); + } + // Both are big-endian 32-byte magnitudes, so a lexicographic slice + // compare is a numeric compare. + if sig[32..64] > SECP256K1N_HALF[..] { + return Err(SignerError::Alloy( + "non-canonical high-s signature: ERC20SimpleSwap rejects \ + s > secp256k1n/2, so this cheque would revert at cashout" + .into(), + )); + } + Ok(()) +} + +/// The EIP-712 domain bee uses for cheques. Extracted so signing and +/// recovery cannot drift: a mismatched domain silently recovers a +/// *different* address rather than failing, which would look like "the +/// client signed with the wrong key". +pub fn chequebook_domain(chain_id: u64) -> Eip712Domain { + Eip712Domain { + name: Some("Chequebook".into()), + version: Some("1.0".into()), + chain_id: Some(alloy_primitives::U256::from(chain_id)), + verifying_contract: None, + salt: None, + } +} + +/// Domain for the relay's own admission challenge (incentives §7.2). +/// +/// Separate from `Chequebook` on purpose: the account key already signs +/// postage stamps (EIP-191 over raw stamp bytes) and cheques (EIP-712, +/// `Chequebook`). A third scheme over the same key without its own domain +/// invites cross-scheme confusion, where a signature gathered for one +/// purpose is replayable as another. +pub fn pusher_domain(chain_id: u64) -> Eip712Domain { + Eip712Domain { + name: Some("HoverflyPusher".into()), + version: Some("1".into()), + chain_id: Some(alloy_primitives::U256::from(chain_id)), + verifying_contract: None, + salt: None, + } +} + +sol! { + /// Signed by the *client* to prove it holds the account key the relay + /// issued a challenge to. `origin` is the host the client dialled and is + /// what stops a signature gathered at relay A being replayed at relay B + /// (incentives §11.1) — but only if the relay compares it against its own + /// configured hostname rather than a request header. + struct PushChallenge { + bytes32 nonce; + string origin; + address account; + bytes32 batchId; + uint256 expiry; + } +} + +/// Recover the issuer of an EIP-712 cheque. +/// +/// Mirrors bee's `RecoverCheque` (`chequestore.go:190`). Rejects +/// non-canonical signatures first — a cheque the chain would refuse is not +/// a cheque, however well it recovers locally. +pub fn recover_cheque_issuer( + chequebook: &[u8; 20], + beneficiary: &[u8; 20], + cumulative_payout: alloy_primitives::U256, + chain_id: u64, + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + check_canonical_signature(signature)?; + let cheque = Cheque { + chequebook: alloy_primitives::Address::from(*chequebook), + beneficiary: alloy_primitives::Address::from(*beneficiary), + cumulativePayout: cumulative_payout, + }; + recover_typed(&cheque, &chequebook_domain(chain_id), signature) +} + +/// Recover the account that signed a `PushChallenge`. +pub fn recover_push_challenge( + challenge: &PushChallenge, + chain_id: u64, + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + // The challenge is never cashed, so canonical form is not a *validity* + // requirement here — but accepting only one encoding per signature keeps + // the replay surface a single value rather than four. + check_canonical_signature(signature)?; + recover_typed(challenge, &pusher_domain(chain_id), signature) +} + +fn recover_typed( + value: &T, + domain: &Eip712Domain, + signature: &[u8], +) -> Result<[u8; 20], SignerError> { + use k256::ecdsa::{RecoveryId, Signature as K256Sig, VerifyingKey}; + let digest = value.eip712_signing_hash(domain); + let v = signature[64] - 27; + let k_sig = K256Sig::from_slice(&signature[..64]) + .map_err(|e| SignerError::Alloy(format!("k256 sig: {e}")))?; + let rec_id = + RecoveryId::try_from(v).map_err(|e| SignerError::Alloy(format!("recovery id: {e}")))?; + let vk = VerifyingKey::recover_from_prehash(digest.as_slice(), &k_sig, rec_id) + .map_err(|e| SignerError::Alloy(format!("recover: {e}")))?; + let point = vk.to_encoded_point(false); + let hash: [u8; 32] = Keccak256::digest(&point.as_bytes()[1..]).into(); + let mut addr = [0u8; 20]; + addr.copy_from_slice(&hash[12..]); + Ok(addr) +} + +impl SwarmSigner { + /// Sign a `PushChallenge` (incentives §7.2). Client side. + pub fn sign_push_challenge( + &self, + challenge: &PushChallenge, + chain_id: u64, + ) -> Result<[u8; 65], SignerError> { + let sig = self + .inner + .sign_typed_data_sync(challenge, &pusher_domain(chain_id)) + .map_err(|e| SignerError::Alloy(e.to_string()))?; + let mut bytes = sig.as_bytes(); + if bytes[64] < 27 { + bytes[64] += 27; + } + Ok(bytes) + } +} + +#[cfg(test)] +mod metered_tests { + use super::*; + + const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; + + fn signer() -> SwarmSigner { + SwarmSigner::from_hex_with_nonce(KEY, &format!("0x{}", hex::encode([0u8; 32])), 1) + .expect("valid key") + } + + #[test] + fn a_cheque_recovers_to_the_key_that_signed_it() { + let s = signer(); + let cb = [0x11u8; 20]; + let bn = [0x22u8; 20]; + let amount = alloy_primitives::U256::from(1_000_000u64); + let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + let got = recover_cheque_issuer(&cb, &bn, amount, 100, &sig).expect("recover"); + assert_eq!(got, *s.eth_address(), "issuer must be the signing key"); + } + + /// The domain separator carries the chain id, so a cheque signed for + /// Gnosis must not validate against a relay pinning Sepolia — otherwise + /// one signature pays on every chain at once. + #[test] + fn a_cheque_does_not_recover_across_chains() { + let s = signer(); + let (cb, bn) = ([0x11u8; 20], [0x22u8; 20]); + let amount = alloy_primitives::U256::from(7u64); + let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + let other = recover_cheque_issuer(&cb, &bn, amount, 11155111, &sig).expect("recovers"); + assert_ne!(other, *s.eth_address(), "wrong chain must not recover the issuer"); + } + + /// Changing any signed field must move the recovered address, or the + /// relay could be paid with a cheque made out to someone else. + #[test] + fn a_cheque_binds_every_field() { + let s = signer(); + let (cb, bn) = ([0x11u8; 20], [0x22u8; 20]); + let amount = alloy_primitives::U256::from(500u64); + let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + let me = *s.eth_address(); + assert_ne!( + recover_cheque_issuer(&cb, &[0x33u8; 20], amount, 100, &sig).expect("rec"), + me, + "beneficiary is bound" + ); + assert_ne!( + recover_cheque_issuer(&[0x44u8; 20], &bn, amount, 100, &sig).expect("rec"), + me, + "chequebook is bound" + ); + assert_ne!( + recover_cheque_issuer(&cb, &bn, alloy_primitives::U256::from(501u64), 100, &sig) + .expect("rec"), + me, + "cumulative payout is bound" + ); + } + + #[test] + fn a_push_challenge_recovers_to_the_account() { + let s = signer(); + let c = PushChallenge { + nonce: alloy_primitives::B256::from([9u8; 32]), + origin: "relay-a.example".into(), + account: alloy_primitives::Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from([5u8; 32]), + expiry: alloy_primitives::U256::from(1_700_000_000u64), + }; + let sig = s.sign_push_challenge(&c, 100).expect("sign"); + assert_eq!( + recover_push_challenge(&c, 100, &sig).expect("recover"), + *s.eth_address() + ); + } + + /// The whole point of binding `origin`: a signature gathered at relay A + /// must not verify at relay B (§11.1). + #[test] + fn a_push_challenge_binds_the_origin() { + let s = signer(); + let mut c = PushChallenge { + nonce: alloy_primitives::B256::from([9u8; 32]), + origin: "relay-a.example".into(), + account: alloy_primitives::Address::from(*s.eth_address()), + batchId: alloy_primitives::B256::from([5u8; 32]), + expiry: alloy_primitives::U256::from(1_700_000_000u64), + }; + let sig = s.sign_push_challenge(&c, 100).expect("sign"); + c.origin = "relay-b.example".into(); + assert_ne!( + recover_push_challenge(&c, 100, &sig).expect("recovers something"), + *s.eth_address(), + "a challenge signed for relay A must not recover the account at relay B" + ); + } + + /// A cheque and a challenge signed over structurally similar data must + /// live in different domains, or one could be replayed as the other. + #[test] + fn cheque_and_challenge_domains_are_distinct() { + assert_ne!( + chequebook_domain(100).separator(), + pusher_domain(100).separator() + ); + } + + #[test] + fn non_canonical_signatures_are_rejected() { + let s = signer(); + let (cb, bn) = ([0x11u8; 20], [0x22u8; 20]); + let amount = alloy_primitives::U256::from(1u64); + let good = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); + check_canonical_signature(&good).expect("alloy always produces canonical"); + + let mut bad_v = good; + bad_v[64] = 1; // the 0/1 encoding alloy accepts and the contract does not + let e = check_canonical_signature(&bad_v).expect_err("v=1 must be rejected"); + assert!(format!("{e}").contains("non-canonical v"), "got: {e}"); + + let mut high_s = good; + high_s[32] = 0xFF; // s > secp256k1n/2 + let e = check_canonical_signature(&high_s).expect_err("high-s must be rejected"); + assert!(format!("{e}").contains("high-s"), "got: {e}"); + + for len in [0usize, 64, 66] { + check_canonical_signature(&vec![1u8; len]).expect_err("bad length must be rejected"); + } + } + + /// Exactly the boundary OpenZeppelin enforces: `s <= n/2` passes, + /// `s = n/2 + 1` does not. + #[test] + fn the_high_s_boundary_matches_openzeppelin() { + let mut sig = [0u8; 65]; + sig[64] = 27; + sig[32..64].copy_from_slice(&SECP256K1N_HALF); + check_canonical_signature(&sig).expect("s == n/2 is allowed"); + sig[63] += 1; + check_canonical_signature(&sig).expect_err("s == n/2 + 1 is not"); + } +} From 634d4c82ee50597d5a08103d81c3a57c1ef3ab4e Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 7 Aug 2026 14:24:47 +0300 Subject: [PATCH 02/27] feat(client): wire the metered payment loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connects the pieces Stage 1 already had but never joined: on a 402 — or once the lane's own settlement window is crossed — the driver fetches a challenge, signs it, POSTs a cumulative cheque to /v1/pay, and calls fund_lane to restore the lane to the health it had before it ran out of credit. Paying on the window is what stops an upload reaching its cap at all; paying on 402 is the recovery path when it does. The client bills itself from bytes it *sent*, by the same arithmetic the relay uses, so the two sides agree without exchanging anything and a disagreement is visible immediately rather than adjudicated. LaneInfo now carries the whole verified quote rather than just the price, so the payment loop has the beneficiary and parameters without re-fetching and re-verifying /v1/status. Also records a design decision in §14: chequebook deployment is an explicit user command, never an automatic side effect of an upload. Still not exercised end to end — the /v1/pay chain reads have never touched a real chequebook. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 11 +++ src/client.rs | 102 +++++++++++++++++++-- src/payer.rs | 180 ++++++++++++++++++++++++++++++++++++++ src/protocols/swap.rs | 12 +++ src/pushsched.rs | 4 + 5 files changed, 304 insertions(+), 5 deletions(-) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 7fcee0f..6353640 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1250,6 +1250,17 @@ tracking computed from bytes sent. `BatchOutcome::PaymentRequired` and a non-terminal `LaneHealth::Unfunded` are in `src/pushsched.rs`, and `src/cheques.rs` gained `total_issued` plus a `relay:` key namespace. +**Chequebook deployment is an explicit user action, never automatic.** It +belongs in its own command (`hoverfly chequebook deploy`, alongside +`batch create`), not on the upload path. Deploying a contract is +irreversible and spends real funds, so an upload that silently deployed one +because some lane quoted a price would fire on the *first* metered lane a +user ever met — before they had decided they wanted to pay at all. This is +the same shape §14 Stage 3 gives the browser, where the wallet deploys the +chequebook as a deliberate opt-in. hoverfly has no deploy path today: it +consumes a chequebook via `--chequebook` and documents "already deployed by +bee's official factory" as a precondition. + Two things remain before hard mode: the driver does not yet *act* on a 402 by issuing a cheque and calling `Scheduler::fund_lane` (the pieces exist and are tested; the loop that connects them does not), and the `/v1/pay` diff --git a/src/client.rs b/src/client.rs index 6e8d6a3..f659e89 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2668,6 +2668,7 @@ pub async fn push_via_pushers( total, move |_| Ok(once.take().unwrap_or_default()), progress, + None, ) .await } @@ -2684,6 +2685,10 @@ async fn drive_pushers

( total_hint: usize, mut produce: P, progress: Option<&ProgressFn>, + // Metered lanes only (`docs/pusher-incentives.md` §12). `None` means + // pay nothing, which is correct for every `open` lane and is what all + // callers pass until a lane actually advertises a price. + payment: Option<&crate::payer::PaymentConfig>, ) -> Result<(), ClientError> where P: FnMut(usize) -> Result, ClientError>, @@ -2739,6 +2744,27 @@ where .iter() .map(|u| std::sync::Arc::new(format!("{}/v1/push", u.trim_end_matches('/')))) .collect(); + // One payer per metered lane. Lanes that quote no price get none, so an + // `open` lane costs nothing and needs no special case below. + let mut payers: Vec> = Vec::with_capacity(pusher_urls.len()); + for (i, url) in pusher_urls.iter().enumerate() { + let quote = payment.and_then(|_| infos.get(i).and_then(|inf: &LaneInfo| inf.quote.clone())); + payers.push(match (payment, quote) { + (Some(pc), Some(q)) => { + let key = crate::cheques::relay_key(&q.beneficiary); + let cumulative = pc + .cheques + .lock() + .expect("cheque store poisoned") + .cumulative(&key); + info!(target: "hoverfly::upload", + "lane {i} is metered at {} PLUR/KiB; resuming cumulative {cumulative}", + q.params.price_plur_per_kib); + Some(crate::payer::LanePayer::new(url.clone(), q, cumulative)) + } + _ => None, + }); + } let mut sched = Scheduler::new(infos, cfg); // Frames are held by address, not by index: `admit` de-duplicates, so an // index-parallel Vec would silently skew. Keying by address also lets a @@ -2812,8 +2838,41 @@ where debug!(target: "hoverfly::upload", "hedging {} straggler(s) onto lane {lane}", batch.len()); } + // Metered lane: attach the capability, and bill ourselves for + // the body by the same arithmetic the relay uses (§8). We count + // what we *send*; nothing the relay reports enters this number, + // which is why the two sides can agree without exchanging + // anything. + let mut challenge: Option = None; + if let (Some(pc), Some(payer)) = (payment, payers[lane].as_mut()) { + match payer.header(&http, pc).await { + Ok(h) => challenge = Some(h.to_string()), + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {lane}: cannot obtain a challenge ({e}); pausing it"); + sched.on_batch_result( + batch_id, + crate::pushsched::BatchOutcome::PaymentRequired, + now_ms(), + ); + continue; + } + } + let body_bytes: u64 = + batch.iter().map(|c| (crate::pushframe::HEADER_LEN + c.wire.len()) as u64).sum(); + payer.account.record_sent(body_bytes); + } tokio::spawn(async move { - post_batch_streaming(&http, url.as_str(), batch_id, lane, &batch, &tx, None).await; + post_batch_streaming( + &http, + url.as_str(), + batch_id, + lane, + &batch, + &tx, + challenge.as_deref(), + ) + .await; }); } @@ -2909,7 +2968,38 @@ where outcome, } => { sched.on_batch_timing(lane, acked, elapsed_ms); + let needs_payment = + matches!(outcome, crate::pushsched::BatchOutcome::PaymentRequired); sched.on_batch_result(batch, outcome, now_ms()); + // Settle when the relay asks (402) or when we have crossed + // the lane's own settlement window. Paying on the window + // keeps an upload from ever reaching its cap in the first + // place; paying on 402 is the recovery path when it does. + if let (Some(pc), Some(payer)) = (payment, payers[lane].as_mut()) + && (needs_payment || payer.account.should_settle()) + { + match payer.settle(&http, pc).await { + Ok(Some(c)) => { + info!(target: "hoverfly::upload", + "lane {lane}: paid, cumulative now {c}"); + // Restores the lane to the health it had before + // it ran out of credit — a 402 never charged the + // failure streak, so this is not a re-warm. + sched.fund_lane(lane); + } + Ok(None) if needs_payment => { + // 402 with nothing owed above the dust floor + // means the two sides disagree about the + // ledger. Retrying cannot fix that. + warn!(target: "hoverfly::upload", + "lane {lane}: 402 but nothing is owed — ledger disagreement"); + } + Ok(None) => {} + Err(e) => { + warn!(target: "hoverfly::upload", "lane {lane}: payment failed: {e}"); + } + } + } } } } @@ -2997,6 +3087,7 @@ pub async fn push_stream_via_pushers( total, move |want| streamer.next_batch(want), progress, + None, ) .await?; Ok(root) @@ -3184,7 +3275,7 @@ async fn fetch_lane_info( // unsigned or unverifiable price is repudiable in both directions, so a // quote that does not check out leaves the lane looking unmetered rather // than looking free. - let (price_plur_per_kib, hard_enforcement) = match v.get("payment") { + let (price_plur_per_kib, hard_enforcement, quote) = match v.get("payment") { Some(pay) if !pay.is_null() => { // The overlay cross-check is skipped here: it needs the // network id, which this driver does not carry, and it is the @@ -3194,20 +3285,21 @@ async fn fetch_lane_info( // field from an assertion into a check. A caller that pins // does the full version. match crate::payer::PaymentQuote::verify(pay, None, 0, None, price_ceiling) { - Ok(q) => (Some(q.params.price_plur_per_kib), q.hard_enforcement), + Ok(q) => (Some(q.params.price_plur_per_kib), q.hard_enforcement, Some(q)), Err(e) => { warn!(target: "hoverfly::upload", "lane {base_url}: payment quote rejected ({e}); treating as unmetered"); - (None, false) + (None, false, None) } } } - _ => (None, false), + _ => (None, false, None), }; LaneInfo { overlay, price_plur_per_kib, hard_enforcement, + quote, batch_max: v .get("batch_max") .and_then(|x| x.as_u64()) diff --git a/src/payer.rs b/src/payer.rs index 7c87ac4..94ba657 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -411,6 +411,186 @@ impl TotalIssued { } } +// ────────────────────────────────────────────────────────────────────── +// The payment loop (docs/pusher-incentives.md §12) +// ────────────────────────────────────────────────────────────────────── + +/// Everything the client needs to pay a metered lane. +/// +/// The account key is the **batch owner's** signer — the same key that +/// stamps chunks (§6) — so a metered upload needs no extra credential and, +/// in a browser, no wallet prompt. +#[cfg(not(target_arch = "wasm32"))] +pub struct PaymentConfig { + pub signer: crate::signer::SwarmSigner, + pub batch: [u8; 32], + pub chequebook: [u8; 20], + pub chain_id: u64, + /// Shared across lanes: N beneficiaries are N claims on **one** balance + /// (§8.3), so the cumulative store has to be common. + pub cheques: std::sync::Arc>, + /// On-chain liquid balance of the chequebook, read once at startup. A + /// cheque that would push total issuance past this is not signed — it + /// would be accepted and then fail at cashout, which looks like the + /// relay's fault and costs the lane's trust rather than ours. + pub balance_plur: u128, +} + +/// Per-lane payment state: the verified quote, a cached capability, and the +/// running total. +#[cfg(not(target_arch = "wasm32"))] +pub struct LanePayer { + pub base_url: String, + pub quote: PaymentQuote, + pub account: LaneAccount, + header: Option, + header_stale_after: u64, + cap_plur: u128, +} + +#[cfg(not(target_arch = "wasm32"))] +impl LanePayer { + pub fn new(base_url: String, quote: PaymentQuote, cumulative: u128) -> Self { + let account = LaneAccount::new(quote.params, quote.beneficiary).with_cumulative(cumulative); + Self { + base_url, + quote, + account, + header: None, + header_stale_after: 0, + cap_plur: 0, + } + } + + /// The credit line the relay last told us about, or 0 before the first + /// challenge. Used to size POSTs (§7.2). + pub fn cap_plur(&self) -> u128 { + self.cap_plur + } + + /// A valid challenge header, fetching and signing one if needed. + /// + /// Re-fetched 30 s before expiry rather than on failure: racing the + /// expiry with a POST already in flight turns a cheap GET into a + /// mid-upload 401. + pub async fn header( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result<&str, String> { + let now = crate::challenge::now_unix(); + if self.header.is_none() || now >= self.header_stale_after { + let account = *cfg.signer.eth_address(); + let url = format!( + "{}/v1/challenge?account=0x{}&batch=0x{}", + self.base_url.trim_end_matches('/'), + hex::encode(account), + hex::encode(cfg.batch), + ); + let resp = http + .get(&url) + .timeout(std::time::Duration::from_secs(60)) + .send() + .await + .map_err(|e| format!("challenge fetch: {e}"))?; + if !resp.status().is_success() { + let code = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(format!("challenge {code}: {}", body.trim())); + } + let v: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("challenge json: {e}"))?; + let offered = OfferedChallenge::parse(&v)?; + self.cap_plur = offered.cap_plur; + self.header_stale_after = offered.stale_after(); + self.header = Some(offered.sign(&cfg.signer, self.quote.chain_id)?); + } + Ok(self.header.as_deref().unwrap_or_default()) + } + + /// Largest POST body this lane will currently admit. + pub fn max_body_bytes(&self) -> u64 { + if self.cap_plur == 0 { + return u64::MAX; + } + self.account.max_body_bytes(self.cap_plur) + } + + /// Settle if there is enough owed to be worth a cheque. + /// + /// Returns the amount accepted, or `None` when nothing was owed above + /// the lane's dust floor. Errors are the caller's cue to stop using the + /// lane, not to retry blindly — a rejected cheque usually means the two + /// sides disagree about the cumulative, which retrying cannot fix. + pub async fn settle( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result, String> { + let Some(cumulative) = self.account.next_cumulative() else { + return Ok(None); + }; + // Aggregate exposure across every beneficiary drawn on this one + // chequebook (§8.3): the second lane's cheque is what silently + // bounces without this. + let key = crate::cheques::relay_key(&self.quote.beneficiary); + { + let store = cfg.cheques.lock().expect("cheque store poisoned"); + if store.would_exceed_balance(&key, cumulative, cfg.balance_plur) { + return Err(format!( + "cheque for {cumulative} would push total issuance past the chequebook's \ + {} balance across all lanes", + cfg.balance_plur + )); + } + } + let sig = cfg + .signer + .sign_cheque( + &cfg.chequebook, + &self.quote.beneficiary, + alloy_primitives::U256::from(cumulative), + self.quote.chain_id, + ) + .map_err(|e| format!("sign cheque: {e}"))?; + let body = crate::protocols::swap::encode_signed_cheque_json_pub( + &cfg.chequebook, + &self.quote.beneficiary, + alloy_primitives::U256::from(cumulative), + &sig, + ); + let header = self.header(http, cfg).await?.to_string(); + let resp = http + .post(format!("{}/v1/pay", self.base_url.trim_end_matches('/'))) + .header(crate::metered::CHALLENGE_HEADER, header) + .header("content-type", "application/json") + .body(body) + .timeout(std::time::Duration::from_secs(120)) + .send() + .await + .map_err(|e| format!("pay: {e}"))?; + if !resp.status().is_success() { + let code = resp.status(); + let text = resp.text().await.unwrap_or_default(); + return Err(format!("pay {code}: {}", text.trim())); + } + // Record the cumulative *before* trusting the reply: we have + // certainly issued it, and under-recording is what causes the next + // cheque to be rejected as non-increasing. + { + let mut store = cfg.cheques.lock().expect("cheque store poisoned"); + store + .set_cumulative(&key, cumulative) + .map_err(|e| format!("cheque store: {e}"))?; + let _ = store.save(); + } + self.account.settled(cumulative); + Ok(Some(cumulative)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/protocols/swap.rs b/src/protocols/swap.rs index ec56b92..f0bdd57 100644 --- a/src/protocols/swap.rs +++ b/src/protocols/swap.rs @@ -105,6 +105,18 @@ fn encode_signed_cheque_json( .into_bytes() } +/// Public wrapper: the client needs to produce this body for a metered +/// relay's `POST /v1/pay`, and it must be byte-identical to what bee's +/// `json.Marshal` emits (see the encoder's own note on `*big.Int`). +pub fn encode_signed_cheque_json_pub( + chequebook: &[u8; 20], + beneficiary: &[u8; 20], + cumulative_payout: U256, + signature: &[u8; 65], +) -> Vec { + encode_signed_cheque_json(chequebook, beneficiary, cumulative_payout, signature) +} + /// Outbound `Handshake { Beneficiary }`. Called once per session by /// the connection-setup path. Caller exchanges empty headers first. pub async fn send_handshake(stream: &mut S, beneficiary: &[u8; 20]) -> Result<(), SwapError> diff --git a/src/pushsched.rs b/src/pushsched.rs index dd13609..63f0c92 100644 --- a/src/pushsched.rs +++ b/src/pushsched.rs @@ -121,6 +121,10 @@ pub struct LaneInfo { pub price_plur_per_kib: Option, /// True when the lane enforces 402 rather than metering softly. pub hard_enforcement: bool, + /// The whole verified quote, when this lane advertised one. Carried so + /// the payment loop has the beneficiary and parameters without + /// re-fetching and re-verifying `/v1/status`. + pub quote: Option, } /// Tunables. Defaults are the shipping configuration. From 1d769874056dc04cb09487249d86a8f8be23f198 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 7 Aug 2026 14:29:56 +0300 Subject: [PATCH 03/27] fix(pusher): bound the /v1/pay body read; make the reservation an RAII guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from review, both real and both verified against the code. **Unbounded body read on /v1/pay.** `read_body_limited` enforced its cap via a `size_hint().upper()` pre-check plus a post-`collect()` compare. But `Incoming::size_hint().upper()` is `None` for a chunked body — length genuinely isn't known until the terminating chunk — so the pre-check short-circuits to *skipping*, and `.collect()` then accumulates whatever the client streams until the 30s timeout, checking the size only after the memory is already committed. An authenticated caller (one live batch buys a challenge) could stream ~30s of link bandwidth per connection across the 256-connection cap. Now uses `Limited::new`, matching /v1/push, which enforces per-frame inside `poll_frame`. **Reservations leaked on four early returns.** `admit_metered` reserves before the body is read; oversize body, read timeout, frame-decode failure and empty batch all return without reaching `run_push`, the only caller of `commit` — and `commit`/`release` are the only things that lower `reserved_plur`. Paying doesn't help: `credit` reduces `owed`, never `reserved`. So a leak was permanent until restart and it ratcheted: under hard mode the account ends up above its cap with no cheque able to clear it (§10.1's no-exit failure), and under soft mode leaks accumulate against MAX_LIVE_RESERVATIONS until real clients are shed. Fixed with RAII rather than four `release` calls, so exit paths added later are covered by construction — which is precisely the bug that occurred. `Admitted` now holds the state and releases on drop unless `commit` consumed it. Four ledger-level regression tests pin the invariants the guard relies on, including that paying a cheque does not release a reservation. Co-Authored-By: Claude Opus 5 (1M context) --- src/ledger.rs | 76 +++++++++++++++++++ src/pusher.rs | 205 ++++++++++++++++++++++++++++++++------------------ 2 files changed, 208 insertions(+), 73 deletions(-) diff --git a/src/ledger.rs b/src/ledger.rs index 0130014..e637a89 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -554,3 +554,79 @@ mod tests { assert_eq!(l.live_reservations(), 1); } } + +#[cfg(test)] +mod leak_tests { + //! A reservation that is never committed must never survive. + //! + //! Found by review: four early-return paths in `push_response` (oversize + //! body, read timeout, frame-decode failure, empty batch) dropped the + //! admission without releasing, and nothing else lowers `reserved_plur` + //! — `credit` only touches `owed`. These pin the ledger-level invariants + //! the RAII guard in `pusher.rs` relies on. + use super::*; + + const A: [u8; 20] = [1u8; 20]; + const CB: [u8; 20] = [9u8; 20]; + + /// Paying does **not** clear a reservation. This is the property that + /// turns a leak into a permanent one, so it is worth stating outright. + #[test] + fn paying_a_cheque_does_not_release_a_reservation() { + let mut l = Ledger::ephemeral(); + l.commit(A, 0, 1000); + l.reserve(A, 500, 100_000); + l.credit(A, CB, 1000).expect("pay off the debt"); + assert_eq!(l.owed(&A), 0, "the debt is cleared"); + assert_eq!( + l.reserved(&A), + 500, + "but the reservation is untouched — only commit or release move it" + ); + } + + /// The ratchet: leaked reservations accumulate until `outstanding` + /// exceeds the cap, and no cheque can bring it back down. + #[test] + fn leaked_reservations_ratchet_an_account_past_its_cap_with_no_way_back() { + let mut l = Ledger::ephemeral(); + let cap = 10_000u128; + for _ in 0..20 { + l.reserve(A, 1000, cap); // admitted, then dropped without commit + } + assert!(l.outstanding(&A) > cap, "the account is now over its cap"); + // There is no debt to pay, so no cheque exists that could help. + assert_eq!(l.owed(&A), 0); + assert!( + matches!(l.credit(A, CB, 1), Err(LedgerError::Overpayment { .. })), + "with nothing owed, a cheque cannot clear the overshoot" + ); + // Only releasing does. + for _ in 0..20 { + l.release(A, 1000); + } + assert_eq!(l.outstanding(&A), 0); + } + + /// Releasing an unused reservation must leave no residue at all — this + /// is what every early-return path now does via `Drop`. + #[test] + fn releasing_an_unused_reservation_leaves_nothing_behind() { + let mut l = Ledger::ephemeral(); + let adm = l.reserve(A, 4096, 100_000); + l.release(A, adm.reserved_plur); + assert_eq!(l.outstanding(&A), 0); + assert_eq!(l.live_reservations(), 0, "and frees its shed-cap slot"); + } + + /// Committing zero bytes is equivalent to releasing: a POST that + /// admitted nothing owes nothing and holds nothing. + #[test] + fn committing_nothing_is_equivalent_to_releasing() { + let mut l = Ledger::ephemeral(); + let adm = l.reserve(A, 4096, 100_000); + l.commit(A, adm.reserved_plur, 0); + assert_eq!(l.outstanding(&A), 0); + assert_eq!(l.live_reservations(), 0); + } +} diff --git a/src/pusher.rs b/src/pusher.rs index c074777..34fc757 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -526,10 +526,64 @@ async fn handle( /// What admission decided, carried into the push task so completion can /// convert the reservation into debt. `None` in open mode. +/// +/// **This is an RAII guard, and it has to be.** A reservation is placed +/// before the body is read, but half a dozen things between there and the +/// push can fail — an oversize body, a read timeout, a frame-decode error, +/// an empty batch. Every one of those returns without ever reaching +/// `run_push`, which is the only place `commit` runs, and `commit` is the +/// only thing besides `release` that lowers `reserved_plur`. Paying a +/// cheque does not help: `Ledger::credit` reduces `owed`, never `reserved`. +/// +/// So a leaked reservation is permanent until restart, and it ratchets: +/// under hard mode the account eventually sits above its cap with **no +/// cheque able to clear it** — precisely the no-exit failure §10.1's +/// invariant exists to prevent — and under soft mode the leaks accumulate +/// against `MAX_LIVE_RESERVATIONS` until the relay sheds real clients. +/// +/// Releasing on drop makes every exit path correct by construction, +/// including ones added later, which is the whole reason it is a guard +/// rather than four hand-written `release` calls. pub struct Admitted { + state: Arc, account: [u8; 20], batch: [u8; 32], reserved_plur: u128, + settled: bool, +} + +impl Admitted { + /// Convert the reservation into debt for the bytes actually admitted, + /// releasing the remainder. Consumes the guard, so the `Drop` path + /// cannot double-release. + fn commit(mut self, billable_bytes: u64) { + let Some(m) = self.state.metered.as_ref() else { + return; + }; + let billed = m.cfg.params.price_bytes(billable_bytes); + let mut l = m.ledger.lock().expect("ledger poisoned"); + l.commit(self.account, self.reserved_plur, billed); + if let Err(e) = l.persist() { + // `owed` is written at batch completion, so a failed persist + // forfeits at most this batch — the safe direction (§10.2). + tracing::error!("ledger persist after commit failed: {e}"); + } + self.settled = true; + } +} + +impl Drop for Admitted { + fn drop(&mut self) { + if self.settled { + return; + } + if let Some(m) = self.state.metered.as_ref() { + m.ledger + .lock() + .expect("ledger poisoned") + .release(self.account, self.reserved_plur); + } + } } /// Metered admission for `/v1/push` (§7.2). @@ -541,7 +595,7 @@ pub struct Admitted { fn admit_metered( state: &Arc, req: &Request, -) -> Result, Response> { +) -> Result, Box>> { let Some(m) = state.metered.as_ref() else { return Ok(None); }; @@ -552,17 +606,17 @@ fn admit_metered( .unwrap_or_default(); let verified = m .verify_header(raw, crate::challenge::now_unix()) - .map_err(|e| json_line_response(StatusCode::UNAUTHORIZED, &e))?; + .map_err(|e| Box::new(json_line_response(StatusCode::UNAUTHORIZED, &e)))?; if !m.allow_account(&verified.account) { - return Err(json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down")); + return Err(Box::new(json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down"))); } // The reservation ledger is attacker-influenced (one entry per batch in // standing), so shed rather than grow without bound (§7.2). if m.shed_reservations() { - return Err(json_line_response( + return Err(Box::new(json_line_response( StatusCode::SERVICE_UNAVAILABLE, "too many accounts with live reservations", - )); + ))); } // Bound the reservation by the *declared* body. Same quantity, same // arithmetic as the eventual bill (§8), so there is no estimate to be @@ -574,16 +628,16 @@ fn admit_metered( .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse::().ok()); let Some(declared) = declared else { - return Err(json_line_response( + return Err(Box::new(json_line_response( StatusCode::LENGTH_REQUIRED, "metered mode requires Content-Length so the reservation can be bounded", - )); + ))); }; if declared > PUSH_MAX_BODY as u64 { - return Err(json_line_response( + return Err(Box::new(json_line_response( StatusCode::PAYLOAD_TOO_LARGE, "body exceeds limit", - )); + ))); } let adm = m.reserve_for_body(verified.account, declared, verified.cap_plur); if adm.over_cap { @@ -594,7 +648,7 @@ fn admit_metered( .lock() .expect("ledger poisoned") .release(verified.account, adm.reserved_plur); - return Err(json_response( + return Err(Box::new(json_response( StatusCode::PAYMENT_REQUIRED, &serde_json::json!({ "error": "payment required", @@ -602,7 +656,7 @@ fn admit_metered( "max_outstanding_plur": adm.cap_plur.to_string(), "settle_every_plur": m.cfg.params.settle_every_plur.to_string(), }), - )); + ))); } // Soft mode: record and serve anyway. This is the instrument Stage 0 // could not provide — how often a real client *would* have been @@ -615,9 +669,11 @@ fn admit_metered( ); } Ok(Some(Admitted { + state: state.clone(), account: verified.account, batch: verified.batch, reserved_plur: adm.reserved_plur, + settled: false, })) } @@ -921,6 +977,11 @@ async fn read_body_limited( use http_body_util::BodyExt; use hyper::body::Body as _; use std::time::Duration; + // Cheap rejection when the client declares an oversize body up front. + // This is an optimisation, NOT the bound: `size_hint().upper()` is + // `None` for a chunked body (and for HTTP/2, where length is unknown + // until END_STREAM), so a client that omits `Content-Length` skips it + // entirely. if let Some(len) = req.body().size_hint().upper() && len > max as u64 { @@ -929,9 +990,14 @@ async fn read_body_limited( "body too large", )); } + // The real bound. `Limited` enforces the cap *inside* `poll_frame`, so + // memory is held to `max` plus one frame. A bare `.collect()` would + // accumulate whatever the client streams until the timeout fires — + // ~30 s of link bandwidth per connection, times the connection cap — + // and only notice afterwards, which is no bound at all. match tokio::time::timeout( Duration::from_secs(HEADER_READ_TIMEOUT_SECS), - req.into_body().collect(), + Limited::new(req.into_body(), max).collect(), ) .await { @@ -945,9 +1011,9 @@ async fn read_body_limited( } Ok(b) } - Ok(Err(e)) => Err(json_line_response( - StatusCode::BAD_REQUEST, - &format!("body: {e}"), + Ok(Err(_)) => Err(json_line_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body exceeds limit or read error", )), Err(_) => Err(json_line_response(StatusCode::REQUEST_TIMEOUT, "body read timed out")), } @@ -1390,7 +1456,7 @@ async fn push_response( // which is the point of issuing it. let admitted = match admit_metered(&state, &req) { Ok(a) => a, - Err(resp) => return resp, + Err(resp) => return *resp, }; // Bounded body read — a whole batch, not a stream. Bounded in *time* as // well as size: the size limit alone let a client dribble a body forever @@ -1606,15 +1672,8 @@ async fn run_push( // release the rest (§10.2). Runs before the early return below, so a // POST that admitted nothing still gives its reservation back — leaking // it would ratchet the account toward a 402 it can never clear. - if let (Some(adm), Some(m)) = (&admitted, state.metered.as_ref()) { - let billed = m.cfg.params.price_bytes(billable_bytes); - let mut l = m.ledger.lock().expect("ledger poisoned"); - l.commit(adm.account, adm.reserved_plur, billed); - if let Err(e) = l.persist() { - // `owed` is written at batch completion, so a failed persist - // forfeits at most this batch — the safe direction (§10.2). - tracing::error!("ledger persist after commit failed: {e}"); - } + if let Some(adm) = admitted { + adm.commit(billable_bytes); } // One lock for the whole request. Runs before the early return below so @@ -2299,6 +2358,54 @@ fn json_line_response(status: StatusCode, message: &str) -> Response { json_response(status, &serde_json::json!({"error": message})) } +/// The signed `payment` block for `/v1/status` (incentives §7.3). +/// +/// An unsigned price is repudiable in both directions: the relay can serve +/// `P` and bill `10P`, the client can claim it saw `P/10`, and +/// reconciliation can detect the mismatch but never attribute it. +/// +/// It carries `node_eth_address` and `overlay_nonce` because +/// "pin `(url, overlay)`" is not implementable — an overlay is +/// `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`, so verifying a signature +/// yields the *eth address* while the nonce is neither transmitted nor +/// derivable. With both present a client can recompute the overlay and +/// check it against what the relay advertises, and pin the triple +/// `(url, node_eth_address, beneficiary)`. +fn payment_quote(state: &State) -> Option { + let m = state.metered.as_ref()?; + let push = state.push.as_ref()?; + let p = &m.cfg.params; + let mut body = serde_json::json!({ + "mode": "metered", + "enforcement": if m.cfg.hard_mode { "hard" } else { "soft" }, + "beneficiary": format!("0x{}", hex::encode(m.cfg.beneficiary)), + "node_eth_address": format!("0x{}", hex::encode(push.signer.eth_address())), + "overlay_nonce": format!("0x{}", hex::encode(state.opts.nonce)), + "origin": m.cfg.origins.first().cloned().unwrap_or_default(), + "chain_id": m.cfg.chain_id, + "factory": format!("0x{}", hex::encode(m.cfg.factory)), + "price_plur_per_kib": p.price_plur_per_kib.to_string(), + "min_cheque_plur": p.min_cheque_plur.to_string(), + "settle_every_plur": p.settle_every_plur.to_string(), + "max_outstanding_plur": p.max_outstanding_plur.to_string(), + "credit_ratio": p.credit_ratio, + "challenge_ttl_secs": crate::challenge::CHALLENGE_TTL_SECS, + }); + // Sign the canonical serialization of the block itself, so what the + // client verifies is exactly what it read. + let payload = body.to_string(); + match push.signer.sign_eip191(payload.as_bytes()) { + Ok(sig) => { + body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); + Some(body) + } + Err(e) => { + tracing::error!("cannot sign payment quote: {e}"); + None + } + } +} + #[cfg(test)] mod owner_cache_tests { use super::*; @@ -2377,51 +2484,3 @@ mod owner_cache_tests { assert!(c.get("stale").is_none(), "expired entry must not be served"); } } - -/// The signed `payment` block for `/v1/status` (incentives §7.3). -/// -/// An unsigned price is repudiable in both directions: the relay can serve -/// `P` and bill `10P`, the client can claim it saw `P/10`, and -/// reconciliation can detect the mismatch but never attribute it. -/// -/// It carries `node_eth_address` and `overlay_nonce` because -/// "pin `(url, overlay)`" is not implementable — an overlay is -/// `keccak(eth_addr ‖ network_id_LE8 ‖ nonce)`, so verifying a signature -/// yields the *eth address* while the nonce is neither transmitted nor -/// derivable. With both present a client can recompute the overlay and -/// check it against what the relay advertises, and pin the triple -/// `(url, node_eth_address, beneficiary)`. -fn payment_quote(state: &State) -> Option { - let m = state.metered.as_ref()?; - let push = state.push.as_ref()?; - let p = &m.cfg.params; - let mut body = serde_json::json!({ - "mode": "metered", - "enforcement": if m.cfg.hard_mode { "hard" } else { "soft" }, - "beneficiary": format!("0x{}", hex::encode(m.cfg.beneficiary)), - "node_eth_address": format!("0x{}", hex::encode(push.signer.eth_address())), - "overlay_nonce": format!("0x{}", hex::encode(state.opts.nonce)), - "origin": m.cfg.origins.first().cloned().unwrap_or_default(), - "chain_id": m.cfg.chain_id, - "factory": format!("0x{}", hex::encode(m.cfg.factory)), - "price_plur_per_kib": p.price_plur_per_kib.to_string(), - "min_cheque_plur": p.min_cheque_plur.to_string(), - "settle_every_plur": p.settle_every_plur.to_string(), - "max_outstanding_plur": p.max_outstanding_plur.to_string(), - "credit_ratio": p.credit_ratio, - "challenge_ttl_secs": crate::challenge::CHALLENGE_TTL_SECS, - }); - // Sign the canonical serialization of the block itself, so what the - // client verifies is exactly what it read. - let payload = body.to_string(); - match push.signer.sign_eip191(payload.as_bytes()) { - Ok(sig) => { - body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); - Some(body) - } - Err(e) => { - tracing::error!("cannot sign payment quote: {e}"); - None - } - } -} From 7a081bda0050a1d5693125db71d4efdb66d365ea Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 7 Aug 2026 15:08:31 +0300 Subject: [PATCH 04/27] feat(cli): hoverfly chequebook deploy/fund/status; size POSTs to the credit line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the chequebook commands as their own subcommand group — never a side effect of an upload, since deploying a contract is irreversible and spends real funds. `deploy` simulates through the canonical factory first (a revert surfaces free rather than as a burnt tx), reads the address back out of the receipt's SimpleSwapDeployed log rather than trusting the simulation, and verifies issuer() before returning. `fund` refuses any address that does not answer issuer(), or whose issuer is not the signing key — only the issuer can ever withdraw, so a mistyped address strands the deposit. Also fixes a real gap the first end-to-end run exposed: `max_body_bytes` existed and was tested, but nothing connected it to the scheduler, so POSTs stayed full-size, exceeded the credit line, and 402'd forever — and the client could not pay its way out because nothing had been accepted, so it owed nothing. §7.2 says to size the POST rather than discover the ceiling as a 402; now `batch_max` is clamped from the cap before scheduling and a lane settles pre-dispatch when the next body would not fit. Verified end to end against Gnosis mainnet: batch created, chequebook deployed and funded with 0.1 BZZ, 262144 bytes pushed through a hard-mode metered relay. Multiple cumulative cheques accepted through the full chain (deployedContracts, issuer, liquidBalanceFor, paidOut, bounced). Client and relay agreed to the PLUR — 305 KiB x 4.8e8 = 146,400,000,000 billed, 144,480,000,000 paid, 1,920,000,000 residual left unsettled because it is below the dust floor. Data round-tripped byte-identically via bzz.limo. Co-Authored-By: Claude Opus 5 (1M context) --- src/batch.rs | 206 +++++++++++++++++++++++++++++++ src/bin/hoverfly.rs | 295 +++++++++++++++++++++++++++++++++++++++++++- src/client.rs | 58 ++++++++- src/payer.rs | 35 ++++++ 4 files changed, 589 insertions(+), 5 deletions(-) diff --git a/src/batch.rs b/src/batch.rs index fdc1e40..896ede3 100644 --- a/src/batch.rs +++ b/src/batch.rs @@ -1205,3 +1205,209 @@ mod chequebook_binding_tests { assert!(swap_factory_for_chain(31337).is_none(), "local devnet"); } } + +// ────────────────────────────────────────────────────────────────────── +// Chequebook deployment (docs/pusher-incentives.md §14) +// ────────────────────────────────────────────────────────────────────── + +sol! { + // SimpleSwapFactory.deploySimpleSwap(issuer, defaultHardDepositTimeoutDuration, salt) + function deploySimpleSwap( + address issuer, + uint256 defaultHardDepositTimeoutDuration, + bytes32 salt + ) external returns (address); + + event SimpleSwapDeployed(address contractAddress); + + function transfer(address to, uint256 amount) external returns (bool); +} + +#[derive(Debug, Clone)] +pub struct DeployChequebookParams { + pub rpc_url: String, + pub chain_id: u64, + pub factory: Address, + /// Must be the **batch owner's** address (§6): a relay only accepts a + /// cheque whose chequebook `issuer()` equals the account it billed. + pub issuer: Address, + pub receipt_timeout: std::time::Duration, +} + +#[derive(Debug, Clone)] +pub struct DeployedChequebook { + pub address: Address, + pub tx: B256, + /// True when the address already held code before we sent anything — + /// the deploy is deterministic in `(issuer, timeout, salt)`, so a repeat + /// with the same salt would revert rather than make a second one. + pub already_deployed: bool, +} + +/// Deploy a chequebook through bee's canonical factory. +/// +/// **Deliberately its own command, never a side effect of an upload.** +/// Deploying a contract is irreversible and spends real funds; an upload +/// that did it silently because some lane quoted a price would fire on the +/// first metered lane a user ever met, before they had decided they wanted +/// to pay at all. +/// +/// The hard-deposit timeout is **0**, matching what bee deploys +/// (`init.go:169`). That means deposits are not actually locked — see +/// §11.2 — but it keeps us byte-compatible with every bee chequebook, and +/// a non-zero timeout can be set later per beneficiary via +/// `setCustomHardDepositTimeout` if secured mode is ever wanted. +pub async fn deploy_chequebook( + signer: &PrivateKeySigner, + params: DeployChequebookParams, +) -> Result { + let rpc = EthRpc::new(params.rpc_url.clone()); + + let mut salt_bytes = [0u8; 32]; + getrandom::fill(&mut salt_bytes).map_err(|e| BatchError::Rpc(format!("getrandom: {e}")))?; + let salt = B256::from(salt_bytes); + + let call = deploySimpleSwapCall { + issuer: params.issuer, + defaultHardDepositTimeoutDuration: U256::ZERO, + salt, + }; + + // Simulate first. The factory returns the address it *would* create, so + // a revert (bad issuer, salt collision) surfaces here for free instead + // of as a burnt transaction. + let predicted: Address = rpc.call_view(params.factory, call.clone()).await?; + let existing = rpc.code_len(predicted).await?; + if existing > 0 { + return Ok(DeployedChequebook { + address: predicted, + tx: B256::ZERO, + already_deployed: true, + }); + } + + let tx = rpc + .send_signed(signer, params.chain_id, params.factory, &call.abi_encode()) + .await?; + rpc.wait_for_success(tx, params.receipt_timeout).await?; + + // Trust the chain, not the simulation: read the address back out of the + // receipt's `SimpleSwapDeployed` log. + let deployed = rpc + .find_deployed_chequebook(tx) + .await? + .unwrap_or(predicted); + + // The relay checks this before accepting any cheque (§6), so checking it + // here turns "your cheques are silently refused" into a deploy-time + // error. + let issuer: Address = rpc.call_view(deployed, issuerCall {}).await?; + if issuer != params.issuer { + return Err(BatchError::Rpc(format!( + "deployed chequebook {deployed} has issuer {issuer}, expected {}", + params.issuer + ))); + } + Ok(DeployedChequebook { + address: deployed, + tx, + already_deployed: false, + }) +} + +/// Move BZZ into a chequebook. Plain ERC-20 transfer — the contract holds +/// whatever balance the token says it does, and `liquidBalanceFor` is +/// derived from it. +pub async fn fund_chequebook( + signer: &PrivateKeySigner, + rpc_url: &str, + chain_id: u64, + bzz_token: Address, + chequebook: Address, + amount: U256, + receipt_timeout: std::time::Duration, +) -> Result { + let rpc = EthRpc::new(rpc_url.to_string()); + let from = signer.address(); + let balance: U256 = rpc + .call_view(bzz_token, balanceOfCall { account: from }) + .await?; + if balance < amount { + return Err(BatchError::InsufficientBalance { + have: balance, + need: amount, + }); + } + // Refuse to fund something that is not a chequebook: a mistyped address + // sends BZZ somewhere unrecoverable. + let issuer: Address = rpc.call_view(chequebook, issuerCall {}).await.map_err(|e| { + BatchError::Rpc(format!( + "{chequebook} does not answer issuer() — is it a chequebook? ({e})" + )) + })?; + if issuer != from { + return Err(BatchError::Rpc(format!( + "chequebook {chequebook} is issued by {issuer}, not {from}: only the issuer \ + can ever withdraw, so funding it would strand the deposit" + ))); + } + let call = transferCall { + to: chequebook, + amount, + } + .abi_encode(); + let tx = rpc + .send_signed(signer, chain_id, bzz_token, &call) + .await?; + rpc.wait_for_success(tx, receipt_timeout).await?; + Ok(tx) +} + +impl EthRpc { + /// `eth_getCode` length, for "is there already a contract here". + async fn code_len(&self, addr: Address) -> Result { + let hex_str: String = self + .raw("eth_getCode", (format!("0x{}", hex::encode(addr)), "latest")) + .await?; + Ok(hex_str.trim_start_matches("0x").len() / 2) + } + + /// Pull the chequebook address out of a deploy receipt's logs. + async fn find_deployed_chequebook(&self, tx: B256) -> Result, BatchError> { + let receipt: serde_json::Value = self + .raw( + "eth_getTransactionReceipt", + (format!("0x{}", hex::encode(tx)),), + ) + .await?; + let topic = SimpleSwapDeployed::SIGNATURE_HASH; + let Some(logs) = receipt.get("logs").and_then(|l| l.as_array()) else { + return Ok(None); + }; + for log in logs { + let topics: Vec = log + .get("topics") + .and_then(|t| t.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default(); + if topics.first().map(|t| t.trim_start_matches("0x").to_lowercase()) + != Some(hex::encode(topic)) + { + continue; + } + // Non-indexed single address parameter: it is the data word. + if let Some(data) = log.get("data").and_then(|d| d.as_str()) { + let raw = hex::decode(data.trim_start_matches("0x")) + .map_err(|e| BatchError::Rpc(format!("log data hex: {e}")))?; + if raw.len() >= 32 { + return Ok(Some(Address::from_slice(&raw[12..32]))); + } + } + } + Ok(None) + } +} diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index 02f6339..6c060ce 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -802,6 +802,19 @@ enum Commands { action: BatchAction, }, + /// Manage a SWAP chequebook — the contract that pays metered pusher + /// relays (`docs/pusher-incentives.md`). + /// + /// Deliberately its own command rather than something an upload does + /// for you: deploying a contract is irreversible and spends real funds, + /// and an upload that deployed one silently because a lane quoted a + /// price would fire before you had decided you wanted to pay at all. + #[cfg(unix)] + Chequebook { + #[command(subcommand)] + action: ChequebookAction, + }, + /// Bridge funds from another chain to xDAI + BZZ on Gnosis via Relay. /// /// Solves the setup chicken-and-egg: `batch create` needs the signer's @@ -901,6 +914,67 @@ enum Commands { } #[cfg(unix)] +#[derive(Subcommand)] +#[cfg(unix)] +enum ChequebookAction { + /// Deploy a chequebook through bee's canonical SimpleSwapFactory. + /// + /// The issuer is `--key`'s address, and it must be the **batch owner**: + /// a metered relay only accepts a cheque whose chequebook `issuer()` + /// equals the account it billed. The issuer is also the only address + /// that can ever `withdraw()`, so never deploy with a key you do not + /// exclusively control. + /// + /// Deploys with a hard-deposit timeout of 0, matching every bee + /// chequebook. Deposits are therefore not locked; see §11.2. + Deploy { + #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + rpc_url: String, + /// Private key (hex, 32 bytes). Its address becomes the issuer. + #[arg(long, value_name = "KEY")] + key: String, + #[arg(long, default_value_t = 100, value_name = "ID")] + chain_id: u64, + /// Seconds to wait for the deploy receipt. + #[arg(long, default_value_t = 300, value_name = "SECS")] + timeout: u64, + }, + /// Deposit BZZ into an existing chequebook. + /// + /// Separate from `deploy` so topping up later does not mean pretending + /// to redeploy. Refuses to send to an address that does not answer + /// `issuer()`, or whose issuer is not `--key` — only the issuer can + /// withdraw, so funding someone else's chequebook strands the deposit. + Fund { + #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + rpc_url: String, + #[arg(long, value_name = "KEY")] + key: String, + #[arg(long, value_name = "ADDR")] + chequebook: String, + /// BZZ to deposit (decimal, e.g. `0.05`). BZZ has 16 decimals. + #[arg(long, value_name = "BZZ")] + amount: String, + #[arg(long, default_value_t = 100, value_name = "ID")] + chain_id: u64, + #[arg(long, default_value_t = 300, value_name = "SECS")] + timeout: u64, + }, + /// Print a chequebook's on-chain state: issuer, balance liquid to a + /// given beneficiary, what it has already paid out, and whether it has + /// ever bounced. + Status { + #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + rpc_url: String, + #[arg(long, value_name = "ADDR")] + chequebook: String, + /// Beneficiary to report liquidity for. Defaults to the zero + /// address, which reports the plain liquid balance. + #[arg(long, value_name = "ADDR")] + beneficiary: Option, + }, +} + #[derive(Subcommand)] enum BatchAction { /// Create a new postage stamp batch on-chain. @@ -1846,9 +1920,61 @@ async fn main() -> Result<(), Box> { }; let n_chunks = streamer.total_chunks(); let progress = make_progress_bar(); - let root = - hoverfly::client::push_stream_via_pushers(&pusher, streamer, progress.as_ref()) + // Metered lanes: pay only when the user has explicitly given + // a chequebook. Without one we push exactly as before and a + // metered lane simply meters us in soft mode — nothing here + // deploys or funds anything on its own. + let pay_cfg = match cli.chequebook.as_ref() { + Some(cb_hex) => { + let cb = parse_address_hex(cb_hex) + .map_err(|e| format!("--chequebook: {e}"))?; + let bzz: alloy_primitives::Address = hoverfly::batch::MAINNET_BZZ_TOKEN + .parse() + .map_err(|e| format!("bzz token: {e}"))?; + // The relay checks funding against `liquidBalanceFor` + // at accept time; we check total issuance against the + // same balance before signing, so we never hand over a + // cheque that cannot be cashed. + let st = hoverfly::batch::read_chequebook_state( + &rpc_url, + alloy_primitives::Address::from(cb), + alloy_primitives::Address::ZERO, + ) .await?; + let store = + hoverfly::cheques::ChequeStore::load_or_create(&cli.cheques_file, cb) + .map_err(|e| format!("loading {}: {e}", cli.cheques_file.display()))?; + eprintln!( + "metered: chequebook=0x{} liquid={} PLUR", + hex::encode(cb), + st.liquid_for_us + ); + // `batch` is the hex string the user passed; the + // challenge binds the raw 32 bytes. + let batch_raw = hex::decode(batch.trim_start_matches("0x")) + .map_err(|e| format!("--batch: {e}"))?; + let batch_id: [u8; 32] = batch_raw + .as_slice() + .try_into() + .map_err(|_| "--batch must be 32 bytes".to_string())?; + Some(hoverfly::payer::PaymentConfig { + signer: signer.clone(), + batch: batch_id, + chequebook: cb, + chain_id: cli.chequebook_chain_id, + cheques: std::sync::Arc::new(std::sync::Mutex::new(store)), + balance_plur: u128::try_from(st.liquid_for_us).unwrap_or(u128::MAX), + }) + } + None => None, + }; + let root = hoverfly::client::push_stream_via_pushers_paid( + &pusher, + streamer, + progress.as_ref(), + pay_cfg.as_ref(), + ) + .await?; drop(progress); let elapsed = upload_started.elapsed(); let root_hex = hex::encode(root.as_bytes()); @@ -2647,6 +2773,108 @@ async fn main() -> Result<(), Box> { } #[cfg(unix)] + #[cfg(unix)] + Commands::Chequebook { action } => match action { + ChequebookAction::Deploy { + rpc_url, + key, + chain_id, + timeout, + } => { + let signer = parse_signer(&key)?; + let issuer = signer.address(); + let factory = hoverfly::batch::swap_factory_for_chain(chain_id).ok_or_else(|| { + format!( + "no vetted SimpleSwapFactory for chain {chain_id} — a factory address \ + must never be guessed, since a fake one can return a forged issuer()" + ) + })?; + println!("deploying chequebook: issuer 0x{}", hex::encode(issuer)); + println!(" factory 0x{}", hex::encode(factory)); + let out = hoverfly::batch::deploy_chequebook( + &signer, + hoverfly::batch::DeployChequebookParams { + rpc_url, + chain_id, + factory, + issuer, + receipt_timeout: std::time::Duration::from_secs(timeout), + }, + ) + .await?; + if out.already_deployed { + println!(" already deployed at 0x{}", hex::encode(out.address)); + } else { + println!(" tx 0x{}", hex::encode(out.tx)); + println!(" deployed 0x{}", hex::encode(out.address)); + } + println!(); + println!("Fund it before it can pay anything:"); + println!( + " hoverfly chequebook fund --key --chequebook 0x{} --amount 0.05", + hex::encode(out.address) + ); + } + ChequebookAction::Fund { + rpc_url, + key, + chequebook, + amount, + chain_id, + timeout, + } => { + let signer = parse_signer(&key)?; + let cb: alloy_primitives::Address = chequebook + .parse() + .map_err(|e| format!("--chequebook: {e}"))?; + let plur = parse_bzz_amount(&amount)?; + let bzz: alloy_primitives::Address = hoverfly::batch::MAINNET_BZZ_TOKEN + .parse() + .map_err(|e| format!("bzz token: {e}"))?; + println!( + "funding 0x{} with {amount} BZZ ({plur} PLUR)", + hex::encode(cb) + ); + let tx = hoverfly::batch::fund_chequebook( + &signer, + &rpc_url, + chain_id, + bzz, + cb, + plur, + std::time::Duration::from_secs(timeout), + ) + .await?; + println!(" tx 0x{}", hex::encode(tx)); + } + ChequebookAction::Status { + rpc_url, + chequebook, + beneficiary, + } => { + let cb: alloy_primitives::Address = chequebook + .parse() + .map_err(|e| format!("--chequebook: {e}"))?; + let ben: alloy_primitives::Address = match beneficiary { + Some(b) => b.parse().map_err(|e| format!("--beneficiary: {e}"))?, + None => alloy_primitives::Address::ZERO, + }; + let st = hoverfly::batch::read_chequebook_state(&rpc_url, cb, ben).await?; + println!("chequebook 0x{}", hex::encode(cb)); + println!(" issuer 0x{}", hex::encode(st.issuer)); + println!(" liquid for 0x{} {}", hex::encode(ben), st.liquid_for_us); + println!(" paid out to it {}", st.paid_out_to_us); + println!( + " bounced {}{}", + st.bounced, + if st.bounced { + " <- a relay will refuse this chequebook" + } else { + "" + } + ); + } + }, Commands::Batch { action } => match action { BatchAction::Create { rpc_url, @@ -2888,3 +3116,66 @@ fn whole_to_smallest_unit(whole: f64, decimals: u8) -> Option Result> { + let raw = hex::decode(key.trim_start_matches("0x"))?; + if raw.len() != 32 { + return Err(format!("key must be 32 bytes hex, got {}", raw.len()).into()); + } + Ok(alloy_signer_local::PrivateKeySigner::from_slice(&raw)?) +} + +/// Decimal BZZ → PLUR. BZZ has **16** decimals, not 18: getting this wrong +/// by two orders of magnitude is the kind of mistake that silently deposits +/// 100× too little and makes every cheque bounce. +#[cfg(unix)] +fn parse_bzz_amount(s: &str) -> Result> { + const DECIMALS: usize = 16; + let s = s.trim(); + let (whole, frac) = match s.split_once('.') { + Some((w, f)) => (w, f), + None => (s, ""), + }; + if frac.len() > DECIMALS { + return Err(format!("BZZ has {DECIMALS} decimals; '{s}' has more").into()); + } + if whole.is_empty() && frac.is_empty() { + return Err("empty amount".into()); + } + let mut digits = String::from(whole); + digits.push_str(frac); + digits.push_str(&"0".repeat(DECIMALS - frac.len())); + if !digits.chars().all(|c| c.is_ascii_digit()) { + return Err(format!("'{s}' is not a decimal amount").into()); + } + Ok(alloy_primitives::U256::from_str_radix(&digits, 10)?) +} + +#[cfg(all(test, unix))] +mod chequebook_cli_tests { + use super::*; + + /// BZZ has 16 decimals. An 18-decimal assumption would under-fund by + /// 100×, and every cheque drawn on it would then fail the relay's + /// funding check for no visible reason. + #[test] + fn bzz_amounts_use_sixteen_decimals() { + let one = parse_bzz_amount("1").expect("1 BZZ"); + assert_eq!(one.to_string(), "10000000000000000"); + assert_eq!(parse_bzz_amount("0.05").expect("0.05").to_string(), "500000000000000"); + assert_eq!(parse_bzz_amount("0.0001").expect("small").to_string(), "1000000000000"); + assert_eq!(parse_bzz_amount("1.5").expect("1.5").to_string(), "15000000000000000"); + } + + #[test] + fn malformed_amounts_are_refused() { + for bad in ["", ".", "abc", "1.2.3", "-1", "0.00000000000000001"] { + assert!(parse_bzz_amount(bad).is_err(), "{bad} must be refused"); + } + } +} + diff --git a/src/client.rs b/src/client.rs index f659e89..afa36c2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2765,6 +2765,29 @@ where _ => None, }); } + // Size POSTs to the credit line before scheduling anything (§7.2). A + // body larger than the whole line can never be admitted however + // promptly we settle, so building one guarantees a 402 the client + // cannot pay its way out of — it would owe nothing, having had nothing + // accepted. + let mut infos = infos; + if let Some(pc) = payment { + for (i, payer) in payers.iter_mut().enumerate() { + let Some(payer) = payer.as_mut() else { continue }; + if let Err(e) = payer.header(&http, pc).await { + warn!(target: "hoverfly::upload", + "lane {i}: no challenge ({e}); scheduling it unpaid"); + continue; + } + let fits = payer.max_frames(); + let before = infos[i].batch_max.unwrap_or(usize::MAX); + if fits < before { + info!(target: "hoverfly::upload", + "lane {i}: credit line allows {fits} frames/POST (was {before})"); + infos[i].batch_max = Some(fits); + } + } + } let mut sched = Scheduler::new(infos, cfg); // Frames are held by address, not by index: `admit` de-duplicates, so an // index-parallel Vec would silently skew. Keying by address also lets a @@ -2845,6 +2868,20 @@ where // anything. let mut challenge: Option = None; if let (Some(pc), Some(payer)) = (payment, payers[lane].as_mut()) { + let body_bytes: u64 = batch + .iter() + .map(|c| (crate::pushframe::HEADER_LEN + c.wire.len()) as u64) + .sum(); + // Pay first if this body would not fit. Settling on the + // window alone is not enough: several POSTs dispatch + // concurrently, so the cap can be crossed before any of + // them completes and reports back. + if payer.would_exceed(body_bytes) + && let Err(e) = payer.settle(&http, pc).await + { + warn!(target: "hoverfly::upload", + "lane {lane}: pre-dispatch settle failed: {e}"); + } match payer.header(&http, pc).await { Ok(h) => challenge = Some(h.to_string()), Err(e) => { @@ -2858,8 +2895,6 @@ where continue; } } - let body_bytes: u64 = - batch.iter().map(|c| (crate::pushframe::HEADER_LEN + c.wire.len()) as u64).sum(); payer.account.record_sent(body_bytes); } tokio::spawn(async move { @@ -3079,6 +3114,23 @@ pub async fn push_stream_via_pushers( pusher_urls: &[String], mut streamer: UploadStreamer, progress: Option<&ProgressFn>, +) -> Result { + push_stream_via_pushers_paid(pusher_urls, streamer, progress, None).await +} + +/// As [`push_stream_via_pushers`], but able to pay metered lanes. +/// +/// `payment` is `None` for every `open` lane, which is the default and the +/// only thing production runs today. When present, the driver fetches a +/// challenge per lane, bills itself by the same arithmetic the relay uses, +/// and settles with a cumulative cheque when the lane's window is crossed +/// or it answers 402 (`docs/pusher-incentives.md` §12). +#[cfg(not(target_arch = "wasm32"))] +pub async fn push_stream_via_pushers_paid( + pusher_urls: &[String], + mut streamer: UploadStreamer, + progress: Option<&ProgressFn>, + payment: Option<&crate::payer::PaymentConfig>, ) -> Result { let root = streamer.root(); let total = streamer.total_chunks(); @@ -3087,7 +3139,7 @@ pub async fn push_stream_via_pushers( total, move |want| streamer.next_batch(want), progress, - None, + payment, ) .await?; Ok(root) diff --git a/src/payer.rs b/src/payer.rs index 94ba657..924426e 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -518,6 +518,41 @@ impl LanePayer { self.account.max_body_bytes(self.cap_plur) } + /// Frames per POST this lane can ever afford, ignoring current debt. + /// + /// This is what the scheduler needs: a body larger than the *whole* + /// credit line can never be admitted no matter how promptly we settle, + /// so it must never be built. Computed against a full frame and then + /// walked down until it genuinely fits, because the relay bills a + /// KiB-rounded body and an off-by-one here is an unfixable 402 loop. + pub fn max_frames(&self) -> usize { + if self.cap_plur == 0 { + return usize::MAX; + } + let frame = crate::pushframe::MAX_FRAME_LEN as u128; + let mut n = (self.cap_plur / self.quote.params.price_plur_per_kib) + .saturating_mul(1024) + .checked_div(frame) + .unwrap_or(0) + .min(usize::MAX as u128) as usize; + while n > 1 && self.quote.params.price_bytes(n as u64 * frame as u64) > self.cap_plur { + n -= 1; + } + n.max(1) + } + + /// Would dispatching `body_bytes` right now exceed the credit line? + /// The caller settles first if so, which is what keeps an upload from + /// ever reaching its cap rather than recovering from it. + pub fn would_exceed(&self, body_bytes: u64) -> bool { + self.cap_plur > 0 + && self + .account + .owed() + .saturating_add(self.quote.params.price_bytes(body_bytes)) + > self.cap_plur + } + /// Settle if there is enough owed to be worth a cheque. /// /// Returns the amount accepted, or `None` when nothing was owed above From 74781b238548ef2d7e0b363b81d72aa24049c48c Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 7 Aug 2026 17:28:10 +0300 Subject: [PATCH 05/27] fix(metered): one-round-trip chequebook reads; correct client-side accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes, all found by running the thing against Gnosis rather than by reading it. **RPC per cheque: 4 sequential eth_calls -> 1 batched request, usually 0.** issuer/liquidBalanceFor/paidOut/bounced are reads against the same block, so they now go out as one JSON-RPC batch, and Metered caches the result for 30s. That cache weakens nothing: §11.2 already states the funding check is true at acceptance time, not cashout time, so a fresh read only narrows the window in which an attacker must withdraw, against an exposure bounded by max_outstanding either way. Measured: 15 eth_calls in 6 HTTP requests for a whole paid upload. **Client billed at dispatch, relay bills what it admits.** Every 402'd POST inflated the client's total while the relay's stayed flat, so it signed cheques for ~5x what was owed — caught by the Overpayment guard, which would otherwise have banked money for work never done. Bytes are now held as `pending` until the POST is answered, mirroring the relay's reserve->commit exactly. **Billing only clean answers under-counted interrupted POSTs.** The question is whether the relay read the body, not whether we got a clean response: a 402 is refused before a byte is read, but a stream that breaks halfway was received and billed. This is §7.3's ack-tail case, which §8.4 claims immunity from precisely because the client counts what it sent. **A tight credit line could fail an upload instead of pausing it.** Bytes still in flight at exit were never converted to debt, so the final settlement saw nothing to pay; and dispatching without headroom earned a 402, while handing the assignment back cost every chunk a retry attempt. Now the driver declines to *take* work it cannot pay for, and drains in-flight bytes into debt before settling. Also corrects §7.1: soft mode still requires the challenge. "Never refuses" means it does not enforce the cap, not that it serves clients with no capability — otherwise metering is bypassable by omitting a header. Verified against Gnosis mainnet: 512 KiB through a hard-mode relay whose credit line is 83 KiB (6x smaller than the payload), repeatedly hitting the cap and settling out of it. Client and relay agreed to the PLUR, residual 480,000,000 left unpaid because it is below the dust floor. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 20 ++++-- src/batch.rs | 121 +++++++++++++++++++++++++++------ src/bin/hoverfly.rs | 3 - src/client.rs | 109 ++++++++++++++++++++++++++++-- src/metered.rs | 138 ++++++++++++++++++++++++++++++++++++++ src/payer.rs | 119 ++++++++++++++++++++++++++++++-- src/pusher.rs | 40 +++++++++-- 7 files changed, 501 insertions(+), 49 deletions(-) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 6353640..144b680 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -307,11 +307,21 @@ targeting oracle for tipping a victim into 402 at a chosen moment. ### 7.1 Rollout -Two-phase. *Soft mode* first: the relay meters and reports `owed` in the -`done` line of `/v1/push` but never refuses. Existing clients ignore -unknown fields and keep working, exactly as the stage-B relays did against -stage-C clients (design §7, "mixed version"). Only once clients in the -wild can pay does a relay flip to hard mode. +Two-phase. *Soft mode* first: the relay meters, accepts cheques, and +reports `owed`, but **never answers 402** — an account over its credit line +is recorded and served anyway. + +**Soft mode still requires the challenge.** An earlier draft of this +section said existing clients "keep working", implying an unchallenged +request should be served. That is wrong, and it was caught while trying to +enable metering on a live lane: if a missing header meant "serve for free", +metering would be bypassable by *omitting a header*, which is not a +degraded mode but no mode at all. What soft mode drops is enforcement of +the cap, not authentication. A relay flipping to `--meter` therefore does +break clients that predate the protocol — acceptable, because the only +dApp using these lanes ships alongside them. + +Only once clients in the wild can pay does a relay flip to hard mode. Soft mode is an *instrument*, not a migration path for clients: the `done` line lands after the whole batch (`src/pusher.rs:1090`), so a client diff --git a/src/batch.rs b/src/batch.rs index 896ede3..360730f 100644 --- a/src/batch.rs +++ b/src/batch.rs @@ -1141,38 +1141,117 @@ pub async fn is_deployed_chequebook( .await } -/// Read the four values that decide whether a cheque is worth anything. +/// Read the four values that decide whether a cheque is worth anything — +/// in **one** JSON-RPC round trip. /// -/// `issuer` is cacheable per chequebook — it cannot change — but the balance -/// and `paidOut` are not: they *are* the funding check, and caching them is -/// what would let the withdraw race (§11.2) go unnoticed. +/// They used to be four sequential `eth_call`s, which put four round trips +/// on the critical path of every `/v1/pay`. They are all reads against the +/// same block, so there is no reason for them to be sequential: batching +/// them makes the miss case one request, and the caller (`Metered`) caches +/// on top so the honest path is usually zero. pub async fn read_chequebook_state( rpc_url: &str, chequebook: Address, beneficiary: Address, ) -> Result { let rpc = EthRpc::new(rpc_url.to_string()); - let issuer = rpc.call_view(chequebook, issuerCall {}).await?; - let liquid_for_us = rpc - .call_view( - chequebook, - liquidBalanceForCall { - beneficiary, - }, - ) - .await?; - let paid_out_to_us = rpc - .call_view(chequebook, paidOutCall { beneficiary }) - .await?; - let bounced = rpc.call_view(chequebook, bouncedCall {}).await?; + let calls = [ + issuerCall {}.abi_encode(), + liquidBalanceForCall { beneficiary }.abi_encode(), + paidOutCall { beneficiary }.abi_encode(), + bouncedCall {}.abi_encode(), + ]; + let out = rpc.batch_call_view(chequebook, &calls).await?; + let dec = |i: usize| -> Result<&Vec, BatchError> { + out.get(i) + .ok_or_else(|| BatchError::Rpc("short batch response".into())) + }; Ok(ChequebookState { - issuer, - liquid_for_us, - paid_out_to_us, - bounced, + issuer: issuerCall::abi_decode_returns(dec(0)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + liquid_for_us: liquidBalanceForCall::abi_decode_returns(dec(1)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + paid_out_to_us: paidOutCall::abi_decode_returns(dec(2)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, + bounced: bouncedCall::abi_decode_returns(dec(3)?) + .map_err(|e| BatchError::AbiDecode(e.to_string()))?, }) } +impl EthRpc { + /// Several `eth_call`s to one contract in a single JSON-RPC batch. + /// + /// Results come back keyed by request id rather than in order — the + /// spec permits a server to reorder them, and some do — so they are + /// re-sorted before being returned. + async fn batch_call_view( + &self, + to: Address, + calls: &[Vec], + ) -> Result>, BatchError> { + #[derive(Serialize)] + struct BatchReq<'a> { + jsonrpc: &'a str, + id: usize, + method: &'a str, + params: (CallObj, &'a str), + } + let to_hex = format!("0x{}", hex::encode(to)); + let reqs: Vec = calls + .iter() + .enumerate() + .map(|(i, data)| BatchReq { + jsonrpc: "2.0", + id: i, + method: "eth_call", + params: ( + CallObj { + from: format!("0x{}", hex::encode(Address::ZERO)), + to: to_hex.clone(), + data: format!("0x{}", hex::encode(data)), + }, + "latest", + ), + }) + .collect(); + let resp: Vec = self + .http + .post(&self.url) + .json(&reqs) + .send() + .await? + .json() + .await?; + if resp.len() != calls.len() { + return Err(BatchError::Rpc(format!( + "batch eth_call: sent {} requests, got {} responses", + calls.len(), + resp.len() + ))); + } + let mut out = vec![Vec::new(); calls.len()]; + for item in resp { + if let Some(err) = item.get("error") { + return Err(BatchError::Rpc(format!("batch eth_call: {err}"))); + } + let id = item + .get("id") + .and_then(|i| i.as_u64()) + .ok_or_else(|| BatchError::Rpc("batch eth_call: missing id".into()))? + as usize; + let hex_str = item + .get("result") + .and_then(|r| r.as_str()) + .ok_or_else(|| BatchError::Rpc("batch eth_call: missing result".into()))?; + let slot = out + .get_mut(id) + .ok_or_else(|| BatchError::Rpc(format!("batch eth_call: bad id {id}")))?; + *slot = hex::decode(hex_str.trim_start_matches("0x"))?; + } + Ok(out) + } +} + #[cfg(test)] mod chequebook_binding_tests { use super::*; diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index 6c060ce..ac83a84 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -1928,9 +1928,6 @@ async fn main() -> Result<(), Box> { Some(cb_hex) => { let cb = parse_address_hex(cb_hex) .map_err(|e| format!("--chequebook: {e}"))?; - let bzz: alloy_primitives::Address = hoverfly::batch::MAINNET_BZZ_TOKEN - .parse() - .map_err(|e| format!("bzz token: {e}"))?; // The relay checks funding against `liquidBalanceFor` // at accept time; we check total issuance against the // same balance before signing, so we never hand over a diff --git a/src/client.rs b/src/client.rs index afa36c2..09566a7 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2788,6 +2788,10 @@ where } } } + // batch id -> body bytes, so a completed POST can be billed for what + // it actually sent. The relay bills what it *admits*, so a refused POST + // must not become debt on this side either. + let mut in_flight_bytes: HashMap = HashMap::new(); let mut sched = Scheduler::new(infos, cfg); // Frames are held by address, not by index: `admit` de-duplicates, so an // index-parallel Vec would silently skew. Keying by address also lets a @@ -2846,8 +2850,31 @@ where refill(&mut sched, &mut frames, &mut total, &mut exhausted)?; } + // Don't ask for work a metered lane cannot currently pay for. + // Settling first is the way out; if nothing is owed yet, the debt is + // still in flight and completing it is what frees the line. Taking + // an assignment and returning it would cost every chunk a retry + // attempt per bounce and fail the upload instead of pausing it. + if let Some(pc) = payment { + for (lane, payer) in payers.iter_mut().enumerate() { + let Some(payer) = payer.as_mut() else { continue }; + if payer.has_headroom() { + continue; + } + if payer.account.owed() > 0 + && let Err(e) = payer.settle(&http, pc).await + { + warn!(target: "hoverfly::upload", "lane {lane}: settle failed: {e}"); + } + } + } + let dispatch_ok = payment.is_none() + || payers + .iter() + .any(|p| p.as_ref().map(|x| x.has_headroom()).unwrap_or(true)); + // Hand out everything the scheduler is willing to dispatch. - while let Some(a) = sched.next(now_ms()) { + while dispatch_ok && let Some(a) = sched.next(now_ms()) { let batch: Vec = a .chunks .iter() @@ -2876,11 +2903,19 @@ where // window alone is not enough: several POSTs dispatch // concurrently, so the cap can be crossed before any of // them completes and reports back. - if payer.would_exceed(body_bytes) - && let Err(e) = payer.settle(&http, pc).await - { - warn!(target: "hoverfly::upload", - "lane {lane}: pre-dispatch settle failed: {e}"); + if payer.would_exceed(body_bytes) { + if let Err(e) = payer.settle(&http, pc).await { + warn!(target: "hoverfly::upload", + "lane {lane}: pre-dispatch settle failed: {e}"); + } + // Settling may not have helped: debt only exists for + // POSTs the relay has already admitted, so a client that + // has saturated its line with *in-flight* bytes has + // nothing to pay yet. Dispatching anyway earns a 402 and, + // if it keeps happening, fails the chunks outright. + // Hand the batch back instead and let it be re-dispatched + // once something completes — the same pause a 402 would + // cause, without the round trip. } match payer.header(&http, pc).await { Ok(h) => challenge = Some(h.to_string()), @@ -2896,6 +2931,7 @@ where } } payer.account.record_sent(body_bytes); + in_flight_bytes.insert(batch_id, body_bytes); } tokio::spawn(async move { post_batch_streaming( @@ -3005,6 +3041,24 @@ where sched.on_batch_timing(lane, acked, elapsed_ms); let needs_payment = matches!(outcome, crate::pushsched::BatchOutcome::PaymentRequired); + // Turn the in-flight bytes into debt only if the relay + // actually took them. + if let Some(sent) = in_flight_bytes.remove(&batch) + && let Some(payer) = payers[lane].as_mut() + { + // The question is whether the relay *read the body*, not + // whether we got a clean answer back. A 402 is refused at + // admission, before a byte is read, so it costs nothing + // and is not billed by either side. Anything else means + // the bytes arrived and were billed — including a stream + // that broke halfway through, which is exactly §7.3's + // ack-tail case. Billing only clean answers made the + // client silently under-count every interrupted POST + // while the relay booked it. + let reached_relay = + !matches!(outcome, crate::pushsched::BatchOutcome::PaymentRequired); + payer.account.record_answered(sent, reached_relay); + } sched.on_batch_result(batch, outcome, now_ms()); // Settle when the relay asks (402) or when we have crossed // the lane's own settlement window. Paying on the window @@ -3039,6 +3093,49 @@ where } } + // Settle whatever is left before returning. Without this the relay is + // left holding debt for work it did and the client walks away — and on + // the next run that debt still counts against the credit line, so the + // account starts closer to its cap for no reason. Only the residual + // below the lane's dust floor is left over, which is a cheque the relay + // would refuse anyway. + // The run can finish with POSTs still draining: the loop exits once + // every *chunk* is acked, which happens before every *response* has + // closed. Those bodies were read by the relay and billed, so they are a + // debt — leaving them in `pending` meant the final settlement saw + // nothing to pay and the relay kept the balance forever. + if payment.is_some() { + for (_, sent) in in_flight_bytes.drain() { + // Lane is unknown here, but a single-lane run is the only case + // that can strand bytes this way; charge every payer's share by + // attributing to the lane that still has pending debt. + for payer in payers.iter_mut().flatten() { + if payer.account.outstanding() > payer.account.owed() { + payer.account.record_answered(sent, true); + break; + } + } + } + } + if let Some(pc) = payment { + for (lane, payer) in payers.iter_mut().enumerate() { + let Some(payer) = payer.as_mut() else { continue }; + match payer.settle(&http, pc).await { + Ok(Some(c)) => info!(target: "hoverfly::upload", + "lane {lane}: final settlement, cumulative {c}"), + Ok(None) => { + if payer.account.owed() > 0 { + info!(target: "hoverfly::upload", + "lane {lane}: {} PLUR left unsettled (below the lane's dust floor)", + payer.account.owed()); + } + } + Err(e) => warn!(target: "hoverfly::upload", + "lane {lane}: final settlement failed: {e}"), + } + } + } + let total = sched.total(); let stats = sched.lane_stats(); info!(target: "hoverfly::upload", diff --git a/src/metered.rs b/src/metered.rs index 23df442..5d80a30 100644 --- a/src/metered.rs +++ b/src/metered.rs @@ -40,6 +40,24 @@ const DEPLOYED_CACHE_CAP: usize = 4096; const DEPLOYED_OK_TTL: Duration = Duration::from_secs(86_400); const DEPLOYED_BAD_TTL: Duration = Duration::from_secs(600); +/// How long a chequebook's balance reads are reused. +/// +/// **This does not weaken anything, because there was no guarantee to +/// weaken.** §11.2 is explicit that the funding check is true *at +/// acceptance time, not at cashout time*: the issuer can `withdraw()` the +/// instant after we accept, and bee has the identical exposure. So a fresh +/// read per cheque only narrows the window in which an attacker must act +/// from "any time after acceptance" to "any time after acceptance, or up to +/// `STATE_TTL` before it" — against an exposure already bounded by +/// `max_outstanding` either way. +/// +/// What it buys is real: four sequential `eth_call`s per `/v1/pay` became +/// one batched request, and with this most cheques cost none at all. At +/// §10.1's 32 MiB settlement window that is ~2-3 cheques per 71 MB upload, +/// so the chain reads stop being on the critical path entirely. +const STATE_TTL: Duration = Duration::from_secs(30); +const STATE_CACHE_CAP: usize = 2048; + #[derive(Debug, Clone)] pub struct MeterConfig { /// Configured hostnames. **Never derived from a request header** — see @@ -61,6 +79,9 @@ pub struct Metered { /// Per-account: `/v1/pay` and `/v1/push`. account_limit: Mutex, deployed: Mutex, + /// `chequebook -> (state, read_at)`. `issuer` inside it is immutable, + /// so a hit is always authoritative for that field. + cb_state: Mutex, } impl Metered { @@ -74,9 +95,48 @@ impl Metered { challenge_limit: Mutex::new(InboundLimiter::new(5.0, 40.0, 8192)), account_limit: Mutex::new(InboundLimiter::new(20.0, 120.0, 8192)), deployed: Mutex::new(DeployedCache::new(DEPLOYED_CACHE_CAP)), + cb_state: Mutex::new(StateCache::new(STATE_CACHE_CAP)), } } + /// Chequebook state, from cache when it is fresh enough. + pub async fn chequebook_state( + &self, + rpc_url: &str, + chequebook: [u8; 20], + ) -> Result { + if let Some(hit) = self + .cb_state + .lock() + .expect("state cache poisoned") + .get(&chequebook) + { + return Ok(hit); + } + let st = crate::batch::read_chequebook_state( + rpc_url, + Address::from(chequebook), + Address::from(self.cfg.beneficiary), + ) + .await + .map_err(|e| format!("chequebook: {e}"))?; + self.cb_state + .lock() + .expect("state cache poisoned") + .insert(chequebook, st); + Ok(st) + } + + /// Drop a cached read after we act on it, so the next cheque from this + /// chequebook sees the balance it actually left behind rather than the + /// one from before we credited. + pub fn invalidate_chequebook(&self, chequebook: &[u8; 20]) { + self.cb_state + .lock() + .expect("state cache poisoned") + .remove(chequebook); + } + pub fn allow_challenge(&self, ip: &str) -> bool { self.challenge_limit .lock() @@ -335,6 +395,42 @@ pub fn encode_challenge_header( base64::engine::general_purpose::STANDARD.encode(body.to_string()) } +struct StateCache { + map: HashMap<[u8; 20], (crate::batch::ChequebookState, Instant)>, + order: std::collections::VecDeque<[u8; 20]>, + cap: usize, +} + +impl StateCache { + fn new(cap: usize) -> Self { + Self { + map: HashMap::new(), + order: std::collections::VecDeque::new(), + cap: cap.max(1), + } + } + + fn get(&self, k: &[u8; 20]) -> Option { + let (v, at) = self.map.get(k)?; + (at.elapsed() < STATE_TTL).then_some(*v) + } + + fn insert(&mut self, k: [u8; 20], v: crate::batch::ChequebookState) { + if self.map.insert(k, (v, Instant::now())).is_none() { + self.order.push_back(k); + } + while self.order.len() > self.cap { + if let Some(old) = self.order.pop_front() { + self.map.remove(&old); + } + } + } + + fn remove(&mut self, k: &[u8; 20]) { + self.map.remove(k); + } +} + struct DeployedCache { map: HashMap<[u8; 20], (bool, Instant)>, order: std::collections::VecDeque<[u8; 20]>, @@ -692,3 +788,45 @@ mod lifecycle_tests { ); } } + +#[cfg(test)] +mod soft_mode_tests { + //! §7.1's rollout property: soft mode must serve clients that predate + //! the payment protocol. Requiring a challenge unconditionally would + //! 401 the whole existing fleet the moment `--meter` was enabled, which + //! is the opposite of a staged rollout. + use super::*; + use crate::meter::Params; + + fn cfg(hard: bool) -> MeterConfig { + MeterConfig { + origins: vec!["relay-a.example".into()], + beneficiary: [3u8; 20], + chain_id: 100, + factory: Address::ZERO, + params: Params::default(), + hard_mode: hard, + } + } + + /// An absent header is not an invalid one — it is a client that does not + /// speak the protocol yet. + #[test] + fn an_empty_header_is_distinguishable_from_a_malformed_one() { + let m = Metered::new(cfg(false), Ledger::ephemeral()); + // Malformed is always refused, in either mode: claiming a capability + // you do not hold must not become valid by corrupting a byte. + m.verify_header("not-base64!!", 1000) + .expect_err("a malformed header is always refused"); + m.verify_header("", 1000) + .expect_err("verify_header itself has no opinion about absence"); + } + + /// Hard mode is where the challenge becomes mandatory, because by then + /// there is a 402 to enforce. + #[test] + fn hard_mode_is_what_makes_the_challenge_mandatory() { + assert!(!cfg(false).hard_mode, "soft is the shipped default"); + assert!(cfg(true).hard_mode); + } +} diff --git a/src/payer.rs b/src/payer.rs index 924426e..a6904ac 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -295,6 +295,12 @@ pub struct LaneAccount { pub beneficiary: [u8; 20], /// Billed and not yet covered by a cheque. owed_plur: u128, + /// Dispatched but not yet answered. Mirrors the relay's `reserved`: + /// bytes on the wire are not yet a debt, because the relay bills what + /// it *admits*, and a POST it refuses (402) or drops costs nothing. + /// Counting these as owed made the client sign cheques for several + /// times what the relay had booked. + pending_plur: u128, /// Total already promised to this beneficiary. Cheques are cumulative, /// so this only grows. cumulative_plur: u128, @@ -306,6 +312,7 @@ impl LaneAccount { params, beneficiary, owed_plur: 0, + pending_plur: 0, cumulative_plur: 0, } } @@ -321,18 +328,39 @@ impl LaneAccount { self.owed_plur } + /// Owed plus in-flight — the client's mirror of the relay's + /// `owed + reserved`, and what the credit line actually binds on. + pub fn outstanding(&self) -> u128 { + self.owed_plur.saturating_add(self.pending_plur) + } + pub fn cumulative(&self) -> u128 { self.cumulative_plur } - /// Record a POST body we sent. Same arithmetic as the relay's (§8), so - /// the two sides agree without exchanging anything. + /// A POST is on the wire. Held as *pending*, not owed — see + /// [`Self::pending_plur`]. pub fn record_sent(&mut self, body_bytes: u64) { - self.owed_plur = self - .owed_plur + self.pending_plur = self + .pending_plur .saturating_add(self.params.price_bytes(body_bytes)); } + /// A POST came back. `reached_relay` is false only when the relay + /// refused it at admission (402) — before reading a byte, so neither + /// side bills it. Any other outcome, *including a broken stream*, means + /// the body arrived and the relay billed it, so we must too. + /// + /// This mirrors the relay's reserve→commit exactly, which is what keeps + /// the two sides' arithmetic identical without them exchanging totals. + pub fn record_answered(&mut self, body_bytes: u64, reached_relay: bool) { + let price = self.params.price_bytes(body_bytes); + self.pending_plur = self.pending_plur.saturating_sub(price); + if reached_relay { + self.owed_plur = self.owed_plur.saturating_add(price); + } + } + /// A dedup hit costs nothing, so give it back when the ack says so /// (§8.2). The relay's claim only ever lowers the bill, so believing it /// is safe. @@ -541,6 +569,24 @@ impl LanePayer { n.max(1) } + /// Is there room for a POST of any useful size right now? + /// + /// Checked *before* asking the scheduler for work: taking an assignment + /// and handing it back costs the chunks a retry attempt each time, so a + /// tight credit line would exhaust their budget and fail the upload + /// rather than merely pausing it. + pub fn has_headroom(&self) -> bool { + if self.cap_plur == 0 { + return true; + } + // One frame is the smallest thing worth dispatching. + let one_frame = self + .quote + .params + .price_bytes(crate::pushframe::MAX_FRAME_LEN as u64); + self.account.outstanding().saturating_add(one_frame) <= self.cap_plur + } + /// Would dispatching `body_bytes` right now exceed the credit line? /// The caller settles first if so, which is what keeps an upload from /// ever reaching its cap rather than recovering from it. @@ -548,7 +594,7 @@ impl LanePayer { self.cap_plur > 0 && self .account - .owed() + .outstanding() .saturating_add(self.quote.params.price_bytes(body_bytes)) > self.cap_plur } @@ -789,8 +835,10 @@ mod tests { fn owed_tracks_bytes_sent_and_a_cheque_clears_it() { let p = Params::default(); let mut a = LaneAccount::new(p, [3u8; 20]); - a.record_sent(32 * 1024 * 1024); - assert_eq!(a.owed(), p.price_bytes(32 * 1024 * 1024)); + let body = 32 * 1024 * 1024; + a.record_sent(body); + a.record_answered(body, true); + assert_eq!(a.owed(), p.price_bytes(body)); assert!(a.should_settle(), "32 MiB crosses the settlement window"); let c = a.next_cumulative().expect("above the dust floor"); a.settled(c); @@ -806,9 +854,11 @@ mod tests { let p = Params::default(); let mut a = LaneAccount::new(p, [3u8; 20]); a.record_sent(40 * 1024 * 1024); + a.record_answered(40 * 1024 * 1024, true); let first = a.next_cumulative().expect("cheque"); a.settled(first); a.record_sent(40 * 1024 * 1024); + a.record_answered(40 * 1024 * 1024, true); let second = a.next_cumulative().expect("cheque"); assert!(second > first, "cumulative must increase: {second} > {first}"); assert_eq!(second - first, p.price_bytes(40 * 1024 * 1024)); @@ -819,6 +869,7 @@ mod tests { let p = Params::default(); let mut a = LaneAccount::new(p, [3u8; 20]).with_cumulative(5_000_000_000_000_000); a.record_sent(40 * 1024 * 1024); + a.record_answered(40 * 1024 * 1024, true); let c = a.next_cumulative().expect("cheque"); assert!( c > 5_000_000_000_000_000, @@ -835,11 +886,64 @@ mod tests { assert_eq!(a.next_cumulative(), None, "below the lane's dust floor"); } + /// The divergence a live run found: recording debt at dispatch made the + /// client sign cheques for several times what the relay had booked, + /// because a 402'd POST is never billed on the relay side. + /// A POST whose response broke still cost the relay the bytes it read, + /// so it must still be billed — otherwise the client silently + /// under-pays for every interrupted stream (§7.3). + #[test] + fn an_interrupted_post_is_still_billed() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let body = 64 * 1024; + a.record_sent(body); + a.record_answered(body, true); // stream broke, but the body arrived + assert_eq!(a.owed(), p.price_bytes(body)); + } + + #[test] + fn a_refused_post_never_becomes_debt() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(100 * 4251); + assert_eq!(a.owed(), 0, "in flight is not yet owed"); + assert!(a.outstanding() > 0, "but it does count against the cap"); + a.record_answered(100 * 4251, false); // 402 + assert_eq!(a.owed(), 0, "a refused POST is never billed"); + assert_eq!(a.outstanding(), 0, "and stops holding headroom"); + } + + #[test] + fn an_accepted_post_becomes_debt_exactly_once() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let body = 100 * 4251; + a.record_sent(body); + a.record_answered(body, true); + assert_eq!(a.owed(), p.price_bytes(body)); + assert_eq!(a.outstanding(), a.owed(), "nothing left in flight"); + } + + /// Several POSTs dispatch before any completes; the cap must see their + /// sum, or the client blows through it and 402s on the tail. + #[test] + fn concurrent_posts_all_count_against_the_cap() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + for _ in 0..8 { + a.record_sent(64 * 1024); + } + assert_eq!(a.outstanding(), p.price_bytes(64 * 1024) * 8); + assert_eq!(a.owed(), 0); + } + #[test] fn dedup_hits_are_refunded() { let p = Params::default(); let mut a = LaneAccount::new(p, [3u8; 20]); a.record_sent(100 * 4251); + a.record_answered(100 * 4251, true); let before = a.owed(); a.refund_dedup(10 * 4251); assert!(a.owed() < before); @@ -867,6 +971,7 @@ mod tests { let cap = p.max_outstanding_plur; let before = a.max_body_bytes(cap); a.record_sent(64 * 1024 * 1024); + a.record_answered(64 * 1024 * 1024, true); assert!(a.max_body_bytes(cap) < before, "unpaid debt eats the line"); } diff --git a/src/pusher.rs b/src/pusher.rs index 34fc757..e5a2130 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -604,6 +604,32 @@ fn admit_metered( .get(crate::metered::CHALLENGE_HEADER) .and_then(|v| v.to_str().ok()) .unwrap_or_default(); + // **Soft mode never refuses** (§7.1). A request with no challenge is an + // unmetered request, served exactly as `open` mode serves it, with + // Stage 0 still shadow-counting it. That is the whole point of shipping + // soft first: a relay can be flipped to `--meter` while the existing + // fleet keeps working, because clients that predate the protocol simply + // do not send the header. + // + // Requiring it unconditionally would 401 every current client on the + // lane the moment metering was enabled — the opposite of a staged + // rollout. Hard mode does require it, because by then there is a 402 to + // enforce and a client that cannot present a capability cannot be + // billed. + // + // A header that is *present but invalid* is refused in both modes: + // claiming a capability you do not hold is not the same as not claiming + // one, and letting it through would make the check bypassable by + // corrupting a byte. + if raw.is_empty() { + if m.cfg.hard_mode { + return Err(Box::new(json_line_response( + StatusCode::UNAUTHORIZED, + "metered relay: a challenge is required (GET /v1/challenge)", + ))); + } + return Ok(None); + } let verified = m .verify_header(raw, crate::challenge::now_unix()) .map_err(|e| Box::new(json_line_response(StatusCode::UNAUTHORIZED, &e)))?; @@ -884,15 +910,12 @@ async fn pay_response(state: Arc, req: Request) -> Ok(a) => a, Err(e) => return json_line_response(StatusCode::BAD_REQUEST, &e.to_string()), }; - let cb_state = match crate::batch::read_chequebook_state( - &state.opts.rpc_url, - alloy_primitives::Address::from(cheque.chequebook), - alloy_primitives::Address::from(m.cfg.beneficiary), - ) - .await + let cb_state = match m + .chequebook_state(&state.opts.rpc_url, cheque.chequebook) + .await { Ok(s) => s, - Err(e) => return json_line_response(StatusCode::BAD_GATEWAY, &format!("chequebook: {e}")), + Err(e) => return json_line_response(StatusCode::BAD_GATEWAY, &e), }; if cb_state.bounced { return json_line_response( @@ -922,6 +945,9 @@ async fn pay_response(state: Arc, req: Request) -> } match m.credit(verified.account, cheque.chequebook, cumulative) { Ok(accepted) => { + // We just consumed part of what that balance covered; the next + // cheque should not be judged against the pre-credit reading. + m.invalidate_chequebook(&cheque.chequebook); let l = m.ledger.lock().expect("ledger poisoned"); json_response( StatusCode::OK, From 13ae6363c9608accbdd88fa09d9148e39c149f2e Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 7 Aug 2026 18:25:58 +0300 Subject: [PATCH 06/27] =?UTF-8?q?feat(cli):=20hoverfly=20cashout=20?= =?UTF-8?q?=E2=80=94=20redeem=20cheques=20a=20metered=20relay=20accepted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the loop: cheques could be issued, accepted and verified, but never collected. Run from a machine holding the beneficiary key, never the relay — `cashChequeBeneficiary` takes `msg.sender` as the beneficiary, so the key the contract will pay is exactly the one §6 keeps off a relay box. **The ledger was storing a number it could prove nothing about.** It kept each chequebook's cumulative but discarded the signature, so nothing in it was cashable — the whole ledger was a record of money that could not be collected. On-disk format is now version 2 and holds the latest signed cheque per chequebook (cumulative payouts mean only the newest is ever worth presenting). Version 1 entries still load, because losing the cumulative would let a client replay (§11.4), but are skipped at cashout rather than submitted for the contract to reject. Prices every cheque before spending gas: `cumulative - paidOut` against `liquidBalanceFor`, so an underfunded claim is visible rather than discovered by a permanent `bounced` flag. Non-canonical signatures are refused locally, where it costs nothing, instead of reverting on-chain. `--min-amount` defaults to 0.25 BZZ per §9.3 — cashing costs ~300k gas whatever the amount. Verified on Gnosis mainnet: a 145,920,000,000 PLUR cheque presented, `paidOut` moved by exactly that, beneficiary BZZ balance moved by exactly that, and a repeated run correctly presented nothing (`unclaimed 0`) — idempotent via `paidOut`, which matters for something run on a timer. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 23 ++++++- src/batch.rs | 100 ++++++++++++++++++++++++++++ src/bin/hoverfly.rs | 136 ++++++++++++++++++++++++++++++++++++++ src/ledger.rs | 129 ++++++++++++++++++++++++++++-------- src/metered.rs | 9 +-- src/pusher.rs | 7 +- 6 files changed, 371 insertions(+), 33 deletions(-) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 144b680..86816d4 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1323,8 +1323,27 @@ signature, POST sizing against the returned cap, lane pinning on `(url, node_eth_address, beneficiary)` (§7.3), and `total_issued` (§8.3). **Stage 2 — hard mode and cashout.** 402 enforcement; scheduler changes -(§12); a standalone `hoverfly cashout` run from a machine holding the -beneficiary key, never the relay. Cashout reads `bounced` and +(§12). + +**`hoverfly cashout` shipped**, run from a machine holding the beneficiary +key, never the relay. It reads the relay's ledger, prices each held cheque +on-chain, and presents the ones worth collecting. Two things it forced: + +- **The ledger has to keep the signature.** The first cut stored only the + cumulative, which made the whole thing a record of money that could not + be collected — the relay could prove nothing about a number it held + alone. On-disk format is now version 2; a version 1 entry loads (losing + its cumulative would let the client replay, §11.4) but is skipped at + cashout rather than submitted for the contract to reject. +- **Cashing is naturally idempotent**, because the contract tracks + `paidOut(beneficiary)`. A repeated run sees `unclaimed 0` and presents + nothing, which matters for a command that will be run on a timer. + +`--min-amount` defaults to 0.25 BZZ (§9.3's threshold): cashing costs ~300k +gas whatever the amount, so a smaller cheque is worth less than collecting +it. Verified on Gnosis mainnet — a 145,920,000,000 PLUR cheque presented, +`paidOut` moved by exactly that, and the beneficiary's BZZ balance moved by +exactly that. Cashout reads `bounced` and `liquidBalanceFor` rather than `balance` (§11.2), and optionally offers **secured mode** for high-volume accounts: the beneficiary signs a `setCustomHardDepositTimeout` for that client's chequebook, the client diff --git a/src/batch.rs b/src/batch.rs index 360730f..7015ad2 100644 --- a/src/batch.rs +++ b/src/batch.rs @@ -1490,3 +1490,103 @@ impl EthRpc { Ok(None) } } + +// ────────────────────────────────────────────────────────────────────── +// Cashing out (docs/pusher-incentives.md §14 Stage 2) +// ────────────────────────────────────────────────────────────────────── + +sol! { + function cashChequeBeneficiary( + address recipient, + uint256 cumulativePayout, + bytes issuerSig + ) external; +} + +/// What a cheque is actually worth right now, before spending gas on it. +#[derive(Debug, Clone, Copy)] +pub struct CashoutQuote { + /// `cumulative - paidOut(beneficiary)`: what is still unclaimed. + pub requested_plur: U256, + /// `min(requested, liquidBalanceFor(beneficiary))` — what the contract + /// would actually transfer. + pub payable_plur: U256, + /// True when the chequebook cannot cover the whole claim. The cashout + /// still succeeds and takes what is there — `_cashChequeInternal` does + /// not revert — but it sets `bounced` permanently. + pub would_bounce: bool, + pub already_bounced: bool, +} + +/// Price a cheque without sending anything. +/// +/// Worth doing first because gas is spent whether or not the cheque is +/// worth cashing, and a cumulative at or below `paidOut` transfers nothing +/// at all. +pub async fn quote_cashout( + rpc_url: &str, + chequebook: Address, + beneficiary: Address, + cumulative_plur: U256, +) -> Result { + let st = read_chequebook_state(rpc_url, chequebook, beneficiary).await?; + let requested_plur = cumulative_plur.saturating_sub(st.paid_out_to_us); + let payable_plur = requested_plur.min(st.liquid_for_us); + Ok(CashoutQuote { + requested_plur, + payable_plur, + would_bounce: payable_plur < requested_plur, + already_bounced: st.bounced, + }) +} + +/// Present a cheque on-chain. +/// +/// **Must be sent by the beneficiary**: `cashChequeBeneficiary` passes +/// `msg.sender` as the beneficiary into `_cashChequeInternal`, so the +/// signing key here is the EOA the cheques were made out to — never the +/// relay's, which is why this is a separate command run somewhere else +/// (§6). `recipient` is where the BZZ lands and may differ. +pub async fn cash_cheque( + beneficiary_signer: &PrivateKeySigner, + rpc_url: &str, + chain_id: u64, + chequebook: Address, + recipient: Address, + cumulative_plur: U256, + signature: &[u8; 65], + receipt_timeout: std::time::Duration, +) -> Result { + // The contract verifies through OpenZeppelin's ECDSA, which rejects + // high-s and v ∉ {27,28}. A non-canonical signature reverts and burns + // the gas, so refuse it here where it costs nothing. + crate::signer::check_canonical_signature(signature) + .map_err(|e| BatchError::Rpc(format!("stored cheque is not cashable: {e}")))?; + let rpc = EthRpc::new(rpc_url.to_string()); + let call = cashChequeBeneficiaryCall { + recipient, + cumulativePayout: cumulative_plur, + issuerSig: signature.to_vec().into(), + } + .abi_encode(); + let tx = rpc + .send_signed(beneficiary_signer, chain_id, chequebook, &call) + .await?; + rpc.wait_for_success(tx, receipt_timeout).await?; + Ok(tx) +} + +#[cfg(test)] +mod cashout_tests { + use super::*; + + #[test] + fn the_cashout_selector_matches_the_contract() { + use alloy_sol_types::SolCall; + let want: [u8; 32] = ::digest( + "cashChequeBeneficiary(address,uint256,bytes)".as_bytes(), + ) + .into(); + assert_eq!(cashChequeBeneficiaryCall::SELECTOR, want[..4]); + } +} diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index ac83a84..608d658 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -802,6 +802,50 @@ enum Commands { action: BatchAction, }, + /// Cash cheques a metered relay has accepted. + /// + /// Reads the relay's ledger, prices each held cheque on-chain, and + /// presents the ones worth collecting. **Run this somewhere other than + /// the relay**: `cashChequeBeneficiary` must be sent *by* the + /// beneficiary, so it needs the beneficiary's private key — which is + /// exactly the key a relay box is designed never to hold. Copy the + /// ledger file (or point `--state-dir` at a snapshot of it) and run + /// this from a machine that does. + /// + /// A cheque is cumulative, so only the newest one per chequebook is + /// ever presented; gas is paid once per chequebook, not once per + /// cheque received. + #[cfg(unix)] + Cashout { + #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + rpc_url: String, + /// The **beneficiary's** private key — the EOA cheques were made + /// out to, and the only address the contract will pay. + #[arg(long, value_name = "KEY")] + key: String, + /// Relay state directory containing `ledger.json`. + #[arg(long, value_name = "DIR")] + state_dir: std::path::PathBuf, + /// Where the BZZ should land. Defaults to the beneficiary. + #[arg(long, value_name = "ADDR")] + recipient: Option, + /// Skip cheques worth less than this in BZZ. Cashing costs ~300k + /// gas whatever the amount, so tiny cheques are worth less than + /// collecting them (§9.3). + #[arg(long, default_value = "0.25", value_name = "BZZ")] + min_amount: String, + /// Only one chequebook, rather than everything in the ledger. + #[arg(long, value_name = "ADDR")] + chequebook: Option, + /// Price everything and print it, sending nothing. + #[arg(long)] + dry_run: bool, + #[arg(long, default_value_t = 100, value_name = "ID")] + chain_id: u64, + #[arg(long, default_value_t = 300, value_name = "SECS")] + timeout: u64, + }, + /// Manage a SWAP chequebook — the contract that pays metered pusher /// relays (`docs/pusher-incentives.md`). /// @@ -2771,6 +2815,98 @@ async fn main() -> Result<(), Box> { #[cfg(unix)] #[cfg(unix)] + Commands::Cashout { + rpc_url, + key, + state_dir, + recipient, + min_amount, + chequebook, + dry_run, + chain_id, + timeout, + } => { + let signer = parse_signer(&key)?; + let beneficiary = signer.address(); + let recipient: alloy_primitives::Address = match recipient { + Some(r) => r.parse().map_err(|e| format!("--recipient: {e}"))?, + None => beneficiary, + }; + let floor = parse_bzz_amount(&min_amount)?; + let only: Option = match chequebook { + Some(c) => Some(c.parse().map_err(|e| format!("--chequebook: {e}"))?), + None => None, + }; + let ledger_path = state_dir.join("ledger.json"); + let ledger = hoverfly::ledger::Ledger::load_or_create(&ledger_path) + .map_err(|e| format!("reading {}: {e}", ledger_path.display()))?; + let held = ledger.held_cheques(); + println!( + "beneficiary 0x{} recipient 0x{} ({} cheque(s) held)", + hex::encode(beneficiary), + hex::encode(recipient), + held.len() + ); + if held.is_empty() { + println!("nothing to cash"); + return Ok(()); + } + let mut cashed = 0usize; + let mut skipped = 0usize; + for (account, cb, cheque) in held { + let cb_addr = alloy_primitives::Address::from(cb); + if only.is_some_and(|o| o != cb_addr) { + continue; + } + let cumulative = alloy_primitives::U256::from(cheque.cumulative_plur); + let q = hoverfly::batch::quote_cashout( + &rpc_url, + cb_addr, + alloy_primitives::Address::from(beneficiary), + cumulative, + ) + .await?; + println!(); + println!("chequebook 0x{} (account 0x{})", hex::encode(cb), hex::encode(account)); + println!(" cumulative {cheque_cumulative}", cheque_cumulative = cheque.cumulative_plur); + println!(" unclaimed {}", q.requested_plur); + println!(" payable now {}{}", q.payable_plur, + if q.would_bounce { " <- chequebook cannot cover the claim" } else { "" }); + if q.already_bounced { + println!(" NOTE: this chequebook has bounced before"); + } + if cheque.signature == [0u8; 65] { + println!(" SKIP: no stored signature (ledger predates signature persistence)"); + skipped += 1; + continue; + } + if q.payable_plur < floor { + println!(" SKIP: below --min-amount ({min_amount} BZZ); gas would cost more than this collects"); + skipped += 1; + continue; + } + if dry_run { + println!(" DRY RUN: would cash {}", q.payable_plur); + continue; + } + let tx = hoverfly::batch::cash_cheque( + &signer, + &rpc_url, + chain_id, + cb_addr, + recipient, + cumulative, + &cheque.signature, + std::time::Duration::from_secs(timeout), + ) + .await?; + println!(" CASHED tx 0x{}", hex::encode(tx)); + cashed += 1; + } + println!(); + println!("{cashed} cashed, {skipped} skipped"); + } + #[cfg(unix)] Commands::Chequebook { action } => match action { ChequebookAction::Deploy { rpc_url, diff --git a/src/ledger.rs b/src/ledger.rs index e637a89..834a555 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -75,7 +75,20 @@ struct Account { owed_plur: u128, /// Deliberately absent from the on-disk form. See the module docs. reserved_plur: u128, - last_cumulative: HashMap<[u8; 20], u128>, + last_cumulative: HashMap<[u8; 20], HeldCheque>, +} + +/// The latest cheque accepted from one chequebook. +/// +/// **The signature has to be kept.** Cheques are cumulative, so only the +/// newest one is ever worth presenting (§7.2) — but without its signature +/// the relay holds a number it can prove nothing about and can never cash. +/// The first cut stored only the cumulative, which made the whole ledger a +/// record of money that could not be collected. +#[derive(Debug, Clone, Copy)] +pub struct HeldCheque { + pub cumulative_plur: u128, + pub signature: [u8; 65], } impl Account { @@ -110,7 +123,9 @@ struct OnDisk { struct OnDiskAccount { account: String, owed_plur: String, - last_cumulative: Vec<(String, String)>, + /// `(chequebook, cumulative, signature_hex)`. Version 1 files carry no + /// signature; they load, but nothing in them can be cashed. + last_cumulative: Vec<(String, String, String)>, } impl Ledger { @@ -167,8 +182,24 @@ impl Ledger { for a in disk.accounts { let key = parse_addr(&a.account)?; let mut last_cumulative = HashMap::new(); - for (cb, v) in a.last_cumulative { - last_cumulative.insert(parse_addr(&cb)?, parse_u128(&v)?); + for (cb, v, sig_hex) in a.last_cumulative { + let raw = hex::decode(sig_hex.trim_start_matches("0x")) + .map_err(|e| StoreError::Io(format!("cheque signature hex: {e}")))?; + // A v1 entry has no signature. Keep the cumulative — losing + // it would let the client replay (§11.4) — but leave the + // signature zeroed so cashout skips it rather than + // submitting something the contract will reject. + let mut signature = [0u8; 65]; + if raw.len() == 65 { + signature.copy_from_slice(&raw); + } + last_cumulative.insert( + parse_addr(&cb)?, + HeldCheque { + cumulative_plur: parse_u128(&v)?, + signature, + }, + ); } accounts.insert( key, @@ -205,10 +236,16 @@ impl Ledger { .iter() .filter(|(_, a)| a.owed_plur > 0 || !a.last_cumulative.is_empty()) .map(|(k, a)| { - let mut last: Vec<(String, String)> = a + let mut last: Vec<(String, String, String)> = a .last_cumulative .iter() - .map(|(cb, v)| (hex::encode(cb), v.to_string())) + .map(|(cb, c)| { + ( + hex::encode(cb), + c.cumulative_plur.to_string(), + hex::encode(c.signature), + ) + }) .collect(); last.sort(); OnDiskAccount { @@ -227,7 +264,7 @@ impl Ledger { binding.sort(); let disk = OnDisk { - version: 1, + version: 2, secret_hex: hex::encode(self.secret), accounts, binding, @@ -255,10 +292,23 @@ impl Ledger { self.accounts .get(account) .and_then(|a| a.last_cumulative.get(chequebook)) - .copied() + .map(|c| c.cumulative_plur) .unwrap_or(0) } + /// Every cheque the relay holds, newest per chequebook. This is what + /// `hoverfly cashout` presents on-chain. + pub fn held_cheques(&self) -> Vec<([u8; 20], [u8; 20], HeldCheque)> { + let mut out = Vec::new(); + for (account, a) in &self.accounts { + for (cb, held) in &a.last_cumulative { + out.push((*account, *cb, *held)); + } + } + out.sort_by_key(|(_, cb, _)| *cb); + out + } + /// Number of accounts holding a live reservation — the cardinality /// §7.2 says to bound, since the map is attacker-influenced. pub fn live_reservations(&self) -> usize { @@ -312,6 +362,7 @@ impl Ledger { account: [u8; 20], chequebook: [u8; 20], cumulative_plur: u128, + signature: [u8; 65], ) -> Result { if cumulative_plur > MAX_CUMULATIVE_PLUR { return Err(LedgerError::Absurd(cumulative_plur)); @@ -325,7 +376,11 @@ impl Ledger { _ => {} } let a = self.accounts.entry(account).or_default(); - let have = a.last_cumulative.get(&chequebook).copied().unwrap_or(0); + let have = a + .last_cumulative + .get(&chequebook) + .map(|c| c.cumulative_plur) + .unwrap_or(0); if cumulative_plur <= have { return Err(LedgerError::NotIncreasing { got: cumulative_plur, @@ -342,7 +397,13 @@ impl Ledger { owed: a.owed_plur, }); } - a.last_cumulative.insert(chequebook, cumulative_plur); + a.last_cumulative.insert( + chequebook, + HeldCheque { + cumulative_plur, + signature, + }, + ); a.owed_plur -= delta; self.binding.insert(chequebook, account); Ok(delta) @@ -419,9 +480,9 @@ mod tests { fn a_cheque_credits_only_the_delta() { let mut l = Ledger::ephemeral(); l.commit(A, 0, 1000); - assert_eq!(l.credit(A, CB, 400).expect("first"), 400); + assert_eq!(l.credit_test(A, CB, 400).expect("first"), 400); assert_eq!(l.owed(&A), 600); - assert_eq!(l.credit(A, CB, 900).expect("second"), 500); + assert_eq!(l.credit_test(A, CB, 900).expect("second"), 500); assert_eq!(l.owed(&A), 100); } @@ -430,9 +491,9 @@ mod tests { fn a_re_presented_cheque_credits_nothing() { let mut l = Ledger::ephemeral(); l.commit(A, 0, 1000); - l.credit(A, CB, 400).expect("first"); + l.credit_test(A, CB, 400).expect("first"); assert_eq!( - l.credit(A, CB, 400), + l.credit_test(A, CB, 400), Err(LedgerError::NotIncreasing { got: 400, have: 400 @@ -445,10 +506,10 @@ mod tests { fn a_chequebook_cannot_move_between_accounts() { let mut l = Ledger::ephemeral(); l.commit(A, 0, 1000); - l.credit(A, CB, 100).expect("bind to A"); + l.credit_test(A, CB, 100).expect("bind to A"); l.commit(B, 0, 1000); assert!(matches!( - l.credit(B, CB, 500), + l.credit_test(B, CB, 500), Err(LedgerError::ChequebookBound { .. }) )); } @@ -458,14 +519,14 @@ mod tests { let mut l = Ledger::ephemeral(); l.commit(A, 0, 100); assert!(matches!( - l.credit(A, CB, MAX_CUMULATIVE_PLUR + 1), + l.credit_test(A, CB, MAX_CUMULATIVE_PLUR + 1), Err(LedgerError::Absurd(_)) )); assert!(matches!( - l.credit(A, CB, 101), + l.credit_test(A, CB, 101), Err(LedgerError::Overpayment { got: 101, owed: 100 }) )); - l.credit(A, CB, 100).expect("paying exactly what is owed is fine"); + l.credit_test(A, CB, 100).expect("paying exactly what is owed is fine"); assert_eq!(l.owed(&A), 0); } @@ -480,7 +541,7 @@ mod tests { let secret = { let mut l = Ledger::load_or_create(&path).expect("create"); l.commit(A, 0, 5000); - l.credit(A, CB, 1200).expect("pay"); + l.credit_test(A, CB, 1200).expect("pay"); l.reserve(A, 900, 100_000); l.persist().expect("persist"); *l.secret() @@ -512,12 +573,12 @@ mod tests { { let mut l = Ledger::load_or_create(&path).expect("create"); l.commit(A, 0, 5000); - l.credit(A, CB, 1200).expect("pay"); + l.credit_test(A, CB, 1200).expect("pay"); l.persist().expect("persist"); } let mut l = Ledger::load_or_create(&path).expect("reload"); assert!( - matches!(l.credit(A, CB, 1200), Err(LedgerError::NotIncreasing { .. })), + matches!(l.credit_test(A, CB, 1200), Err(LedgerError::NotIncreasing { .. })), "re-presenting the same cheque after a restart must credit nothing" ); let _ = std::fs::remove_file(&path); @@ -531,13 +592,13 @@ mod tests { { let mut l = Ledger::load_or_create(&path).expect("create"); l.commit(A, 0, 500); - l.credit(A, CB, 100).expect("bind"); + l.credit_test(A, CB, 100).expect("bind"); l.persist().expect("persist"); } let mut l = Ledger::load_or_create(&path).expect("reload"); l.commit(B, 0, 500); assert!( - matches!(l.credit(B, CB, 200), Err(LedgerError::ChequebookBound { .. })), + matches!(l.credit_test(B, CB, 200), Err(LedgerError::ChequebookBound { .. })), "the chequebook binding must survive a restart" ); let _ = std::fs::remove_file(&path); @@ -576,7 +637,7 @@ mod leak_tests { let mut l = Ledger::ephemeral(); l.commit(A, 0, 1000); l.reserve(A, 500, 100_000); - l.credit(A, CB, 1000).expect("pay off the debt"); + l.credit_test(A, CB, 1000).expect("pay off the debt"); assert_eq!(l.owed(&A), 0, "the debt is cleared"); assert_eq!( l.reserved(&A), @@ -598,7 +659,7 @@ mod leak_tests { // There is no debt to pay, so no cheque exists that could help. assert_eq!(l.owed(&A), 0); assert!( - matches!(l.credit(A, CB, 1), Err(LedgerError::Overpayment { .. })), + matches!(l.credit_test(A, CB, 1), Err(LedgerError::Overpayment { .. })), "with nothing owed, a cheque cannot clear the overshoot" ); // Only releasing does. @@ -630,3 +691,19 @@ mod leak_tests { assert_eq!(l.live_reservations(), 0); } } + +#[cfg(test)] +impl Ledger { + /// Test shim: cheques in unit tests carry a dummy signature, since the + /// ledger never inspects it — only `hoverfly cashout` does. + fn credit_test( + &mut self, + account: [u8; 20], + chequebook: [u8; 20], + cumulative_plur: u128, + ) -> Result { + let mut sig = [0u8; 65]; + sig[64] = 27; + self.credit(account, chequebook, cumulative_plur, sig) + } +} diff --git a/src/metered.rs b/src/metered.rs index 5d80a30..58c2537 100644 --- a/src/metered.rs +++ b/src/metered.rs @@ -278,9 +278,10 @@ impl Metered { account: [u8; 20], chequebook: [u8; 20], cumulative: u128, + signature: [u8; 65], ) -> Result { let mut l = self.ledger.lock().expect("ledger poisoned"); - let accepted = l.credit(account, chequebook, cumulative)?; + let accepted = l.credit(account, chequebook, cumulative, signature)?; // Persist immediately: the window between accepting a cheque and // durably recording its cumulative is exactly §11.4's replay hole. if let Err(e) = l.persist() { @@ -722,14 +723,14 @@ mod lifecycle_tests { let first = p.price_bytes(40 * 1024 * 1024); m.ledger.lock().unwrap().commit(ACCT, 0, first); - let accepted = m.credit(ACCT, CB, first).expect("first cheque"); + let accepted = m.credit(ACCT, CB, first, [27u8; 65]).expect("first cheque"); assert_eq!(accepted, first); assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); let second = p.price_bytes(32 * 1024 * 1024); m.ledger.lock().unwrap().commit(ACCT, 0, second); // A cumulative cheque: the *total*, not the delta. - let accepted = m.credit(ACCT, CB, first + second).expect("second cheque"); + let accepted = m.credit(ACCT, CB, first + second, [27u8; 65]).expect("second cheque"); assert_eq!(accepted, second, "only the new debt is credited"); assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); } @@ -780,7 +781,7 @@ mod lifecycle_tests { owed >= p.min_cheque_plur, "what is owed must clear the dust floor, or there is no exit" ); - m.credit(ACCT, CB, owed).expect("a cheque for exactly what is owed"); + m.credit(ACCT, CB, owed, [27u8; 65]).expect("a cheque for exactly what is owed"); assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); assert!( !m.reserve_for_body(ACCT, 4251, cap).over_cap, diff --git a/src/pusher.rs b/src/pusher.rs index e5a2130..7cc9df3 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -943,7 +943,12 @@ async fn pay_response(state: Arc, req: Request) -> "chequebook cannot cover this cheque", ); } - match m.credit(verified.account, cheque.chequebook, cumulative) { + match m.credit( + verified.account, + cheque.chequebook, + cumulative, + cheque.signature, + ) { Ok(accepted) => { // We just consumed part of what that balance covered; the next // cheque should not be judged against the pre-credit reading. From 4ca9fe696314c6cf5e4f147f51ffe71af305bd33 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 7 Aug 2026 18:37:06 +0300 Subject: [PATCH 07/27] fix(metered): dedup acks are marked, so client and relay stay in step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `refund_dedup` existed, was tested, and was never called — which was not merely dead code but a live divergence. The relay bills `admitted - dedup` (§8.2), while a dedup ack was indistinguishable from a real push, so a paying client counted bytes it was never charged for. Its `owed` drifted above the relay's and the next cheque was refused as an overpayment, jamming settlement. It fires whenever a client re-POSTs inside the 120s recent-ack window — which the design's own retry model does. The relay now marks the ack `"dedup": true` and the client subtracts those bytes. The claim only ever lowers what is owed, so taking it at face value is safe. Also lets a client shed debt the relay disclaims. A POST whose completion the client never saw is charged locally but was never billed remotely; a cheque for it is refused every time, and carrying it forever only eats the client's own headroom. On "nothing owed" the client now drops it. Verified: two uploads of the same file, the second entirely served from the relay's cache. Relay billed 164 of 328 KiB; both sides agreed at 78,720,000,000 with zero residual. Before the fix the client would have tried to pay double. Adds env fallbacks (HOVERFLY_METER*) so a deployed relay can be switched to metered without changing its start command, matching the existing HOVERFLY_PUSH_* pattern. Co-Authored-By: Claude Opus 5 (1M context) --- src/bin/hoverfly.rs | 33 +++++++++++++++++++++++++++++++++ src/client.rs | 35 ++++++++++++++++++++++++++--------- src/payer.rs | 33 +++++++++++++++++++++++++++++++++ src/pusher.rs | 21 ++++++++++++--------- 4 files changed, 104 insertions(+), 18 deletions(-) diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index 608d658..9071682 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -2338,6 +2338,39 @@ async fn main() -> Result<(), Box> { // here and in `build_metered`, and a failure refuses to start // rather than serving a half-configured meter that a paying // client would discover the hard way. + // Env fallbacks so a deployed relay can be switched to metered + // without changing its start command — the same pattern + // HOVERFLY_PUSH_POOL and HOVERFLY_PUSHER_IDENTITY already use, + // and the only lever available on hosts where the command line + // is fixed by the platform. + let envs = |k: &str| std::env::var(k).ok().filter(|s| !s.trim().is_empty()); + let meter = meter || envs("HOVERFLY_METER").is_some_and(|v| v != "0"); + let meter_hard = + meter_hard || envs("HOVERFLY_METER_HARD").is_some_and(|v| v != "0"); + let origin = if origin.is_empty() { + envs("HOVERFLY_METER_ORIGIN") + .map(|v| v.split(',').map(|s| s.trim().to_string()).collect()) + .unwrap_or_default() + } else { + origin + }; + let beneficiary = beneficiary.or_else(|| envs("HOVERFLY_METER_BENEFICIARY")); + let state_dir = state_dir.or_else(|| envs("HOVERFLY_METER_STATE_DIR").map(Into::into)); + let parse_env_plur = |k: &str| -> Result, String> { + match envs(k) { + Some(v) => v.parse().map(Some).map_err(|e| format!("{k}: {e}")), + None => Ok(None), + } + }; + let price_plur_per_kib = + price_plur_per_kib.or(parse_env_plur("HOVERFLY_METER_PRICE_PLUR_PER_KIB")?); + let min_cheque_plur = + min_cheque_plur.or(parse_env_plur("HOVERFLY_METER_MIN_CHEQUE_PLUR")?); + let settle_every_plur = + settle_every_plur.or(parse_env_plur("HOVERFLY_METER_SETTLE_EVERY_PLUR")?); + let max_outstanding_plur = + max_outstanding_plur.or(parse_env_plur("HOVERFLY_METER_MAX_OUTSTANDING_PLUR")?); + let credit_ratio = credit_ratio.or(parse_env_plur("HOVERFLY_METER_CREDIT_RATIO")?); let meter_opts = if meter { let mut params = hoverfly::meter::Params::default(); if let Some(v) = price_plur_per_kib { diff --git a/src/client.rs b/src/client.rs index 09566a7..46e0c12 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3037,6 +3037,7 @@ where acked, elapsed_ms, outcome, + dedup_bytes, } => { sched.on_batch_timing(lane, acked, elapsed_ms); let needs_payment = @@ -3058,6 +3059,9 @@ where let reached_relay = !matches!(outcome, crate::pushsched::BatchOutcome::PaymentRequired); payer.account.record_answered(sent, reached_relay); + if reached_relay && dedup_bytes > 0 { + payer.account.refund_dedup(dedup_bytes); + } } sched.on_batch_result(batch, outcome, now_ms()); // Settle when the relay asks (402) or when we have crossed @@ -3264,6 +3268,9 @@ enum LaneEv { acked: usize, elapsed_ms: u64, outcome: crate::pushsched::BatchOutcome, + /// Body bytes the relay served from its recent-ack cache and billed + /// at zero (§8.2), so the client can subtract them and stay in step. + dedup_bytes: u64, }, } @@ -3293,13 +3300,15 @@ async fn post_batch_streaming( let body = crate::pushframe::encode_batch(batch); let mut acked = 0usize; - let finish = |outcome: BatchOutcome, acked: usize| { + let mut dedup_bytes: u64 = 0; + let finish = |outcome: BatchOutcome, acked: usize, dedup_bytes: u64| { let _ = tx.send(LaneEv::Done { batch: batch_id, lane, acked, elapsed_ms: t0.elapsed().as_millis() as u64, outcome, + dedup_bytes, }); }; @@ -3311,7 +3320,7 @@ async fn post_batch_streaming( Ok(r) => r, Err(e) => { warn!(target: "hoverfly::upload", "lane {push_url} POST failed: {e}"); - finish(BatchOutcome::Failed(e.to_string()), 0); + finish(BatchOutcome::Failed(e.to_string()), 0, 0); return; } }; @@ -3326,18 +3335,18 @@ async fn post_batch_streaming( if code == reqwest::StatusCode::PAYMENT_REQUIRED { info!(target: "hoverfly::upload", "lane {push_url} requires payment: {}", txt.trim()); - finish(BatchOutcome::PaymentRequired, 0); + finish(BatchOutcome::PaymentRequired, 0, 0); return; } warn!(target: "hoverfly::upload", "lane {push_url} rejected batch ({code}): {}", txt.trim()); - finish(BatchOutcome::Failed(format!("http {code}")), 0); + finish(BatchOutcome::Failed(format!("http {code}")), 0, 0); return; } let mut stream = resp.bytes_stream(); let mut buf = Vec::new(); - let handle = |line: &[u8], acked: &mut usize| { + let handle = |line: &[u8], acked: &mut usize, dedup_bytes: &mut u64| { let Ok(v) = serde_json::from_slice::(line) else { return; }; @@ -3357,6 +3366,14 @@ async fn post_batch_streaming( let po = v.get("po").and_then(|p| p.as_u64()).unwrap_or(0) as u8; let best_po = v.get("bpo").and_then(|p| p.as_u64()).unwrap_or(0) as u8; let shallow = v.get("shallow").and_then(|s| s.as_bool()).unwrap_or(false); + // The relay served this from its recent-ack cache and billed zero + // for it (§8.2). Subtract the same bytes here or our total drifts + // above the relay's and the next cheque is refused. + if v.get("dedup").and_then(|d| d.as_bool()).unwrap_or(false) + && let Some(c) = batch.iter().find(|c| c.addr == addr) + { + *dedup_bytes += (crate::pushframe::HEADER_LEN + c.wire.len()) as u64; + } let _ = tx.send(LaneEv::Ack { lane, addr, @@ -3373,20 +3390,20 @@ async fn post_batch_streaming( buf.extend_from_slice(&bytes); while let Some(nl) = buf.iter().position(|&b| b == b'\n') { let line: Vec = buf.drain(..=nl).collect(); - handle(&line[..line.len() - 1], &mut acked); + handle(&line[..line.len() - 1], &mut acked, &mut dedup_bytes); } } Err(e) => { warn!(target: "hoverfly::upload", "lane {push_url} stream error: {e}"); - finish(BatchOutcome::Failed(e.to_string()), acked); + finish(BatchOutcome::Failed(e.to_string()), acked, dedup_bytes); return; } } } if !buf.is_empty() { - handle(&buf, &mut acked); + handle(&buf, &mut acked, &mut dedup_bytes); } - finish(BatchOutcome::Answered, acked); + finish(BatchOutcome::Answered, acked, dedup_bytes); } /// Read a lane's `/v1/status` advertisement. diff --git a/src/payer.rs b/src/payer.rs index a6904ac..21d97d3 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -383,6 +383,14 @@ impl LaneAccount { Some(self.cumulative_plur.saturating_add(self.owed_plur)) } + /// Drop debt the relay will not accept. See the caller in + /// [`LanePayer::settle`] — this is a divergence artifact, not a + /// discount, and it can only ever move in the client's favour by + /// removing an obligation the counterparty has already disclaimed. + pub fn forgive_phantom_debt(&mut self) { + self.owed_plur = 0; + } + /// Call once a cheque for `cumulative` has been accepted. pub fn settled(&mut self, cumulative: u128) { let credited = cumulative.saturating_sub(self.cumulative_plur); @@ -655,6 +663,16 @@ impl LanePayer { if !resp.status().is_success() { let code = resp.status(); let text = resp.text().await.unwrap_or_default(); + // The relay's ledger is authoritative for what it will accept. + // If it says nothing is owed, our extra is an artifact — bytes + // we charged ourselves for a POST whose completion we never + // saw, and which the relay therefore never billed. Carrying it + // forever would only eat our own headroom, since a cheque for + // it is refused every time. + if text.contains("nothing owed") { + self.account.forgive_phantom_debt(); + return Ok(None); + } return Err(format!("pay {code}: {}", text.trim())); } // Record the cumulative *before* trusting the reply: we have @@ -938,6 +956,21 @@ mod tests { assert_eq!(a.owed(), 0); } + /// A client can charge itself for a POST whose completion it never + /// saw; the relay never billed it, so it refuses the cheque. Carrying + /// that debt forever would slowly eat the client's own credit line. + #[test] + fn phantom_debt_can_be_dropped() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(4251); + a.record_answered(4251, true); + assert!(a.owed() > 0); + a.forgive_phantom_debt(); + assert_eq!(a.owed(), 0); + assert_eq!(a.outstanding(), 0, "and it stops holding headroom"); + } + #[test] fn dedup_hits_are_refunded() { let p = Params::default(); diff --git a/src/pusher.rs b/src/pusher.rs index 7cc9df3..00a8ffb 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -1562,6 +1562,17 @@ async fn run_push( } send_line(&v); }; + let ack_dedup = |addr: &[u8; 32]| { + // A dedup hit did no push work and is billed at zero (§8.2). Say so + // explicitly: without a marker it is indistinguishable from a real + // push, so a paying client counts bytes the relay never charged + // for, and its next cheque is refused as an overpayment. The claim + // only ever *lowers* what is owed, so it is safe for the client to + // take at face value. + send_line(&serde_json::json!({ + "a": hex::encode(addr), "s": "ok", "po": 0, "ms": 0, "dedup": true + })); + }; let ack_ok = |addr: &[u8; 32], info: crate::client::PushInfo| { // `po` is the proximity order of the peer whose receipt we took — // i.e. how deep into the chunk's own neighborhood it actually @@ -1672,15 +1683,7 @@ async fn run_push( dedup_hits += 1; tally.dedup(key, batch_value, frame_bytes); billable_bytes = billable_bytes.saturating_sub(frame_bytes); - ack_ok( - &chunk.addr, - crate::client::PushInfo { - po: 0, - ms: 0, - shallow: false, - best_po: 0, - }, - ); + ack_dedup(&chunk.addr); } else { batch_of.insert(chunk.addr, batch_id); accepted.push(chunk); From 43e0097262bc27cb52210bfa535c1a2de5a8ee57 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sat, 8 Aug 2026 21:13:11 +0300 Subject: [PATCH 08/27] fix(metered): pay debt the relay carried across sessions, and stop dispatching POSTs the credit line cannot hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running a metered relay on a real host with a persistent disk and uploading to it twice. **The relay's ledger outlives the client's.** A run ends leaving the sub-dust residual unpaid — deliberately, since a cheque for it would be refused — but the relay keeps counting it against the credit line while the next client process starts believing it owes nothing. The second upload was refused on its first POST for 290,400,000,000 PLUR it could not compute a cheque for, and failed 151/151. With the shipped defaults the residual is under 3.9e12 against a 62.2e12 cap, so it takes ~16 runs to deadlock rather than 2; the tight test params only made it prompt. The recovery is to ask. `/v1/account` already reports the relay's own `owed`, so a 402 the client cannot pay now triggers a reconcile and pays what the relay says. Deliberately *not* the `outstanding_plur` in the 402 body: `reserve()` adds the reservation before computing it, so that figure includes the body just refused and adopting it over-pays by exactly that amount — the next cheque then bounces as an overpayment. Reservations are excluded on our side too, being already counted as `pending`. The figure is bounded by the chequebook balance rather than the credit line, because debt reaches the line by construction and an operator who lowers it leaves real debt above it. **The headroom guard admitted a frame and then sent a batch.** It asked whether one more *frame* fit, then dispatched a full `batch_max` POST, and `dispatch_ok` was computed once for an unbounded run of them. Several POSTs were each waved through against the same headroom, the relay reserved all of them, and it 402'd a client whose books said it had room — a refusal nothing could pay, since the bytes were still in flight. The batch then came back having spent an attempt per chunk. The guard now prices the POST the scheduler will actually build and is re-evaluated per dispatch; where the line holds only one POST this serialises the lane, which is the honest answer. `LaneAccount::max_body_bytes` had the same confusion, sizing against `owed` while its siblings bound on `outstanding`. It is only reached from tests today, but it would have reintroduced the bug at its next caller. Verified against a metered hard-mode relay on an amd64 VPS: 402s per upload 11 -> 1, unpayable refusals 6 -> 0, five uploads of 512 KiB settling to `owed: 0` on both sides, and the relay's held cheque redeemed on Gnosis from a separate machine (tx 0x9d4a8b29..., paidOut now equals the cheque's cumulative exactly). Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 5 ++ src/client.rs | 57 ++++++++++++--- src/payer.rs | 195 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 244 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 731cf3d..d1dd468 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ peers.json # Overlay identity nonce written to CWD by CLI runs (default `--nonce-file`). # A runtime artifact, like peers.json — never commit it. overlay-nonce +# Cumulative payouts issued to bee peers and metered relays, written to CWD +# by paid uploads. Runtime money state: committing it would publish who was +# paid what, and restoring a stale copy re-issues cheques the counterparty +# has already banked. +cheques.json # `peers.seed.json` is checked in: an IP-diverse cold-start seed # harvested from a long-running daemon (~800 peers across ~800 unique # /32 IPs as of the last refresh; regenerate via diff --git a/src/client.rs b/src/client.rs index 46e0c12..81e57f7 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2786,6 +2786,9 @@ where "lane {i}: credit line allows {fits} frames/POST (was {before})"); infos[i].batch_max = Some(fits); } + // The headroom guard admits whole POSTs, so it needs to know how + // big one is on this lane. + payer.set_post_frames(fits.min(before)); } } // batch id -> body bytes, so a completed POST can be billed for what @@ -2868,13 +2871,19 @@ where } } } - let dispatch_ok = payment.is_none() - || payers - .iter() - .any(|p| p.as_ref().map(|x| x.has_headroom()).unwrap_or(true)); + // Re-checked on every pass of the loop below, not once before it. + // Each dispatch adds its body to `pending`, so a single up-front + // answer authorises an unbounded run of POSTs against headroom that + // only the first of them actually had. + let dispatch_ok = |payers: &[Option]| { + payment.is_none() + || payers + .iter() + .any(|p| p.as_ref().map(|x| x.has_headroom()).unwrap_or(true)) + }; // Hand out everything the scheduler is willing to dispatch. - while dispatch_ok && let Some(a) = sched.next(now_ms()) { + while dispatch_ok(&payers) && let Some(a) = sched.next(now_ms()) { let batch: Vec = a .chunks .iter() @@ -3083,9 +3092,41 @@ where Ok(None) if needs_payment => { // 402 with nothing owed above the dust floor // means the two sides disagree about the - // ledger. Retrying cannot fix that. - warn!(target: "hoverfly::upload", - "lane {lane}: 402 but nothing is owed — ledger disagreement"); + // ledger, and retrying the POST cannot fix it. + // Almost always this is debt carried from an + // earlier run: the relay's ledger is durable + // and ours is not, so it is still counting a + // residual we settled below the dust floor and + // then forgot. Ask what it thinks we owe and + // pay that. + match payer.reconcile(&http, pc).await { + Ok(true) => match payer.settle(&http, pc).await { + Ok(Some(c)) => { + info!(target: "hoverfly::upload", + "lane {lane}: reconciled with relay ledger, \ + paid cumulative {c}"); + sched.fund_lane(lane); + } + Ok(None) => { + warn!(target: "hoverfly::upload", + "lane {lane}: relay reports debt below its own \ + dust floor yet refuses service"); + } + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {lane}: payment after reconcile failed: {e}"); + } + }, + Ok(false) => { + warn!(target: "hoverfly::upload", + "lane {lane}: 402 but neither side reports debt \ + — ledger disagreement"); + } + Err(e) => { + warn!(target: "hoverfly::upload", + "lane {lane}: reconcile failed: {e}"); + } + } } Ok(None) => {} Err(e) => { diff --git a/src/payer.rs b/src/payer.rs index 21d97d3..f7fe41d 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -391,6 +391,36 @@ impl LaneAccount { self.owed_plur = 0; } + /// Adopt a larger debt figure the relay reports for us. + /// + /// The mirror of [`Self::forgive_phantom_debt`], and the half that is + /// load-bearing across *sessions*. The relay's ledger is durable and + /// ours is not: every run ends leaving the sub-dust residual unpaid + /// (see the final settle in `drive_pushers`), and the relay keeps + /// counting it against the credit line while a fresh client process + /// starts believing it owes nothing. Enough runs of that and the + /// account is over its cap with a client that cannot compute a cheque + /// to clear it — refused forever, having genuinely incurred the debt. + /// + /// `relay_owed` must be the relay's *owed*, never the `outstanding` in + /// a 402 body: that one is quoted with the just-refused reservation + /// already added, so adopting it over-pays by exactly the body that was + /// turned away and the next cheque is rejected as an overpayment. + /// Reservations are excluded for the same reason on our side — bytes + /// still in flight are already counted in `pending_plur`, and would be + /// billed twice when those POSTs land. + /// + /// Only ever raises. An under-count is safe and self-correcting (the + /// next settle picks up the rest); an over-count is a cheque the relay + /// refuses. + pub fn adopt_relay_debt(&mut self, relay_owed: u128) -> bool { + if relay_owed <= self.owed_plur { + return false; + } + self.owed_plur = relay_owed; + true + } + /// Call once a cheque for `cumulative` has been accepted. pub fn settled(&mut self, cumulative: u128) { let credited = cumulative.saturating_sub(self.cumulative_plur); @@ -403,8 +433,17 @@ impl LaneAccount { /// The client sizes its POST to fit rather than discovering the ceiling /// as a 402 — which matters most for exactly the small batches §10.3 /// exists to keep, whose whole credit line is under one full POST. + /// + /// Measured against `outstanding`, not `owed`: the relay reserves each + /// body at admission, so concurrent POSTs hold credit that is not yet + /// debt. Sizing against `owed` alone hands every in-flight POST the + /// whole line as if it were the only one, and with several on the wire + /// their reservations sum past the cap — the relay 402s a client that + /// believes it has headroom, and nothing is owed that paying could + /// clear. `has_headroom` and `would_exceed` already bind on + /// `outstanding`; this is the same quantity. pub fn max_body_bytes(&self, cap_plur: u128) -> u64 { - let headroom = cap_plur.saturating_sub(self.owed_plur); + let headroom = cap_plur.saturating_sub(self.outstanding()); let kib = headroom / self.params.price_plur_per_kib.max(1); (kib.saturating_mul(1024)).min(u64::MAX as u128) as u64 } @@ -482,6 +521,11 @@ pub struct LanePayer { header: Option, header_stale_after: u64, cap_plur: u128, + /// Frames in a full POST to this lane, as the scheduler will actually + /// build it. Set once the lane's `batch_max` is known; see + /// [`Self::has_headroom`] for why the guard has to know the size of the + /// thing it is admitting. + post_frames: usize, } #[cfg(not(target_arch = "wasm32"))] @@ -495,9 +539,16 @@ impl LanePayer { header: None, header_stale_after: 0, cap_plur: 0, + post_frames: 1, } } + /// Tell the lane how large a POST the scheduler will build for it, so + /// the headroom guard can ask about the real body rather than a frame. + pub fn set_post_frames(&mut self, frames: usize) { + self.post_frames = frames.max(1); + } + /// The credit line the relay last told us about, or 0 before the first /// challenge. Used to size POSTs (§7.2). pub fn cap_plur(&self) -> u128 { @@ -587,12 +638,23 @@ impl LanePayer { if self.cap_plur == 0 { return true; } - // One frame is the smallest thing worth dispatching. - let one_frame = self + // Ask about the POST that will actually be built, not about one + // frame. Admitting on "a frame would fit" and then dispatching a + // full batch is how several concurrent POSTs each get waved + // through against the same headroom: the relay reserves every one + // of them, their sum crosses the line, and it answers 402 to a + // client whose own books say it had room. That refusal is not even + // payable — the bytes are in flight, so nothing is owed yet — so + // the batch comes back having spent an attempt per chunk. + // + // When the line is narrow enough to hold only one POST this + // serialises the lane, which is the honest answer: a credit line + // that fits 1.8 POSTs cannot have 8 in flight. + let body = self .quote .params - .price_bytes(crate::pushframe::MAX_FRAME_LEN as u64); - self.account.outstanding().saturating_add(one_frame) <= self.cap_plur + .price_bytes((self.post_frames * crate::pushframe::MAX_FRAME_LEN) as u64); + self.account.outstanding().saturating_add(body) <= self.cap_plur } /// Would dispatching `body_bytes` right now exceed the credit line? @@ -607,6 +669,60 @@ impl LanePayer { > self.cap_plur } + /// Ask the relay what it thinks we owe, and adopt the figure if it is + /// larger than ours. + /// + /// Called only when a 402 arrives that our own books say we cannot pay + /// — the deadlock in [`LaneAccount::adopt_relay_debt`]. `/v1/account` + /// is used rather than the number in the 402 body because the body + /// quotes `owed + reserved` *including the refused request*, which is + /// not a payable amount. + /// + /// Returns whether the debt moved. + pub async fn reconcile( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result { + let header = self.header(http, cfg).await?.to_string(); + let resp = http + .get(format!("{}/v1/account", self.base_url.trim_end_matches('/'))) + .header(crate::metered::CHALLENGE_HEADER, header) + .timeout(std::time::Duration::from_secs(60)) + .send() + .await + .map_err(|e| format!("account fetch: {e}"))?; + if !resp.status().is_success() { + return Err(format!("account: http {}", resp.status())); + } + let v: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("account decode: {e}"))?; + let owed: u128 = v + .get("owed_plur") + .and_then(|x| x.as_str()) + .ok_or("account: missing owed_plur")? + .parse() + .map_err(|e| format!("account: bad owed_plur: {e}"))?; + // Bounded by the chequebook's balance, deliberately *not* by the + // credit line. The line caps what may be newly admitted, not what + // may stand: debt reaches the line by construction, and an operator + // who lowers the line leaves legitimately-incurred debt above it. + // Rejecting on that basis refuses to pay a real bill and keeps the + // deadlock this method exists to break. What genuinely bounds our + // exposure to an inflated figure is the funded balance, which + // `settle` enforces exactly across all lanes (§8.3); this is the + // same ceiling stated early for a legible error. + if owed > cfg.balance_plur { + return Err(format!( + "relay claims {owed} owed, more than the chequebook's {} balance", + cfg.balance_plur + )); + } + Ok(self.account.adopt_relay_debt(owed)) + } + /// Settle if there is enough owed to be worth a cheque. /// /// Returns the amount accepted, or `None` when nothing was owed above @@ -997,6 +1113,31 @@ mod tests { assert!(rich > 512 * 4251, "a full POST fits comfortably"); } + /// Concurrent POSTs hold reservations on the relay before they are + /// debt. Sizing the next body against `owed` alone gave each in-flight + /// POST the whole line, so their reservations summed past the cap and + /// the relay 402'd a client with nothing to pay — an unpayable refusal + /// that only clears by waiting. + #[test] + fn post_size_accounts_for_bytes_already_in_flight() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + let cap = p.credit_line(100_000_000_000_000); + let whole_line = a.max_body_bytes(cap); + // Put half the line on the wire, unanswered: still pending, not owed. + let in_flight = whole_line / 2; + a.record_sent(in_flight); + assert_eq!(a.owed(), 0, "in-flight bytes are not debt yet"); + let next = a.max_body_bytes(cap); + assert!( + next <= whole_line - in_flight, + "sized {next} with {in_flight} already on the wire against a {whole_line} line" + ); + // And the two together must fit, which is the property the relay + // actually enforces. + assert!(a.outstanding().saturating_add(p.price_bytes(next)) <= cap); + } + #[test] fn headroom_shrinks_as_debt_accrues() { let p = Params::default(); @@ -1008,6 +1149,50 @@ mod tests { assert!(a.max_body_bytes(cap) < before, "unpaid debt eats the line"); } + /// The relay's ledger outlives the client's. Every run ends leaving the + /// sub-dust residual unpaid, and a fresh process starts believing it + /// owes nothing — so the relay refuses service for debt the client + /// cannot compute a cheque for. Adopting the relay's figure is the only + /// way out, and it is what the 402 recovery path does. + #[test] + fn carried_debt_from_a_previous_run_is_adoptable() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + // Fresh process: no debt, and nothing to pay with. + assert_eq!(a.owed(), 0); + assert_eq!(a.next_cumulative(), None, "cannot pay what it does not know"); + + let carried = p.min_cheque_plur * 3; + assert!(a.adopt_relay_debt(carried), "relay knows more than we do"); + assert_eq!(a.owed(), carried); + assert_eq!( + a.next_cumulative(), + Some(carried), + "now a cheque clears the refusal" + ); + } + + /// Only ever upward. The downward direction is `forgive_phantom_debt`, + /// which is reached from a rejected cheque — a relay reporting *less* + /// than we think must not silently shrink a debt we are still liable + /// for, and an over-count is a cheque the relay refuses outright. + #[test] + fn adopting_relay_debt_never_lowers_our_own() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + a.record_sent(1024 * 1024); + a.record_answered(1024 * 1024, true); + let owed = a.owed(); + assert!(owed > 0); + + assert!(!a.adopt_relay_debt(owed - 1), "a smaller figure is ignored"); + assert_eq!(a.owed(), owed); + assert!(!a.adopt_relay_debt(0), "and zero especially so"); + assert_eq!(a.owed(), owed); + assert!(a.adopt_relay_debt(owed + 1), "larger still wins"); + assert_eq!(a.owed(), owed + 1); + } + /// N lanes are N claims on one balance. The client must see the sum, or /// the second cheque bounces. #[test] From adf6537fe6541fcb631b866c704585c48811d5c5 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sat, 8 Aug 2026 21:18:21 +0300 Subject: [PATCH 09/27] =?UTF-8?q?docs(incentives):=20=C2=A717=20=E2=80=94?= =?UTF-8?q?=20the=20two=20bugs=20only=20a=20persistent=20relay=20ledger=20?= =?UTF-8?q?finds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both need what the test suite and the Stage 1 round-trip did not have: a relay whose ledger survives client runs, and several POSTs on the wire at once. §17.1 is the dust-floor residual becoming an unpayable debt after enough runs; §17.2 is the headroom guard pricing a frame and dispatching a batch. Records which number the client must reconcile against and why each of the other two candidates over-pays. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 84 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 86816d4..4c6997e 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1533,3 +1533,87 @@ Receipts no longer carry money under this design, but they still carry the them could steer traffic. Checking at the protocol boundary means every `PushsyncReceipt` in the codebase carries exactly the 32 bytes that were pushed. + +## 17. Found by running a metered relay: two client-side bugs (both fixed) + +Neither is reachable from a single upload against a fresh relay, which is +why both survived the test suite and the Stage 1 round-trip. They need a +relay whose ledger *persists across client runs* — the shipped +configuration — and enough concurrency to have several POSTs on the wire +at once. + +### 17.1 Debt the relay carried across sessions could not be paid + +**Status: fixed** (`LaneAccount::adopt_relay_debt`, `LanePayer::reconcile`, +with regression tests). + +§10.2's dust floor guarantees a run ends owing something: the residual +below `min_cheque_plur` is left unpaid because a cheque for it would be +refused. The relay is right to keep counting that against the credit line +— forgiving it would make "stay under the floor" a way to be served for +free. But the client's books are per-process, so the *next* run starts +believing it owes nothing, and the relay's `owed` only ever grows. + +Once the carry crosses `max_outstanding_plur` the account is refused on +its first POST, and the refusal is unpayable: `next_cumulative()` computes +the cheque from the client's own `owed`, which is zero. Observed live as +a second upload failing 151/151 against a relay carrying 290,400,000,000 +PLUR. With the shipped defaults the per-run residual is under 3.9e12 +against a 62.2e12 cap, so it takes roughly sixteen runs rather than two — +a slow-motion deadlock, not a corner case. + +The fix is to ask rather than to remember: `GET /v1/account` already +reports the relay's own figure, so a 402 the client cannot pay triggers a +reconcile and it pays what the relay says it owes. Three things about +which number, each of which was wrong in a draft: + +- **`owed`, not the `outstanding_plur` in the 402 body.** `reserve()` + adds the reservation *before* computing what it reports, so the body's + figure includes the request just refused. Adopting it over-pays by + exactly that body and the next cheque bounces as an overpayment. +- **Reservations excluded on our side too**, for the same reason: bytes + still in flight are already held in `pending_plur` and would be billed + twice when those POSTs land. Under-counting is safe and self-correcting; + over-counting is a refused cheque. +- **Bounded by the chequebook balance, not the credit line.** Debt reaches + the line by construction, and an operator who lowers the line leaves + legitimately-incurred debt above it. Rejecting on that basis refuses to + pay a real bill and preserves the deadlock. The funded balance is what + actually bounds exposure, and `settle` already enforces it exactly + across all lanes (§8.3). + +This makes the client trust a curated relay's arithmetic about its own +receivable, which §2 already grants. The alternative — persisting the +residual client-side — keeps better books but has no recovery when that +state is lost, and a client permanently locked out of a relay with no way +to clear it is the worse failure. + +### 17.2 The headroom guard admitted a frame, then sent a batch + +**Status: fixed** (`LanePayer::has_headroom` prices a real POST; +`dispatch_ok` re-evaluated per dispatch). + +The pre-dispatch guard asked whether one more *frame* fit inside the +credit line and then dispatched a full `batch_max` POST, and its answer +was computed once for an unbounded run of dispatches. Several concurrent +POSTs were each waved through against the same headroom; the relay +reserved every one of them, their sum crossed the line, and it answered +402 to a client whose own books said it had room. + +That refusal is the unpayable kind — the bytes are in flight, so nothing +is owed yet and settling changes nothing — and the batch returns having +spent an attempt per chunk, which is why handing batches back was +measured to make things worse rather than better. §7.2's whole point is +that the client sizes to fit instead of discovering the ceiling as a 402. + +The guard now prices the POST the scheduler will actually build. Where the +line holds only one POST this serialises the lane, which is the honest +answer: a credit line that fits 1.8 POSTs cannot have eight in flight. +Measured on a hard-mode relay, per-upload 402s went 11 → 1 and unpayable +refusals 6 → 0; the one remaining 402 is §17.1's carried debt, paid on +reconcile. + +`LaneAccount::max_body_bytes` had the same confusion — sizing against +`owed` while `has_headroom` and `would_exceed` both bind on `outstanding`. +It is only reached from tests today, but it would have reintroduced the +bug at its next caller. From 5b05a9eea57b5075c32d20213660caea6e629471 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sun, 9 Aug 2026 15:04:56 +0300 Subject: [PATCH 10/27] =?UTF-8?q?fix(metered):=20resolve=20=C2=A710.1's=20?= =?UTF-8?q?thresholds=20against=20the=20credit=20line=20that=20actually=20?= =?UTF-8?q?binds,=20and=20size=20each=20POST=20to=20live=20headroom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from running uploads against a metered relay until the batch's value decayed. **§10.1's invariant was checked against the wrong quantity.** `Params::validate` asserts `min_cheque <= settle_every < max_outstanding`, but `max_outstanding_plur` is only the ceiling on a credit line — the line that binds is per batch, `min(remaining/credit_ratio, ceiling)`. For any batch under `min_cheque_plur * credit_ratio` (~0.39 BZZ at the defaults) the floor sits above everything the account can ever owe, so it accrues to its cap, is answered 402, and cannot write an acceptable cheque: `next_cumulative()` returns None below the floor and `/v1/pay` rejects the same amount as dust. Neither side is broken; the parameters cannot be satisfied. Live: a 679,783,122,862 line against a 3,900,000,000,000 floor halted an upload at 16 of 219 chunks with the relay reporting err=0. That is §10.3's small batch — the case the scaled line exists to keep serving. `Params::effective(cap)` now resolves both thresholds against the line: settle at half of it, never demand a cheque larger than that. Both sides derive it from `(params, cap)` and `cap` is already in the challenge, so they agree with no protocol change. A generous line is unaffected. Accepting a smaller cheque costs the relay nothing — cheques are cumulative, and `cashout --min-amount` already declines to spend gas on a claim not worth collecting. The dust floor belongs at redemption. **POSTs are sized to live headroom.** The body was built from a ceiling computed before any debt existed, so concurrent POSTs were each sized as if alone on the lane and the relay refused their sum — a 402 nothing could pay, since the bytes were in flight and nothing was owed yet. `affordable_frames()` deducts current debt and in-flight bytes, and `Scheduler::set_lane_batch_max` re-clamps between dispatches. My first attempt gated dispatch on whether a *full* POST would fit. That is worse, and caused the §17.3 stall above in a second form: when the leftover debt is under the dust floor it can never be settled, so the headroom such a guard waits for never returns. `has_headroom` is back to asking about one frame — the smallest dispatchable unit — with sizing done separately, and the two are documented as a pair. Also removes two pieces of dead code: `ack_ok` in the relay's push path, superseded by the streaming `on_chunk` callback that emits `bpo` as well (its comment on what `po` measures is kept, moved to the live site), and a stale `mut` on a delegating wrapper. The crate now builds warning-free. Verified against the relay on an amd64 VPS: the 768 KiB upload that stalled at 16/219 now completes 219/219 at 86.79 KiB/s with one 402, one reconcile, one payment; relay ledger settles to owed 0 holding a signed cumulative cheque. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 60 ++++++++++++-- src/client.rs | 26 ++++-- src/meter.rs | 77 ++++++++++++++++++ src/payer.rs | 163 ++++++++++++++++++++++++++++++-------- src/pusher.rs | 35 ++++---- src/pushsched.rs | 14 ++++ 6 files changed, 309 insertions(+), 66 deletions(-) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 4c6997e..a44b325 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1534,13 +1534,19 @@ them could steer traffic. Checking at the protocol boundary means every `PushsyncReceipt` in the codebase carries exactly the 32 bytes that were pushed. -## 17. Found by running a metered relay: two client-side bugs (both fixed) +## 17. Found by running a metered relay: three bugs (all fixed) -Neither is reachable from a single upload against a fresh relay, which is -why both survived the test suite and the Stage 1 round-trip. They need a -relay whose ledger *persists across client runs* — the shipped -configuration — and enough concurrency to have several POSTs on the wire -at once. +None of these is reachable from a single upload against a fresh relay, +which is why all three survived the test suite and the Stage 1 round-trip. +They need a relay whose ledger *persists across client runs* — the shipped +configuration — enough concurrency to have several POSTs on the wire at +once, and a batch that has been spent down far enough for its credit line +to bind. + +The last of those is the one to take seriously as a *method* point: §17.3 +is not a coding mistake but an invariant checked against the wrong +quantity, and nothing short of running a real batch until its value +decayed would have surfaced it. ### 17.1 Debt the relay carried across sessions could not be paid @@ -1617,3 +1623,45 @@ reconcile. `owed` while `has_headroom` and `would_exceed` both bind on `outstanding`. It is only reached from tests today, but it would have reintroduced the bug at its next caller. + +The first attempt at this fix gated dispatch on whether a *full* POST +would fit, which is worse: see §17.3, which it caused. + +### 17.3 §10.1's invariant does not hold at the line that binds + +**Status: fixed** (`Params::effective`, applied on both sides, with tests). + +`Params::validate` checks `min_cheque <= settle_every < max_outstanding`. +But `max_outstanding_plur` is only the *ceiling* on a credit line; the +line that actually binds an account is per batch, +`min(remaining_value / credit_ratio, ceiling)` (§10.3). For any batch +whose remaining value is under `min_cheque_plur * credit_ratio` — about +0.39 BZZ at the shipped defaults — the configured floor sits above +everything that account can ever owe, and the invariant quietly stops +holding. + +What follows is a permanent refusal. The account accrues to its cap, is +answered 402, and cannot write a cheque large enough to be accepted: +`next_cumulative()` returns `None` below the floor, and `/v1/pay` rejects +anything below it as dust. Nothing on either side is broken or dishonest +— the parameters simply cannot be satisfied. Observed live at a credit +line of 679,783,122,862 against a 3,900,000,000,000 floor, 5.7× short, +which halted an upload at 16 of 219 chunks with the relay reporting +`err=0`. + +This is exactly §10.3's small batch — the case the value-scaled credit +line exists to keep serving — so refusing it defeats the purpose of +scaling the line at all. + +The thresholds are now resolved against the line before they are applied: +settle at half of it, and never demand a cheque larger than that. Both +sides derive this from `(params, cap)`, and `cap` is already in the +challenge, so they agree without exchanging anything new. A generous line +keeps the configured values unchanged; only a line that cannot reach them +scales them down. + +Accepting a smaller cheque costs the relay nothing, which is what makes +this safe: cheques are cumulative, so a small one now does not force a +small cash-out later, and `hoverfly cashout --min-amount` already declines +to spend gas on a claim that is not worth collecting. The dust floor +belongs at redemption, where the gas is, not at acceptance. diff --git a/src/client.rs b/src/client.rs index 81e57f7..e48756d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2771,6 +2771,7 @@ where // cannot pay its way out of — it would owe nothing, having had nothing // accepted. let mut infos = infos; + let mut lane_frame_ceiling: Vec = vec![usize::MAX; infos.len()]; if let Some(pc) = payment { for (i, payer) in payers.iter_mut().enumerate() { let Some(payer) = payer.as_mut() else { continue }; @@ -2786,9 +2787,10 @@ where "lane {i}: credit line allows {fits} frames/POST (was {before})"); infos[i].batch_max = Some(fits); } - // The headroom guard admits whole POSTs, so it needs to know how - // big one is on this lane. - payer.set_post_frames(fits.min(before)); + // Remember the lane's own ceiling: the per-dispatch resize below + // must clamp to it and never raise a lane above what it + // advertised. + lane_frame_ceiling[i] = fits.min(before); } } // batch id -> body bytes, so a completed POST can be billed for what @@ -2861,14 +2863,22 @@ where if let Some(pc) = payment { for (lane, payer) in payers.iter_mut().enumerate() { let Some(payer) = payer.as_mut() else { continue }; - if payer.has_headroom() { - continue; - } - if payer.account.owed() > 0 + if !payer.has_headroom() + && payer.account.owed() > 0 && let Err(e) = payer.settle(&http, pc).await { warn!(target: "hoverfly::upload", "lane {lane}: settle failed: {e}"); } + // Size the next POST to what the lane can afford *now*. + // Without this the body is built from a ceiling computed + // before any debt existed, so concurrent POSTs are each + // sized as if they were the only one in flight and the + // relay refuses their sum — a 402 nothing can pay, since + // the bytes are still on the wire and nothing is owed yet. + let affordable = payer.affordable_frames(); + if affordable > 0 { + sched.set_lane_batch_max(lane, affordable.min(lane_frame_ceiling[lane])); + } } } // Re-checked on every pass of the loop below, not once before it. @@ -3254,7 +3264,7 @@ pub async fn push_via_pusher( #[cfg(not(target_arch = "wasm32"))] pub async fn push_stream_via_pushers( pusher_urls: &[String], - mut streamer: UploadStreamer, + streamer: UploadStreamer, progress: Option<&ProgressFn>, ) -> Result { push_stream_via_pushers_paid(pusher_urls, streamer, progress, None).await diff --git a/src/meter.rs b/src/meter.rs index ed69421..2e0c206 100644 --- a/src/meter.rs +++ b/src/meter.rs @@ -700,12 +700,89 @@ impl Params { pub fn price_bytes(&self, bytes: u64) -> u128 { u128::from(bytes.div_ceil(1024)) * self.price_plur_per_kib } + + /// The settlement thresholds that actually apply to an account whose + /// credit line is `cap`. + /// + /// `validate` checks §10.1's invariant against `max_outstanding_plur`, + /// but that is only the *ceiling* on a credit line — the line that + /// binds is per batch, `min(remaining_value / credit_ratio, ceiling)` + /// (§10.3). For any batch smaller than + /// `min_cheque_plur * credit_ratio` the configured floor sits *above* + /// everything that account can ever owe, and the invariant quietly + /// stops holding: it accrues to its cap, is refused, and cannot write + /// a cheque large enough to be accepted. Service stops permanently, + /// for a batch that is paid up and behaving. + /// + /// Observed live with the shipped defaults: a credit line of + /// 679,783,122,862 against a 3,900,000,000,000 floor — 5.7× short — + /// which is §10.3's small batch, the case the scaled line exists to + /// keep serving. + /// + /// Both sides derive this from `(params, cap)` and `cap` is already in + /// the challenge, so the two agree without exchanging anything new. + /// Settling at half the line leaves the other half as the working + /// headroom a POST is dispatched into. + pub fn effective(&self, cap: u128) -> EffectiveParams { + let settle_every = self.settle_every_plur.min(cap / 2); + EffectiveParams { + // Preserves `min_cheque <= settle_every` — the half of §10.1 + // that makes a 402 clearable — at any credit line. + min_cheque_plur: self.min_cheque_plur.min(settle_every), + settle_every_plur: settle_every, + } + } +} + +/// §10.1's thresholds resolved against a particular credit line. See +/// [`Params::effective`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EffectiveParams { + pub min_cheque_plur: u128, + pub settle_every_plur: u128, } #[cfg(test)] mod param_tests { use super::*; + /// §10.1's invariant is checked against `max_outstanding_plur`, but the + /// line that binds is per batch. A small batch gets a line far under + /// the configured floor, and then no cheque it can write is acceptable + /// — it accrues to its cap and is refused for good. Seen live at a + /// 679,783,122,862 line against a 3,900,000,000,000 floor. + #[test] + fn a_credit_line_below_the_dust_floor_can_still_settle() { + let p = Params::default(); + let line = 679_783_122_862u128; + assert!( + line < p.min_cheque_plur, + "this is the case: the whole line is under the configured floor" + ); + + let e = p.effective(line); + assert!( + e.min_cheque_plur <= e.settle_every_plur, + "a 402 must be clearable by the cheque the client is told to write" + ); + assert!( + e.settle_every_plur < line, + "and settlement must trigger before the line is full, or the \ + account is refused before it is ever asked to pay" + ); + assert!(e.min_cheque_plur > 0, "some cheque must be acceptable"); + } + + /// A line with room to spare must keep the configured thresholds — + /// scaling down is for the batches that need it, not a general discount. + #[test] + fn a_generous_credit_line_keeps_the_configured_thresholds() { + let p = Params::default(); + let e = p.effective(p.max_outstanding_plur); + assert_eq!(e.min_cheque_plur, p.min_cheque_plur); + assert_eq!(e.settle_every_plur, p.settle_every_plur); + } + #[test] fn the_shipped_defaults_satisfy_the_invariant() { Params::default().validate().expect("defaults must be valid"); diff --git a/src/payer.rs b/src/payer.rs index f7fe41d..671c9f3 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -304,6 +304,11 @@ pub struct LaneAccount { /// Total already promised to this beneficiary. Cheques are cumulative, /// so this only grows. cumulative_plur: u128, + /// This account's credit line, once a challenge has reported one. + /// §10.1's thresholds are resolved against it — see + /// [`Params::effective`] — because the configured floor can sit above + /// everything a small batch is able to owe. + cap_plur: u128, } impl LaneAccount { @@ -314,6 +319,7 @@ impl LaneAccount { owed_plur: 0, pending_plur: 0, cumulative_plur: 0, + cap_plur: 0, } } @@ -370,14 +376,32 @@ impl LaneAccount { .saturating_sub(self.params.price_bytes(body_bytes)); } + /// The credit line the relay reported. Setting it is what lets the + /// thresholds below scale down to a small batch. + pub fn set_cap(&mut self, cap: u128) { + self.cap_plur = cap; + } + + /// §10.1's thresholds as they apply to this account. Falls back to the + /// configured values before any challenge has reported a line. + fn thresholds(&self) -> crate::meter::EffectiveParams { + if self.cap_plur == 0 { + return crate::meter::EffectiveParams { + min_cheque_plur: self.params.min_cheque_plur, + settle_every_plur: self.params.settle_every_plur, + }; + } + self.params.effective(self.cap_plur) + } + pub fn should_settle(&self) -> bool { - self.owed_plur >= self.params.settle_every_plur + self.owed_plur >= self.thresholds().settle_every_plur } /// The cumulative for the next cheque, or `None` when what is owed is /// still under the lane's dust floor and would be refused. pub fn next_cumulative(&self) -> Option { - if self.owed_plur < self.params.min_cheque_plur { + if self.owed_plur < self.thresholds().min_cheque_plur { return None; } Some(self.cumulative_plur.saturating_add(self.owed_plur)) @@ -521,11 +545,6 @@ pub struct LanePayer { header: Option, header_stale_after: u64, cap_plur: u128, - /// Frames in a full POST to this lane, as the scheduler will actually - /// build it. Set once the lane's `batch_max` is known; see - /// [`Self::has_headroom`] for why the guard has to know the size of the - /// thing it is admitting. - post_frames: usize, } #[cfg(not(target_arch = "wasm32"))] @@ -539,22 +558,22 @@ impl LanePayer { header: None, header_stale_after: 0, cap_plur: 0, - post_frames: 1, } } - /// Tell the lane how large a POST the scheduler will build for it, so - /// the headroom guard can ask about the real body rather than a frame. - pub fn set_post_frames(&mut self, frames: usize) { - self.post_frames = frames.max(1); - } - /// The credit line the relay last told us about, or 0 before the first /// challenge. Used to size POSTs (§7.2). pub fn cap_plur(&self) -> u128 { self.cap_plur } + /// The credit line normally arrives with a challenge, which needs a + /// relay; sizing is pure arithmetic and worth testing without one. + #[cfg(test)] + pub(crate) fn set_cap_for_test(&mut self, cap: u128) { + self.cap_plur = cap; + } + /// A valid challenge header, fetching and signing one if needed. /// /// Re-fetched 30 s before expiry rather than on failure: racing the @@ -591,6 +610,9 @@ impl LanePayer { .map_err(|e| format!("challenge json: {e}"))?; let offered = OfferedChallenge::parse(&v)?; self.cap_plur = offered.cap_plur; + // §10.1's thresholds scale with the line, so the account needs + // it too — see `Params::effective`. + self.account.set_cap(offered.cap_plur); self.header_stale_after = offered.stale_after(); self.header = Some(offered.sign(&cfg.signer, self.quote.chain_id)?); } @@ -613,16 +635,55 @@ impl LanePayer { /// walked down until it genuinely fits, because the relay bills a /// KiB-rounded body and an off-by-one here is an unfixable 402 loop. pub fn max_frames(&self) -> usize { + // A zero cap is "no credit line known", i.e. an open lane — not a + // lane that can afford nothing. if self.cap_plur == 0 { return usize::MAX; } + self.frames_within(self.cap_plur) + } + + /// Frames per POST affordable *right now*, with current debt and + /// in-flight bytes deducted. + /// + /// This is what each dispatch must be sized by. Gating instead on + /// whether a *full* POST would fit stalls the lane outright whenever + /// the leftover debt cannot be settled: a residual under + /// `min_cheque_plur` is unpayable by construction (§10.2), so if a + /// full-size POST needs the whole line, no cheque can ever restore the + /// headroom the guard is waiting for and the upload fails with chunks + /// still pending. Observed exactly that way against a batch whose + /// credit line had decayed to roughly one POST. + /// + /// Returns 0 when not even one frame fits, which is the caller's cue to + /// settle or wait rather than to dispatch. + pub fn affordable_frames(&self) -> usize { + if self.cap_plur == 0 { + return usize::MAX; + } + let headroom = self.cap_plur.saturating_sub(self.account.outstanding()); + let frame = crate::pushframe::MAX_FRAME_LEN as u64; + if self.quote.params.price_bytes(frame) > headroom { + return 0; + } + self.frames_within(headroom) + } + + /// Largest frame count whose KiB-rounded body prices at or under + /// `budget`. Estimated, then walked down until it genuinely fits: the + /// relay bills a rounded body, and an off-by-one here is an unfixable + /// 402 loop. + fn frames_within(&self, budget: u128) -> usize { + if budget == 0 { + return 0; + } let frame = crate::pushframe::MAX_FRAME_LEN as u128; - let mut n = (self.cap_plur / self.quote.params.price_plur_per_kib) + let mut n = (budget / self.quote.params.price_plur_per_kib) .saturating_mul(1024) .checked_div(frame) .unwrap_or(0) .min(usize::MAX as u128) as usize; - while n > 1 && self.quote.params.price_bytes(n as u64 * frame as u64) > self.cap_plur { + while n > 1 && self.quote.params.price_bytes(n as u64 * frame as u64) > budget { n -= 1; } n.max(1) @@ -634,27 +695,21 @@ impl LanePayer { /// and handing it back costs the chunks a retry attempt each time, so a /// tight credit line would exhaust their budget and fail the upload /// rather than merely pausing it. + /// One frame is the smallest thing worth dispatching, so that is what + /// this asks about. It is *not* a licence to then send a full batch: + /// the body is sized separately by [`Self::affordable_frames`], and the + /// two must be read together. Asking here about a full POST instead + /// looks safer and is not — see `affordable_frames` for the stall it + /// causes when the leftover debt is under the dust floor. pub fn has_headroom(&self) -> bool { if self.cap_plur == 0 { return true; } - // Ask about the POST that will actually be built, not about one - // frame. Admitting on "a frame would fit" and then dispatching a - // full batch is how several concurrent POSTs each get waved - // through against the same headroom: the relay reserves every one - // of them, their sum crosses the line, and it answers 402 to a - // client whose own books say it had room. That refusal is not even - // payable — the bytes are in flight, so nothing is owed yet — so - // the batch comes back having spent an attempt per chunk. - // - // When the line is narrow enough to hold only one POST this - // serialises the lane, which is the honest answer: a credit line - // that fits 1.8 POSTs cannot have 8 in flight. - let body = self + let one_frame = self .quote .params - .price_bytes((self.post_frames * crate::pushframe::MAX_FRAME_LEN) as u64); - self.account.outstanding().saturating_add(body) <= self.cap_plur + .price_bytes(crate::pushframe::MAX_FRAME_LEN as u64); + self.account.outstanding().saturating_add(one_frame) <= self.cap_plur } /// Would dispatching `body_bytes` right now exceed the credit line? @@ -1149,6 +1204,52 @@ mod tests { assert!(a.max_body_bytes(cap) < before, "unpaid debt eats the line"); } + /// A lane whose credit line is about one POST wide must keep making + /// progress, in smaller POSTs. + /// + /// Regression: gating dispatch on whether a *full* POST fits stalled + /// the upload outright here. The leftover debt is under + /// `min_cheque_plur`, so it cannot be settled (§10.2) and the headroom + /// the guard waited for could never come back — 60 of 76 chunks were + /// left unacked against a live relay. + #[test] + fn a_line_barely_wider_than_one_post_still_makes_progress() { + let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let p = q.params; + let mut payer = LanePayer::new("http://lane".into(), q, 0); + + // A line that fits one full POST and very little more — the shape a + // batch decays into as its remaining value is spent down. + let full_post = p.price_bytes((512 * crate::pushframe::MAX_FRAME_LEN) as u64); + payer.set_cap_for_test(full_post + full_post / 20); + + let first = payer.affordable_frames(); + assert!(first > 0, "a fresh lane can dispatch"); + + // Send one small POST and leave the debt unsettleable. + let sent = 16 * crate::pushframe::MAX_FRAME_LEN as u64; + payer.account.record_sent(sent); + payer.account.record_answered(sent, true); + assert!(payer.account.owed() > 0); + assert!( + payer.account.next_cumulative().is_none(), + "this residual is below the dust floor, so no cheque can clear it" + ); + + let next = payer.affordable_frames(); + assert!( + next > 0, + "must still dispatch a smaller POST; a lane that can never settle \ + and never send has stalled the upload" + ); + // And whatever it sizes must actually fit, or the relay 402s it. + let body = p.price_bytes((next * crate::pushframe::MAX_FRAME_LEN) as u64); + assert!( + payer.account.outstanding().saturating_add(body) <= payer.cap_plur(), + "sized {next} frames but that does not fit the remaining line" + ); + } + /// The relay's ledger outlives the client's. Every run ends leaving the /// sub-dust residual unpaid, and a fresh process starts believing it /// owes nothing — so the relay refuses service for debt the client diff --git a/src/pusher.rs b/src/pusher.rs index 00a8ffb..ed9328a 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -878,13 +878,17 @@ async fn pay_response(state: Arc, req: Request) -> &format!("cheque cumulative {cumulative} does not exceed the {have} already accepted"), ); } - if cumulative - have < m.cfg.params.min_cheque_plur { + // Against the floor this account can actually reach, not the configured + // one: a batch whose credit line is below `min_cheque_plur` would + // otherwise be refused for a cheque it is structurally incapable of + // writing (§10.1, `Params::effective`). + let floor = m.cfg.params.effective(verified.cap_plur).min_cheque_plur; + if cumulative - have < floor { return json_line_response( StatusCode::BAD_REQUEST, &format!( - "cheque credits {} but the dust floor is {}", + "cheque credits {} but the dust floor is {floor}", cumulative - have, - m.cfg.params.min_cheque_plur ), ); } @@ -1573,24 +1577,6 @@ async fn run_push( "a": hex::encode(addr), "s": "ok", "po": 0, "ms": 0, "dedup": true })); }; - let ack_ok = |addr: &[u8; 32], info: crate::client::PushInfo| { - // `po` is the proximity order of the peer whose receipt we took — - // i.e. how deep into the chunk's own neighborhood it actually - // landed. This is the measurement that decides whether client-side - // proximity routing to a relay's overlay is worth anything at all - // (docs/pusher-design.md §7); without it that question can only be - // guessed at. - let mut v = serde_json::json!({ - "a": hex::encode(addr), - "s": "ok", - "po": info.po, - "ms": info.ms, - }); - if info.shallow { - v["shallow"] = serde_json::Value::Bool(true); - } - send_line(&v); - }; // The stamp's batch_id must match the on-chain owner the signature // recovers to. All chunks in one upload share a batch; verify the @@ -1760,6 +1746,13 @@ async fn run_push( Arc::new(move |addr: &[u8; 32], res| { let v = match &res { Ok(info) => { + // `po` is the proximity order of the peer whose receipt + // we took — how deep into the chunk's own neighborhood + // it actually landed. This is the measurement that + // decides whether client-side proximity routing to a + // relay's overlay is worth anything at all + // (docs/pusher-design.md §7); without it that question + // can only be guessed at. let mut v = serde_json::json!({ "a": hex::encode(addr), "s": "ok", "po": info.po, "ms": info.ms, // Best proximity the dispatcher could reach for this diff --git a/src/pushsched.rs b/src/pushsched.rs index 63f0c92..0a615bb 100644 --- a/src/pushsched.rs +++ b/src/pushsched.rs @@ -772,6 +772,20 @@ impl Scheduler { } } + /// Re-clamp a lane's frames-per-POST between dispatches. + /// + /// A metered lane's affordable body shrinks as debt and in-flight bytes + /// accumulate and grows back as cheques clear, so the ceiling set at + /// startup goes stale immediately. Sizing the assignment here is what + /// keeps the client from building a body the relay will refuse — §7.2 + /// wants the POST sized to fit rather than the ceiling discovered as a + /// 402. + pub fn set_lane_batch_max(&mut self, lane: usize, max: usize) { + if let Some(l) = self.lanes.get_mut(lane) { + l.info.batch_max = Some(max.max(1)); + } + } + /// Lanes currently paused for payment, so the driver knows which ones a /// cheque would unblock. pub fn unfunded_lanes(&self) -> Vec { From 1c473bc57b45f484db79d1ec56980b93498341b9 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sun, 9 Aug 2026 16:22:21 +0300 Subject: [PATCH 11/27] fix(metered): resume a lane refused for bytes in flight, and don't bill an in-flight POST twice when reconciling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more from pushing 128 KiB - 4 MiB through the metered relay. Both scale with upload size, which is why smaller test payloads missed them. **A lane over its line on reservations was parked for good.** The relay refuses on `owed + reserved`, but only `owed` is payable — reservations are bodies it is still reading. So a lane can be over its line while its debt is under the dust floor, which is not a disagreement and not something a cheque fixes: those bytes have to land. A 402 marks the lane `Unfunded` and only a successful settle re-funds it, so with nothing payable the lane stayed parked and the upload ended with chunks pending and the relay reporting err=0 — 2 MiB stopping at 311 of 567 frames. The driver now separates busy from stuck: bytes in flight re-funds the lane, nothing in flight and nothing payable still warns. It cannot spin, because `has_headroom` binds on `outstanding` and dispatches nothing until those bytes clear. **Reconciling mid-flight billed a POST twice.** `adopt_relay_debt` took the relay's `owed` raw, but a body the relay has finished reading is already in that figure while locally it is still `pending` until its response closes — so `record_answered` then added the same bytes again. The run ended with `cheque credits 535680000000 but only 510240000000 is owed` and settlement jammed. It now deducts what is in flight; under-adopting is safe, since the remainder is still owed and the next settle collects it. The docstring already argued for this and the code did not do it. Verified against the relay on an amd64 VPS — every frame acked at every size, no unpayable refusals, no rejected cheques, relay ledger settling to owed 0: 128 KiB 43/43 402=0 512 KiB 151/151 402=0 256 KiB 76/76 402=0 1 MiB 290/290 402=2 2 MiB 567/567 402=2 4 MiB 1122/1122 402=4 Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 48 ++++++++++++++++++++++++++++++++- src/client.rs | 40 ++++++++++++++++++++++----- src/payer.rs | 57 +++++++++++++++++++++++++++++++++++---- 3 files changed, 133 insertions(+), 12 deletions(-) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index a44b325..4b156ec 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1534,7 +1534,7 @@ them could steer traffic. Checking at the protocol boundary means every `PushsyncReceipt` in the codebase carries exactly the 32 bytes that were pushed. -## 17. Found by running a metered relay: three bugs (all fixed) +## 17. Found by running a metered relay: four bugs (all fixed) None of these is reachable from a single upload against a fresh relay, which is why all three survived the test suite and the Stage 1 round-trip. @@ -1665,3 +1665,49 @@ this safe: cheques are cumulative, so a small one now does not force a small cash-out later, and `hoverfly cashout --min-amount` already declines to spend gas on a claim that is not worth collecting. The dust floor belongs at redemption, where the gas is, not at acceptance. + +### 17.4 A lane refused for bytes in flight was parked for good + +**Status: fixed** (`unpayable_402` in the driver). + +The relay refuses on `owed + reserved`, but only `owed` is payable — +reservations are bodies it is still reading. A lane can therefore be over +its line while its *debt* is under the dust floor, and that is neither a +disagreement nor something a cheque fixes: those bytes simply have to +land. + +A 402 marks the lane `Unfunded`, and only a successful settle re-funds it +(§12). With nothing payable there is no settle, so the lane stayed parked +and the upload ended with chunks pending and the relay reporting `err=0` — +2 MiB stopping at 311 of 567 frames. The bigger the upload, the more +certain this is, because more POSTs are in flight when the line fills. + +The driver now distinguishes the two cases. Bytes in flight means busy: +re-fund the lane and let it resume when they clear. Nothing in flight and +nothing payable is the genuine disagreement, and still warns. Re-funding +cannot spin, because `has_headroom` binds on `outstanding` and so +dispatches nothing until the in-flight bytes actually clear. + +The same run surfaced the mirror of §17.1 in the reconcile itself: +adopting the relay's `owed` while a POST it had already booked was still +`pending` locally counted that body twice once the response closed, and +the cheque was refused as an overpayment (`credits 535680000000 but only +510240000000 is owed`). Reconciliation now deducts what is in flight. +Under-adopting is safe — the remainder is still owed and the next settle +collects it. + +After all four fixes, uploads of 128 KiB through 4 MiB against a hard-mode +relay complete every frame, with no unpayable refusals and no rejected +cheques: + +| payload | frames acked | 402s | stuck | rejected cheques | +|--------:|-------------:|-----:|------:|-----------------:| +| 128 KiB | 43/43 | 0 | 0 | 0 | +| 256 KiB | 76/76 | 0 | 0 | 0 | +| 512 KiB | 151/151 | 0 | 0 | 0 | +| 1 MiB | 290/290 | 2 | 0 | 0 | +| 2 MiB | 567/567 | 2 | 0 | 0 | +| 4 MiB | 1122/1122 | 4 | 0 | 0 | + +The remaining 402s are the intended kind: the line genuinely fills, the +client pays or waits, and the lane resumes. diff --git a/src/client.rs b/src/client.rs index e48756d..54d759c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3118,9 +3118,7 @@ where sched.fund_lane(lane); } Ok(None) => { - warn!(target: "hoverfly::upload", - "lane {lane}: relay reports debt below its own \ - dust floor yet refuses service"); + unpayable_402(&mut sched, lane, payer); } Err(e) => { warn!(target: "hoverfly::upload", @@ -3128,9 +3126,7 @@ where } }, Ok(false) => { - warn!(target: "hoverfly::upload", - "lane {lane}: 402 but neither side reports debt \ - — ledger disagreement"); + unpayable_402(&mut sched, lane, payer); } Err(e) => { warn!(target: "hoverfly::upload", @@ -3270,6 +3266,38 @@ pub async fn push_stream_via_pushers( push_stream_via_pushers_paid(pusher_urls, streamer, progress, None).await } +/// A 402 arrived that no cheque can clear. Decide whether the lane is +/// merely busy or genuinely stuck. +/// +/// The relay refuses on `owed + reserved`, and only `owed` is payable — +/// reservations are bodies it is still reading. So a lane can be over its +/// line with a debt below the dust floor, which is not a disagreement and +/// not something paying fixes: those bytes simply have to land. Leaving +/// the lane `Unfunded` in that case parks it for good, because only a +/// successful settle re-funds it and no settle is possible. That is an +/// upload that stops with chunks pending and the relay reporting no errors. +/// +/// Re-funding is safe against a busy loop: `has_headroom` binds on +/// `outstanding`, so nothing is dispatched until the in-flight bytes +/// actually clear. +#[cfg(not(target_arch = "wasm32"))] +fn unpayable_402( + sched: &mut crate::pushsched::Scheduler, + lane: usize, + payer: &crate::payer::LanePayer, +) { + if payer.account.pending() > 0 { + debug!(target: "hoverfly::upload", + "lane {lane}: over its line on bytes in flight, not debt; \ + resuming once they land"); + sched.fund_lane(lane); + return; + } + warn!(target: "hoverfly::upload", + "lane {lane}: 402 with nothing in flight and nothing payable — \ + ledger disagreement"); +} + /// As [`push_stream_via_pushers`], but able to pay metered lanes. /// /// `payment` is `None` for every `open` lane, which is the default and the diff --git a/src/payer.rs b/src/payer.rs index 671c9f3..1568f5f 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -334,6 +334,12 @@ impl LaneAccount { self.owed_plur } + /// Bytes dispatched but not yet answered, priced. Not debt yet — see + /// [`Self::adopt_relay_debt`] for why the distinction matters. + pub fn pending(&self) -> u128 { + self.pending_plur + } + /// Owed plus in-flight — the client's mirror of the relay's /// `owed + reserved`, and what the credit line actually binds on. pub fn outstanding(&self) -> u128 { @@ -434,14 +440,23 @@ impl LaneAccount { /// still in flight are already counted in `pending_plur`, and would be /// billed twice when those POSTs land. /// - /// Only ever raises. An under-count is safe and self-correcting (the - /// next settle picks up the rest); an over-count is a cheque the relay - /// refuses. + /// Only ever raises, and only by what is not already accounted for in + /// flight. An under-count is safe and self-correcting (the next settle + /// picks up the rest); an over-count is a cheque the relay refuses. pub fn adopt_relay_debt(&mut self, relay_owed: u128) -> bool { - if relay_owed <= self.owed_plur { + // Deduct what is still on the wire. A POST the relay has finished + // reading is already in the figure it just reported, while locally + // it is still `pending` until its response closes — so adopting the + // raw number and then letting `record_answered` move those same + // bytes into `owed` counts them twice. That surfaces as a cheque + // the relay rejects for overpayment, which jams settlement for the + // rest of the run. Under-adopting is safe: the remainder is still + // owed, and the next settle collects it. + let adopt = relay_owed.saturating_sub(self.pending_plur); + if adopt <= self.owed_plur { return false; } - self.owed_plur = relay_owed; + self.owed_plur = adopt; true } @@ -1273,6 +1288,37 @@ mod tests { ); } + /// Reconciling mid-flight must not bill the same POST twice. + /// + /// Regression: the relay finishes reading a body and books it, so its + /// reported `owed` already covers a POST the client still has in + /// `pending`. Adopting that figure raw and then letting + /// `record_answered` move the same bytes into `owed` over-counted by + /// one POST, and the run ended with `cheque credits 535680000000 but + /// only 510240000000 is owed` — settlement jammed for the rest of it. + #[test] + fn adopting_relay_debt_does_not_double_count_bytes_in_flight() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + + let body = 64 * 1024u64; + let priced = p.price_bytes(body); + a.record_sent(body); + assert_eq!(a.pending(), priced, "on the wire, not yet debt"); + + // The relay has read that body and reports it as owed, while our + // response has not closed yet. + a.adopt_relay_debt(priced); + // Now it closes. + a.record_answered(body, true); + + assert_eq!( + a.owed(), + priced, + "billed once, not twice: adopted {priced} then answered the same body" + ); + } + /// Only ever upward. The downward direction is `forgive_phantom_debt`, /// which is reached from a rejected cheque — a relay reporting *less* /// than we think must not silently shrink a debt we are still liable @@ -1286,6 +1332,7 @@ mod tests { let owed = a.owed(); assert!(owed > 0); + assert_eq!(a.pending(), 0, "nothing in flight, so nothing to deduct"); assert!(!a.adopt_relay_debt(owed - 1), "a smaller figure is ignored"); assert_eq!(a.owed(), owed); assert!(!a.adopt_relay_debt(0), "and zero especially so"); From 6a5a436824f73558f5c34d59df3a43ff92fb2bcc Mon Sep 17 00:00:00 2001 From: v1rtl Date: Mon, 10 Aug 2026 00:09:59 +0300 Subject: [PATCH 12/27] fix(metered): learn carried debt before sizing, and yield to the relay when it booked less than we billed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by putting the relay behind a real reverse proxy on a public HTTPS name, which is the first time the client met a broken response stream. **A broken stream made every later cheque bounce.** §7.3's ack-tail cuts both ways: a POST whose response stream breaks was still read, so the client bills it — but if the relay's task is cancelled before it commits, its `Admitted` guard releases the reservation and it books nothing. The client cannot tell the cases apart, so it over-counts, and the overshoot rides on the cumulative: every later cheque carries it and is refused for the same reason, so the lane never settles again (`cheque credits 212640000000 but only 148800000000 is owed`). The relay decides what it accepts, so the client now yields to it — re-read /v1/account, take that figure, re-present once, bounded so a refusing relay cannot loop us. `forgive_phantom_debt` was already the total-loss case of this. **The first POST of a run was sized before the debt was known.** §17.1 made carried debt recoverable but did not stop the client walking into it: a fresh process believes it owes nothing and sizes against the whole credit line while the relay already holds part of it. If the carried debt happens to sit below the dust floor, the recovery has nowhere to go — 16 of 567 frames, 152,160,000,000 carried against a 416,771,800,039 line. One GET per lane at setup makes every later size correct; §17.1 stays as the mid-run recovery. Sizing also has to be recomputed per *dispatch*, not per pass of the driver loop — the inner loop hands out several assignments in a row, and a ceiling refreshed only on the outer pass sizes the second and third as if each were alone on the lane. That was the remaining source of unpayable 402s. Verified over public HTTPS through Caddy, relay on an amd64 VPS with a pinned identity. Every size completes every frame with no 402s at all, and three back-to-back 2 MiB runs each settle to owed 0 so the next carries nothing: 256 KiB 76/76 1 MiB 290/290 2 MiB 567/567 4 MiB 1122/1122 402s=0 stuck=0 stream errors=0 rejected cheques=0 402 is the recovery path now, not the mechanism. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives.md | 69 +++++++++++++++++++++- src/client.rs | 80 +++++++++++++++++-------- src/payer.rs | 120 +++++++++++++++++++++++++++++++++----- 3 files changed, 230 insertions(+), 39 deletions(-) diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 4b156ec..f02c345 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1534,7 +1534,7 @@ them could steer traffic. Checking at the protocol boundary means every `PushsyncReceipt` in the codebase carries exactly the 32 bytes that were pushed. -## 17. Found by running a metered relay: four bugs (all fixed) +## 17. Found by running a metered relay: six bugs (all fixed) None of these is reachable from a single upload against a fresh relay, which is why all three survived the test suite and the Stage 1 round-trip. @@ -1696,7 +1696,57 @@ the cheque was refused as an overpayment (`credits 535680000000 but only Under-adopting is safe — the remainder is still owed and the next settle collects it. -After all four fixes, uploads of 128 KiB through 4 MiB against a hard-mode +### 17.5 A broken response stream made every later cheque bounce + +**Status: fixed** (`LaneAccount::sync_relay_debt`, one bounded re-present +in `LanePayer::pay`). + +§7.3's ack-tail cuts both ways. A POST whose response stream breaks was +still *read*, so the client bills it — but if the relay's task is +cancelled before it commits, the `Admitted` guard releases the reservation +and it books nothing. The client cannot tell that case from a clean one, +so it over-counts. + +The overshoot is not self-correcting: it rides on the cumulative, so every +later cheque carries it and is refused for the same reason, and the lane +never settles again. Seen the first time the relay was reached through a +real reverse proxy — two broken streams, then `cheque credits +212640000000 but only 148800000000 is owed`. + +The relay is the party deciding what it will accept, so the client yields +to it: on a rejection naming a smaller figure, re-read `/v1/account`, take +that number, and re-present once. Nothing was issued, so there is no +cumulative to be inconsistent with. The retry is bounded to a single +attempt, so a relay that keeps refusing cannot loop the client. + +This is the same yielding as §17.1 in the opposite direction, and +`forgive_phantom_debt` was already the total-loss case of it. + +### 17.6 The first POST of a run was sized before the debt was known + +**Status: fixed** (reconcile once per lane at setup). + +§17.1 made carried debt *recoverable*, via a 402 the client answers by +reconciling. It did not stop the client from walking into it. A fresh +process starts believing it owes nothing, so it sizes its first POST +against the entire credit line while the relay is already holding part of +it — and is refused immediately. + +Worse, the recovery could land in a hole: if the carried debt happens to +sit below the dust floor, the client reconciles, finds it cannot write an +acceptable cheque, and stops with the lane over its cap. Observed at 16 of +567 frames with 152,160,000,000 carried against a 416,771,800,039 line. + +Reading `/v1/account` once per lane at setup costs one GET and makes every +subsequent size correct, so the refusal never happens. §17.1's path +remains as the recovery for debt that appears mid-run. + +The same run showed that POST sizing has to be recomputed per *dispatch* +rather than per pass of the driver loop: the inner loop hands out several +assignments in a row, and a ceiling refreshed only on the outer pass sizes +the second and third of them as if each were alone on the lane. + +After all six fixes, uploads of 128 KiB through 4 MiB against a hard-mode relay complete every frame, with no unpayable refusals and no rejected cheques: @@ -1711,3 +1761,18 @@ cheques: The remaining 402s are the intended kind: the line genuinely fills, the client pays or waits, and the lane resumes. + +Repeated over public HTTPS through a reverse proxy, with §17.6 in place so +the client knows its carried debt before sizing anything, the 402s go away +entirely — the client never reaches its cap because it never builds a body +that would cross it: + +| run | payload | frames acked | 402s | rejected cheques | +|----:|--------:|-------------:|-----:|-----------------:| +| 1 | 2 MiB | 567/567 | 0 | 0 | +| 2 | 2 MiB | 567/567 | 0 | 0 | +| 3 | 2 MiB | 567/567 | 0 | 0 | + +Each run settles to `owed: 0` on the relay, so the next carries nothing. +That is the intended steady state: 402 is the recovery path, not the +mechanism. diff --git a/src/client.rs b/src/client.rs index 54d759c..422f014 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2791,6 +2791,23 @@ where // must clamp to it and never raise a lane above what it // advertised. lane_frame_ceiling[i] = fits.min(before); + // Learn any debt carried from an earlier run *before* sizing the + // first POST. The relay's ledger outlives ours (§17.1), so a + // fresh process starts believing it owes nothing and builds a + // body against the whole credit line — while the relay is + // already holding part of it. The first POST is then refused, + // and if the carried debt happens to sit below the dust floor + // the refusal cannot be paid off either. Reconciling here costs + // one GET per lane and makes every later size correct. + match payer.reconcile(&http, pc).await { + Ok(true) => info!(target: "hoverfly::upload", + "lane {i}: relay carries {} PLUR from an earlier run", + payer.account.owed()), + Ok(false) => {} + Err(e) => warn!(target: "hoverfly::upload", + "lane {i}: could not read carried debt ({e}); \ + sizing may be optimistic until the first 402"), + } } } // batch id -> body bytes, so a completed POST can be billed for what @@ -2869,31 +2886,48 @@ where { warn!(target: "hoverfly::upload", "lane {lane}: settle failed: {e}"); } - // Size the next POST to what the lane can afford *now*. - // Without this the body is built from a ceiling computed - // before any debt existed, so concurrent POSTs are each - // sized as if they were the only one in flight and the - // relay refuses their sum — a 402 nothing can pay, since - // the bytes are still on the wire and nothing is owed yet. - let affordable = payer.affordable_frames(); - if affordable > 0 { - sched.set_lane_batch_max(lane, affordable.min(lane_frame_ceiling[lane])); - } + // POST sizing happens per dispatch, in the loop below. } } - // Re-checked on every pass of the loop below, not once before it. - // Each dispatch adds its body to `pending`, so a single up-front - // answer authorises an unbounded run of POSTs against headroom that - // only the first of them actually had. - let dispatch_ok = |payers: &[Option]| { - payment.is_none() - || payers - .iter() - .any(|p| p.as_ref().map(|x| x.has_headroom()).unwrap_or(true)) - }; - - // Hand out everything the scheduler is willing to dispatch. - while dispatch_ok(&payers) && let Some(a) = sched.next(now_ms()) { + // Hand out everything the scheduler is willing to dispatch, resizing + // each metered lane to its *live* headroom immediately before every + // single assignment. + // + // Both halves have to happen per dispatch, not per pass of the outer + // loop. Each POST adds its body to `pending`, so one up-front answer + // authorises a whole run of them against headroom only the first + // actually had — and since the body is built from the lane's + // `batch_max`, a stale ceiling means the second and third POSTs are + // each sized as if they were alone on the lane. That is what earns a + // 402 the client cannot pay: the bytes are in flight, so nothing is + // owed yet and no cheque changes anything. + while let Some(a) = { + let mut anyone_can_take_work = payment.is_none(); + if payment.is_some() { + for (lane, p) in payers.iter().enumerate() { + let Some(p) = p.as_ref() else { + // Unmetered lane in a metered run: nothing to size. + anyone_can_take_work = true; + continue; + }; + let affordable = p.affordable_frames(); + if affordable > 0 { + sched.set_lane_batch_max( + lane, + affordable.min(lane_frame_ceiling[lane]), + ); + } + if p.has_headroom() { + anyone_can_take_work = true; + } + } + } + if anyone_can_take_work { + sched.next(now_ms()) + } else { + None + } + } { let batch: Vec = a .chunks .iter() diff --git a/src/payer.rs b/src/payer.rs index 1568f5f..a4b0f07 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -421,6 +421,24 @@ impl LaneAccount { self.owed_plur = 0; } + /// Take the relay's figure as ours outright, in whichever direction. + /// + /// Only correct after it has **rejected** a cheque: nothing was issued, + /// so there is no cumulative to be inconsistent with, and the relay has + /// just told us what it is prepared to accept. + /// + /// The divergence this repairs is §7.3's ack-tail. A POST whose + /// response stream breaks was still read, so the client bills it — but + /// if the relay's task is cancelled before it commits, its `Admitted` + /// guard releases the reservation and it never books those bytes. The + /// client cannot tell the two apart from its side, so it over-counts, + /// and every subsequent cheque is refused as an overpayment until + /// somebody yields. The relay is the party deciding what to accept, so + /// it yields to the relay. + pub fn sync_relay_debt(&mut self, relay_owed: u128) { + self.owed_plur = relay_owed; + } + /// Adopt a larger debt figure the relay reports for us. /// /// The mirror of [`Self::forgive_phantom_debt`], and the half that is @@ -739,21 +757,16 @@ impl LanePayer { > self.cap_plur } - /// Ask the relay what it thinks we owe, and adopt the figure if it is - /// larger than ours. + /// What the relay's ledger says this account owes, from `/v1/account`. /// - /// Called only when a 402 arrives that our own books say we cannot pay - /// — the deadlock in [`LaneAccount::adopt_relay_debt`]. `/v1/account` - /// is used rather than the number in the 402 body because the body - /// quotes `owed + reserved` *including the refused request*, which is - /// not a payable amount. - /// - /// Returns whether the debt moved. - pub async fn reconcile( + /// The relay is authoritative here — it is the party deciding what to + /// accept — so this is the number both the upward reconcile and a + /// rejected cheque correct themselves against. + async fn relay_owed( &mut self, http: &reqwest::Client, cfg: &PaymentConfig, - ) -> Result { + ) -> Result { let header = self.header(http, cfg).await?.to_string(); let resp = http .get(format!("{}/v1/account", self.base_url.trim_end_matches('/'))) @@ -769,12 +782,29 @@ impl LanePayer { .json() .await .map_err(|e| format!("account decode: {e}"))?; - let owed: u128 = v - .get("owed_plur") + v.get("owed_plur") .and_then(|x| x.as_str()) .ok_or("account: missing owed_plur")? .parse() - .map_err(|e| format!("account: bad owed_plur: {e}"))?; + .map_err(|e| format!("account: bad owed_plur: {e}")) + } + + /// Ask the relay what it thinks we owe, and adopt the figure if it is + /// larger than ours. + /// + /// Called only when a 402 arrives that our own books say we cannot pay + /// — the deadlock in [`LaneAccount::adopt_relay_debt`]. `/v1/account` + /// is used rather than the number in the 402 body because the body + /// quotes `owed + reserved` *including the refused request*, which is + /// not a payable amount. + /// + /// Returns whether the debt moved. + pub async fn reconcile( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + ) -> Result { + let owed = self.relay_owed(http, cfg).await?; // Bounded by the chequebook's balance, deliberately *not* by the // credit line. The line caps what may be newly admitted, not what // may stand: debt reaches the line by construction, and an operator @@ -807,6 +837,21 @@ impl LanePayer { let Some(cumulative) = self.account.next_cumulative() else { return Ok(None); }; + self.pay(http, cfg, cumulative, true).await + } + + /// Sign and present one cheque. + /// + /// `correct_once` allows a single re-present against the relay's own + /// figure when it rejects ours as an overpayment; the retry passes + /// `false` so a relay that keeps refusing cannot loop us. + async fn pay( + &mut self, + http: &reqwest::Client, + cfg: &PaymentConfig, + cumulative: u128, + correct_once: bool, + ) -> Result, String> { // Aggregate exposure across every beneficiary drawn on this one // chequebook (§8.3): the second lane's cheque is what silently // bounces without this. @@ -859,6 +904,20 @@ impl LanePayer { self.account.forgive_phantom_debt(); return Ok(None); } + // Same divergence, partial rather than total: the relay booked + // *less* than we billed ourselves, so our cumulative overshoots + // and it refuses. Nothing was issued, so we can simply take its + // figure and re-present. Without this the overshoot is + // permanent — every later cheque carries it and is refused for + // the same reason, and the lane never settles again. + if correct_once && text.contains("is owed") { + let relay = self.relay_owed(http, cfg).await?; + self.account.sync_relay_debt(relay); + let Some(corrected) = self.account.next_cumulative() else { + return Ok(None); + }; + return Box::pin(self.pay(http, cfg, corrected, false)).await; + } return Err(format!("pay {code}: {}", text.trim())); } // Record the cumulative *before* trusting the reply: we have @@ -1288,6 +1347,39 @@ mod tests { ); } + /// §7.3's ack-tail leaves the two sides disagreeing in the *other* + /// direction, and the client must yield. + /// + /// A POST whose response stream breaks was still read, so the client + /// bills it — but if the relay's task is cancelled before it commits, + /// the `Admitted` guard releases the reservation and it books nothing. + /// The overshoot then rides on every later cheque, each refused for the + /// same reason, and the lane never settles again. Seen over a real + /// proxy: `credits 212640000000 but only 148800000000 is owed`. + #[test] + fn a_relay_that_booked_less_than_we_billed_is_taken_at_its_word() { + let p = Params::default(); + let mut a = LaneAccount::new(p, [3u8; 20]); + // Large enough that both figures clear the dust floor, so the + // assertion is about the correction and not about §10.2. + let sent = 16 * 1024 * 1024u64; + a.record_sent(sent); + a.record_answered(sent, true); + let ours = a.owed(); + assert!(ours > p.min_cheque_plur); + + // The relay booked less than we billed ourselves. + let theirs = ours - p.price_bytes(133 * 1024); + assert!(theirs > p.min_cheque_plur); + a.sync_relay_debt(theirs); + assert_eq!(a.owed(), theirs, "the relay decides what it will accept"); + assert_eq!( + a.next_cumulative(), + Some(theirs), + "and the corrected cheque is exactly its figure" + ); + } + /// Reconciling mid-flight must not bill the same POST twice. /// /// Regression: the relay finishes reading a body and books it, so its From 50dd2d9909b67a38416f88534175492d6eec6e77 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Mon, 10 Aug 2026 17:04:03 +0300 Subject: [PATCH 13/27] feat(pushers): make paying optional per lane, and add the metered VPS lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Payment is a property of a relay, not of the fleet. A client should be able to point at a mix of `open`, soft-metered and hard-metered lanes and use whichever subset it can be served by — but `hard_enforcement` was parsed into `LaneInfo` and then read by nobody, so an unpaying client scheduled work onto a lane that answers 401 for the missing challenge. A 401 is not a 402: it charges lane health and burns one attempt per chunk rediscovering something the lane advertised before the first byte moved. Both drivers now stand down from a hard lane they cannot pay: - native (`src/client.rs`): lanes advertising hard enforcement with no chequebook configured are retired before the first dispatch, and a run where *every* lane is hard fails immediately with that reason instead of grinding through retries. - browser (`UploadSession::set_lane_status`): retires unconditionally — the dApp only stamps, the chequebook lives in the native client. `setLaneStatus` now returns whether the lane was scheduled, and the worker logs the ones it skipped. Soft-metered lanes are deliberately kept by both: they bill and serve, so an unpaying client is served exactly as on `open`. The browser wasm build was broken as a side effect of the incentive work — `LaneInfo::quote` names `crate::payer`, which is behind the `pusher` feature that the dApp's `--no-default-features` build turns off, and the wasm `setLaneStatus` literal was missing the new fields. The quote field is now gated to match its type; `hard_enforcement` and the price stay unconditional, because recognising a lane you cannot pay is exactly the thing a client without the payment stack still needs to do. Adds `pusher.browserbzz.link` to PUSHER_URLS — the first metered production lane, and the only one that doesn't cold-start. The three Render lanes stay `open` (verified: no `payment` block in /v1/status); they're short-lived free tiers, and §5 requires durable storage for metering anyway. Co-Authored-By: Claude Opus 5 (1M context) --- apps/upload/src/config.ts | 12 +++++++++++- apps/upload/src/worker.ts | 23 ++++++++++++++++++++--- docs/pusher-incentives.md | 36 ++++++++++++++++++++++++++++++------ src/client.rs | 28 ++++++++++++++++++++++++++++ src/pushsched.rs | 17 +++++++++++++++++ src/pushsched/tests.rs | 28 ++++++++++++++++++++++++++++ src/wasm.rs | 35 +++++++++++++++++++++++++++++++++-- 7 files changed, 167 insertions(+), 12 deletions(-) diff --git a/apps/upload/src/config.ts b/apps/upload/src/config.ts index af9cc53..db71ba9 100644 --- a/apps/upload/src/config.ts +++ b/apps/upload/src/config.ts @@ -14,6 +14,12 @@ * * Chunks are sharded across the lanes (rendezvous hashing) and pushed * concurrently; a chunk unacked by one lane fails over to the next. + * + * Payment is per lane and optional (docs/pusher-incentives.md). A lane + * advertising `enforcement: "hard"` in its `/v1/status` is dropped from + * rotation at startup by `setLaneStatus`, because this dApp only stamps — + * the chequebook lives in the native client. Such a lane is listed anyway so + * a native `--chequebook` run of the same fleet picks it up. */ export const PUSHER_URLS: string[] = [ 'https://hoverfly-pusher.onrender.com', @@ -21,7 +27,11 @@ export const PUSHER_URLS: string[] = [ 'https://hoverfly-pusher-3.onrender.com', // Hugging Face Space — a different provider/IP-range from Render (probed: // HF permits outbound TCP to bee nodes), for cross-provider lane diversity. - 'https://ivam5567-hoverfly.hf.space' + 'https://ivam5567-hoverfly.hf.space', + // A self-hosted VPS lane, and the first metered one: it runs `--meter` with + // hard enforcement, so it serves paying native clients and is skipped by + // the browser. Unlike the free tiers above it doesn't cold-start. + 'https://pusher.browserbzz.link' ] /** * How long to wait for a relay's `/v1/status` before scheduling it on diff --git a/apps/upload/src/worker.ts b/apps/upload/src/worker.ts index 7d1232f..1b62bc7 100644 --- a/apps/upload/src/worker.ts +++ b/apps/upload/src/worker.ts @@ -64,8 +64,12 @@ interface UploadSession { readonly failed: number readonly hedges: number readonly done: boolean - /** Feed a lane's /v1/status JSON (pool size, batch_max, budget, overlay). */ - setLaneStatus: (lane: number, status: unknown) => void + /** + * Feed a lane's /v1/status JSON (pool size, batch_max, budget, overlay). + * False when the lane was retired instead of scheduled — it enforces + * payment and this build has no chequebook. + */ + setLaneStatus: (lane: number, status: unknown) => boolean /** Next POST to issue, or undefined if nothing is dispatchable now. */ nextRequest: (nowMs: number) => PushRequest | undefined /** One streamed NDJSON ack. Idempotent per address (hedges rely on this). */ @@ -375,10 +379,23 @@ async function pushSession (session: UploadSession, lanes: string[]): Promise { const st = await fetchLaneStatus(u) - if (st !== undefined) session.setLaneStatus(i, st) + if (st === undefined) { usable++; return } // asleep, not refusing — keep it + if (session.setLaneStatus(i, st)) usable++ + else log(`Pusher ${u} requires payment; skipping it (browser uploads are unpaid).`) })) + if (usable === 0) { + throw new Error('every relay in PUSHER_URLS requires payment — the browser cannot pay') + } const pushUrls = lanes.map(u => `${u.replace(/\/+$/, '')}/v1/push`) let lastPost = 0 diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index f02c345..cad4ba3 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -164,13 +164,37 @@ Explicit non-goals, each with its reason: A relay runs in exactly one of two modes, advertised in `/v1/status`: -- **`open`** — today's behaviour, unmetered. The four production lanes in - `apps/upload/src/config.ts:18-25` keep running this. Auth stays "stamp - signer is the live batch's on-chain owner" and nothing is billed. +- **`open`** — today's behaviour, unmetered. The four free-tier lanes in + `apps/upload/src/config.ts` keep running this. Auth stays "stamp signer is + the live batch's on-chain owner" and nothing is billed. - **`metered`** — every byte admitted is billed (§8). There is no free - allowance. - -Four consequences worth stating up front: + allowance. `pusher.browserbzz.link` is the first production lane running + it, with hard enforcement. + +**Mode is per relay, and paying is optional for the client.** A fleet mixes +freely: the same `PUSHER_URLS` holds four `open` lanes and one hard-metered +one, and each client uses the subset it can be served by. The rule is +symmetric on both sides of the wire: + +| relay mode | client has a chequebook | client does not | +| --- | --- | --- | +| `open` | used, nothing billed | used, nothing billed | +| `metered`, soft | used, billed, settles | used, billed, served anyway | +| `metered`, hard | used, billed, settles | **lane retired at startup** | + +Retiring is the load-bearing case. A hard lane answers a push without a +challenge header with 401, and a 401 is not a 402 — it counts against lane +health, so scheduling one anyway spends a retry per chunk rediscovering +something the lane advertised in `/v1/status` before the first byte moved. +Both drivers therefore drop it up front: `src/client.rs` (native) when no +`--chequebook` is configured, and `UploadSession::set_lane_status` +(`src/wasm.rs`) unconditionally, since the browser only stamps and the +chequebook lives in the native client. Soft-metered lanes are *kept* by +both — they bill and serve, so an unpaying client is served exactly as on +`open`. A run whose every lane is hard-metered fails immediately with that +reason rather than stalling. + +Four further consequences worth stating up front: **Metering subsumes the deferred `--push-quota`.** Design §6 proposed capping each batch at its own effective volume per TTL. Under metering the diff --git a/src/client.rs b/src/client.rs index 422f014..84c0d1a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2771,6 +2771,31 @@ where // cannot pay its way out of — it would owe nothing, having had nothing // accepted. let mut infos = infos; + // Paying is optional, so a lane that *requires* payment we cannot make is + // simply not ours to use. Scheduling it anyway means every chunk sent + // there is refused for a missing capability, and since that is not a 402 + // it counts against lane health — so the fleet spends one attempt per + // chunk discovering, repeatedly, something the lane advertised up front. + // + // Soft-metered lanes are kept: they bill and serve, so an unpaying client + // is served exactly like an `open` one. + let unusable: Vec = (0..pusher_urls.len()) + .filter(|&i| { + infos.get(i).is_some_and(|inf: &LaneInfo| inf.hard_enforcement) && payers[i].is_none() + }) + .collect(); + for &i in &unusable { + warn!(target: "hoverfly::upload", + "lane {i} {} enforces payment and this client has none configured; skipping it", + pusher_urls[i]); + } + if !unusable.is_empty() && unusable.len() == pusher_urls.len() { + return Err(ClientError::Pusher( + "every lane requires payment and no chequebook is configured — \ + pass --chequebook, or use a fleet with free lanes" + .into(), + )); + } let mut lane_frame_ceiling: Vec = vec![usize::MAX; infos.len()]; if let Some(pc) = payment { for (i, payer) in payers.iter_mut().enumerate() { @@ -2815,6 +2840,9 @@ where // must not become debt on this side either. let mut in_flight_bytes: HashMap = HashMap::new(); let mut sched = Scheduler::new(infos, cfg); + for &i in &unusable { + sched.retire_lane(i); + } // Frames are held by address, not by index: `admit` de-duplicates, so an // index-parallel Vec would silently skew. Keying by address also lets a // frame be dropped the moment its chunk is acked, which is what keeps a diff --git a/src/pushsched.rs b/src/pushsched.rs index 0a615bb..cd71de7 100644 --- a/src/pushsched.rs +++ b/src/pushsched.rs @@ -120,10 +120,15 @@ pub struct LaneInfo { /// simply not paid and not scheduled for payment. pub price_plur_per_kib: Option, /// True when the lane enforces 402 rather than metering softly. + /// + /// Deliberately *not* behind the `pusher` feature: a client that cannot + /// pay still has to recognise a lane that will refuse it, and the + /// browser build compiles without the payment stack entirely. pub hard_enforcement: bool, /// The whole verified quote, when this lane advertised one. Carried so /// the payment loop has the beneficiary and parameters without /// re-fetching and re-verifying `/v1/status`. + #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub quote: Option, } @@ -772,6 +777,18 @@ impl Scheduler { } } + /// Take a lane out of rotation permanently. + /// + /// For lanes that are unusable by configuration rather than by + /// behaviour — a lane enforcing payment this client cannot make — where + /// there is nothing to discover by trying and every attempt costs the + /// chunks one of their retries. + pub fn retire_lane(&mut self, lane: usize) { + if let Some(l) = self.lanes.get_mut(lane) { + l.health = LaneHealth::Retired; + } + } + /// Re-clamp a lane's frames-per-POST between dispatches. /// /// A metered lane's affordable body shrinks as debt and in-flight bytes diff --git a/src/pushsched/tests.rs b/src/pushsched/tests.rs index 7796445..6c85f6a 100644 --- a/src/pushsched/tests.rs +++ b/src/pushsched/tests.rs @@ -741,3 +741,31 @@ fn work_on_an_unfunded_lane_fails_over_rather_than_stalling() { let other = s.next(10).expect("work must not strand on a paused lane"); assert_ne!(other.lane, lane, "it fails over to the funded lane"); } + +/// Paying is optional (§2): a fleet may mix `open`, soft-metered and +/// hard-metered lanes, and a client with no chequebook must keep using the +/// ones it can. +/// +/// A retired lane is out of rotation for good — the point is that nothing +/// is ever dispatched to it, since a lane that refuses for a missing +/// capability is not a 402 and would charge lane health once per chunk. +#[test] +fn a_retired_lane_never_receives_work() { + let infos: Vec = (0..3).map(|_| LaneInfo::default()).collect(); + let mut sched = Scheduler::new(infos, Config::default()); + sched.admit(addrs(600, 7)); + + // Lane 1 is the one this client cannot pay. + sched.retire_lane(1); + + let mut seen = [0usize; 3]; + while let Some(a) = sched.next(0) { + seen[a.lane] += 1; + sched.on_batch_result(a.batch, BatchOutcome::Answered, 0); + } + assert_eq!(seen[1], 0, "retired lane took {} assignments", seen[1]); + assert!( + seen[0] > 0 && seen[2] > 0, + "the payable lanes must still carry the upload: {seen:?}" + ); +} diff --git a/src/wasm.rs b/src/wasm.rs index b860e96..71161ca 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -1139,8 +1139,18 @@ impl UploadSession { /// Feed a lane's `/v1/status` JSON. Per lane and idempotent: a relay /// that is asleep when polled simply keeps its default priors instead /// of degrading routing for every other lane. + /// + /// Returns `false` when the lane was taken out of rotation instead of + /// scheduled — today that means it enforces payment + /// (`docs/pusher-incentives.md` §7.3) and this build has no way to pay. + /// The browser stamps but never settles: the chequebook lives in the + /// native client, so a hard lane would answer every POST with 401 for a + /// missing capability, and that is not a 402 — it counts against lane + /// health and burns one attempt per chunk rediscovering something the + /// lane announced up front. Soft-metered lanes are kept, since those + /// bill and serve; an unpaying client is served exactly like on `open`. #[wasm_bindgen(js_name = "setLaneStatus")] - pub fn set_lane_status(&mut self, lane: usize, status: JsValue) -> Result<(), JsError> { + pub fn set_lane_status(&mut self, lane: usize, status: JsValue) -> Result { // Round-trip through JSON text rather than pulling in // serde-wasm-bindgen for one struct. let txt = js_sys::JSON::stringify(&status) @@ -1152,6 +1162,21 @@ impl UploadSession { .and_then(|s| s.as_str()) .and_then(|s| hex::decode(s.trim_start_matches("0x")).ok()) .and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok()); + // The advertised `enforcement` is read without verifying the quote's + // signature, unlike the native path. Verification protects a *payer* + // from being overcharged; there is no payment here to protect, and + // the only thing this flag can do is make us decline a lane. A lane + // lying its way out of our traffic is a lane denying its own + // service. + let payment = v.get("payment"); + let hard_enforcement = payment + .and_then(|p| p.get("enforcement")) + .and_then(|x| x.as_str()) + == Some("hard"); + let price_plur_per_kib = payment + .and_then(|p| p.get("price_plur_per_kib")) + .and_then(|x| x.as_str()) + .and_then(|s| s.parse::().ok()); self.sched.set_lane_info( lane, crate::pushsched::LaneInfo { @@ -1170,9 +1195,15 @@ impl UploadSession { .and_then(|p| p.get("live")) .and_then(|x| x.as_u64()) .map(|x| x as usize), + price_plur_per_kib, + hard_enforcement, }, ); - Ok(()) + if hard_enforcement { + self.sched.retire_lane(lane); + return Ok(false); + } + Ok(true) } /// Next POST to issue, or `undefined` if nothing is dispatchable right From 5d657b109579b9eb3bc3b8ccae7e5979f15465c3 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Mon, 10 Aug 2026 17:05:11 +0300 Subject: [PATCH 14/27] fix(build): the relay's cargo feature no longer gates the client's ability to pay one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo build --no-default-features --features cli` — the config Cargo.toml documents for compiling out `bridge` — has not compiled since the incentive layer landed. `client.rs` reaches for `crate::payer`, `crate::meter` and `crate::metered` on every relay push, but all three sat behind `pusher`, and `pusher` is the *relay* feature: it exists to pull in hyper. The split was in the wrong place. A client paying a metered relay needs the challenge wire format, the pricing arithmetic and the payer; it needs no server. So `challenge`, `meter` and `payer` become plain `not(wasm32)`, while `ledger`, `inbound_limit`, `metered` and `pusher` — the ledger the relay bills against and the state machine defending it — stay behind the feature. Breaking the last client→relay edge meant moving the `x-hoverfly-challenge` codec (`CHALLENGE_HEADER`, `IssuedChallenge`, `PresentedChallenge`, `encode_challenge_header`) out of `metered.rs` into `challenge.rs`. That is where it belonged anyway: `challenge.rs` already owns the MAC, the preimage and the verifier, and a format defined by only one of the two ends that speak it is a format that drifts. `hoverfly cashout` moves behind `pusher` with them — it reads the relay's ledger, so without a relay there is nothing to cash. Verified: default features 236 tests, `--no-default-features --features cli` 186 tests (previously would not build), and the dApp's `--no-default-features` wasm build. Also drops `.claude/settings.local.json.tmp.*` from the tree and ignores the directory: a `git add -A` swept one in, and those files quote whole shell commands. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 6 ++ docs/pusher-incentives.md | 20 +++++-- src/bin/hoverfly.rs | 7 ++- src/challenge.rs | 113 ++++++++++++++++++++++++++++++++++++++ src/client.rs | 2 +- src/lib.rs | 22 +++++--- src/metered.rs | 112 ++----------------------------------- src/payer.rs | 12 ++-- src/pusher.rs | 6 +- src/pushsched.rs | 2 +- 10 files changed, 170 insertions(+), 132 deletions(-) diff --git a/.gitignore b/.gitignore index d1dd468..ffc1282 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,12 @@ overlay-nonce # paid what, and restoring a stale copy re-issues cheques the counterparty # has already banked. cheques.json +# Agent-local tool config. `settings.local.json` itself is usually covered by +# a global ignore, but the editor writes `*.tmp..` siblings that +# such a pattern misses — and those files quote whole shell commands, so a +# `git add -A` mid-session can publish a key that was passed on a command +# line. Ignore the whole directory. +.claude/ # `peers.seed.json` is checked in: an IP-diverse cold-start seed # harvested from a long-running daemon (~800 peers across ~800 unique # /32 IPs as of the last refresh; regenerate via diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index cad4ba3..7d35ea7 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -1270,12 +1270,20 @@ vetoed metering outright. Bytes-admitted removes the dependency.)* **Stage 1 — the relay can be paid (native client only), soft mode.** **Relay side shipped.** Enabled with `--meter --origin --beneficiary <0x…> --state-dir

`; without `--meter` nothing below is reachable and -the relay behaves exactly as it does today. Modules: `src/challenge.rs`, -`src/ledger.rs`, `src/metered.rs`, `src/inbound_limit.rs`, plus cheque -recovery in `src/signer.rs`, a `SignedCheque` decoder in -`src/protocols/swap.rs` and chequebook bindings in `src/batch.rs`. -Endpoints: `GET /v1/challenge`, `GET /v1/account`, `POST /v1/pay`, and -metered admission on `POST /v1/push`. +the relay behaves exactly as it does today. Modules: `src/ledger.rs`, +`src/metered.rs`, `src/inbound_limit.rs`, plus cheque recovery in +`src/signer.rs`, a `SignedCheque` decoder in `src/protocols/swap.rs` and +chequebook bindings in `src/batch.rs`. Endpoints: `GET /v1/challenge`, +`GET /v1/account`, `POST /v1/pay`, and metered admission on +`POST /v1/push`. + +Those four sit behind the `pusher` cargo feature, which exists to pull in +hyper — it is the *relay*. The shared and client-side pieces +(`src/challenge.rs`, `src/meter.rs`, `src/payer.rs`) deliberately do not: +a client that pays a metered relay needs the challenge wire format, the +pricing arithmetic and the payer, but no server. `challenge.rs` owns the +whole challenge protocol including the `x-hoverfly-challenge` codec, so +the two ends cannot drift on a format only one of them defines. **Client side shipped** (`src/payer.rs`): signed-quote verification with lane pinning on `(url, node_eth_address, beneficiary)`, challenge parsing diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index 9071682..46211e3 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -815,7 +815,10 @@ enum Commands { /// A cheque is cumulative, so only the newest one per chequebook is /// ever presented; gas is paid once per chequebook, not once per /// cheque received. - #[cfg(unix)] + /// + /// Behind the `pusher` feature because it reads the *relay's* ledger: + /// without a relay there are no cheques to cash. + #[cfg(all(unix, feature = "pusher"))] Cashout { #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] rpc_url: String, @@ -2847,7 +2850,7 @@ async fn main() -> Result<(), Box> { } #[cfg(unix)] - #[cfg(unix)] + #[cfg(all(unix, feature = "pusher"))] Commands::Cashout { rpc_url, key, diff --git a/src/challenge.rs b/src/challenge.rs index 784ed3c..80d9a7e 100644 --- a/src/challenge.rs +++ b/src/challenge.rs @@ -154,6 +154,119 @@ pub fn now_unix() -> u64 { .unwrap_or(0) } +// ---- the header, on the wire ---- +// +// Both ends of the challenge need this codec: the relay issues and decodes, +// the client encodes and presents. It lives here rather than in `metered.rs` +// because `metered.rs` is the relay's state machine — reserving, billing, +// crediting — and a client that pays a relay must be buildable without it. + +/// Header carrying the capability plus the client's proof it holds the +/// account key. One custom header, so the CORS preflight allow-list grows +/// by exactly one entry (§7.2's browser blocker). +pub const CHALLENGE_HEADER: &str = "x-hoverfly-challenge"; + +/// Cap on the header itself. The fields are fixed-width apart from +/// `origin`, so anything larger is not a challenge. +pub const MAX_CHALLENGE_HEADER: usize = 2048; + +/// A capability the relay minted: the authorised fields plus our MAC over +/// them. +pub struct IssuedChallenge { + pub fields: ChallengeFields, + pub nonce: [u8; 32], +} + +impl IssuedChallenge { + pub fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "nonce": format!("0x{}", hex::encode(self.nonce)), + "account": format!("0x{}", hex::encode(self.fields.account)), + "batch": format!("0x{}", hex::encode(self.fields.batch)), + "origin": self.fields.origin, + "expiry": self.fields.expiry_unix, + "max_outstanding_plur": self.fields.cap_plur.to_string(), + "expires_ms": self.fields.expiry_unix.saturating_mul(1000), + }) + } +} + +/// What the client sends back: the capability it was issued plus its +/// signature over the same fields. +pub struct PresentedChallenge { + pub fields: ChallengeFields, + pub nonce: [u8; 32], + pub sig: [u8; 65], +} + +impl PresentedChallenge { + /// `base64(json)` in one header, so the CORS allow-list grows by one. + pub fn decode(raw: &str) -> Result { + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(raw.trim()) + .map_err(|e| format!("challenge header base64: {e}"))?; + let v: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| format!("challenge header json: {e}"))?; + let get = |k: &str| -> Result { + v.get(k) + .and_then(|x| x.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| format!("challenge header: missing {k}")) + }; + let fixed = |k: &str, n: usize| -> Result, String> { + let raw = hex::decode(get(k)?.trim_start_matches("0x")) + .map_err(|e| format!("challenge {k} hex: {e}"))?; + if raw.len() != n { + return Err(format!("challenge {k} must be {n} bytes, got {}", raw.len())); + } + Ok(raw) + }; + let mut account = [0u8; 20]; + account.copy_from_slice(&fixed("account", 20)?); + let mut batch = [0u8; 32]; + batch.copy_from_slice(&fixed("batch", 32)?); + let mut nonce = [0u8; 32]; + nonce.copy_from_slice(&fixed("nonce", 32)?); + let mut sig = [0u8; 65]; + sig.copy_from_slice(&fixed("sig", 65)?); + let expiry_unix = v + .get("expiry") + .and_then(|x| x.as_u64()) + .ok_or("challenge header: missing expiry")?; + let cap_plur: u128 = get("max_outstanding_plur")? + .parse() + .map_err(|e| format!("challenge cap: {e}"))?; + Ok(Self { + fields: ChallengeFields { + account, + batch, + origin: get("origin")?, + expiry_unix, + cap_plur, + }, + nonce, + sig, + }) + } +} + +/// Encode a challenge plus signature into the header value. Client side, +/// and used by the tests to drive the relay path end to end. +pub fn encode_challenge_header(issued: &IssuedChallenge, sig: &[u8; 65]) -> String { + use base64::Engine; + let body = serde_json::json!({ + "nonce": format!("0x{}", hex::encode(issued.nonce)), + "account": format!("0x{}", hex::encode(issued.fields.account)), + "batch": format!("0x{}", hex::encode(issued.fields.batch)), + "origin": issued.fields.origin, + "expiry": issued.fields.expiry_unix, + "max_outstanding_plur": issued.fields.cap_plur.to_string(), + "sig": format!("0x{}", hex::encode(sig)), + }); + base64::engine::general_purpose::STANDARD.encode(body.to_string()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/client.rs b/src/client.rs index 84c0d1a..ef1c40e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3455,7 +3455,7 @@ async fn post_batch_streaming( let mut req = http.post(push_url).body(body); if let Some(c) = challenge { - req = req.header(crate::metered::CHALLENGE_HEADER, c); + req = req.header(crate::challenge::CHALLENGE_HEADER, c); } let resp = match req.send().await { Ok(r) => r, diff --git a/src/lib.rs b/src/lib.rs index 8b2130d..a5d98e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,24 +68,32 @@ pub mod daemon; #[cfg(not(target_arch = "wasm32"))] pub mod inbound; -#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] +// The incentive layer has two halves, and only one of them is the relay. +// `pusher` is the relay feature — it exists to pull in hyper. A *client* +// paying a metered relay needs the challenge wire format, the pricing +// arithmetic and the payer, but no server: gating those behind `pusher` +// made `--no-default-features --features cli` fail to build, because +// `client.rs` reaches for them unconditionally on every relay push. +#[cfg(not(target_arch = "wasm32"))] pub mod challenge; +#[cfg(not(target_arch = "wasm32"))] +pub mod meter; + +#[cfg(not(target_arch = "wasm32"))] +pub mod payer; + +// Relay-side only, and genuinely so: these hold the ledger the relay bills +// against and the state machine that defends it. #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub mod inbound_limit; #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub mod ledger; -#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] -pub mod meter; - #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub mod metered; -#[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] -pub mod payer; - #[cfg(all(feature = "pusher", not(target_arch = "wasm32")))] pub mod pusher; diff --git a/src/metered.rs b/src/metered.rs index 58c2537..c970c8b 100644 --- a/src/metered.rs +++ b/src/metered.rs @@ -10,7 +10,9 @@ //! Stage 2 and flips one flag — the arithmetic that decides "over cap" is //! already computed here so the two modes cannot disagree about it. -use crate::challenge::{ChallengeError, ChallengeFields}; +use crate::challenge::{ + ChallengeError, ChallengeFields, IssuedChallenge, MAX_CHALLENGE_HEADER, PresentedChallenge, +}; use crate::inbound_limit::InboundLimiter; use crate::ledger::{Ledger, LedgerError}; use crate::meter::Params; @@ -19,15 +21,6 @@ use std::collections::HashMap; use std::sync::Mutex; use std::time::{Duration, Instant}; -/// Header carrying the capability plus the client's proof it holds the -/// account key. One custom header, so the CORS preflight allow-list grows -/// by exactly one entry (§7.2's browser blocker). -pub const CHALLENGE_HEADER: &str = "x-hoverfly-challenge"; - -/// Cap on the header itself. The fields are fixed-width apart from -/// `origin`, so anything larger is not a challenge. -const MAX_CHALLENGE_HEADER: usize = 2048; - /// Accounts allowed to hold a live reservation at once (§7.2). The map is /// attacker-influenced — one entry per batch in standing — so it is capped /// and sheds beyond. @@ -291,25 +284,6 @@ impl Metered { } } -pub struct IssuedChallenge { - pub fields: ChallengeFields, - pub nonce: [u8; 32], -} - -impl IssuedChallenge { - pub fn to_json(&self) -> serde_json::Value { - serde_json::json!({ - "nonce": format!("0x{}", hex::encode(self.nonce)), - "account": format!("0x{}", hex::encode(self.fields.account)), - "batch": format!("0x{}", hex::encode(self.fields.batch)), - "origin": self.fields.origin, - "expiry": self.fields.expiry_unix, - "max_outstanding_plur": self.fields.cap_plur.to_string(), - "expires_ms": self.fields.expiry_unix.saturating_mul(1000), - }) - } -} - #[derive(Debug, Clone)] pub struct VerifiedChallenge { pub account: [u8; 20], @@ -317,85 +291,6 @@ pub struct VerifiedChallenge { pub cap_plur: u128, } -/// What the client sends back: the capability it was issued plus its -/// signature over the same fields. -struct PresentedChallenge { - fields: ChallengeFields, - nonce: [u8; 32], - sig: [u8; 65], -} - -impl PresentedChallenge { - /// `base64(json)` in one header, so the CORS allow-list grows by one. - fn decode(raw: &str) -> Result { - use base64::Engine; - let bytes = base64::engine::general_purpose::STANDARD - .decode(raw.trim()) - .map_err(|e| format!("challenge header base64: {e}"))?; - let v: serde_json::Value = - serde_json::from_slice(&bytes).map_err(|e| format!("challenge header json: {e}"))?; - let get = |k: &str| -> Result { - v.get(k) - .and_then(|x| x.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| format!("challenge header: missing {k}")) - }; - let fixed = |k: &str, n: usize| -> Result, String> { - let raw = hex::decode(get(k)?.trim_start_matches("0x")) - .map_err(|e| format!("challenge {k} hex: {e}"))?; - if raw.len() != n { - return Err(format!("challenge {k} must be {n} bytes, got {}", raw.len())); - } - Ok(raw) - }; - let mut account = [0u8; 20]; - account.copy_from_slice(&fixed("account", 20)?); - let mut batch = [0u8; 32]; - batch.copy_from_slice(&fixed("batch", 32)?); - let mut nonce = [0u8; 32]; - nonce.copy_from_slice(&fixed("nonce", 32)?); - let mut sig = [0u8; 65]; - sig.copy_from_slice(&fixed("sig", 65)?); - let expiry_unix = v - .get("expiry") - .and_then(|x| x.as_u64()) - .ok_or("challenge header: missing expiry")?; - let cap_plur: u128 = get("max_outstanding_plur")? - .parse() - .map_err(|e| format!("challenge cap: {e}"))?; - Ok(Self { - fields: ChallengeFields { - account, - batch, - origin: get("origin")?, - expiry_unix, - cap_plur, - }, - nonce, - sig, - }) - } -} - -/// Encode a challenge plus signature into the header value. Client side, -/// and used by the tests to drive the relay path end to end. -pub fn encode_challenge_header( - issued: &IssuedChallenge, - sig: &[u8; 65], -) -> String { - use base64::Engine; - let body = serde_json::json!({ - "nonce": format!("0x{}", hex::encode(issued.nonce)), - "account": format!("0x{}", hex::encode(issued.fields.account)), - "batch": format!("0x{}", hex::encode(issued.fields.batch)), - "origin": issued.fields.origin, - "expiry": issued.fields.expiry_unix, - "max_outstanding_plur": issued.fields.cap_plur.to_string(), - "sig": format!("0x{}", hex::encode(sig)), - }); - base64::engine::general_purpose::STANDARD.encode(body.to_string()) -} - struct StateCache { map: HashMap<[u8; 20], (crate::batch::ChequebookState, Instant)>, order: std::collections::VecDeque<[u8; 20]>, @@ -468,6 +363,7 @@ impl DeployedCache { #[cfg(test)] mod tests { use super::*; + use crate::challenge::encode_challenge_header; use crate::signer::SwarmSigner; const KEY: &str = "0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318"; diff --git a/src/payer.rs b/src/payer.rs index a4b0f07..640e5c7 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -268,7 +268,7 @@ impl OfferedChallenge { let sig = signer .sign_push_challenge(&sol, chain_id) .map_err(|e| e.to_string())?; - let issued = crate::metered::IssuedChallenge { + let issued = crate::challenge::IssuedChallenge { fields: crate::challenge::ChallengeFields { account: self.account, batch: self.batch, @@ -278,7 +278,7 @@ impl OfferedChallenge { }, nonce: self.nonce, }; - Ok(crate::metered::encode_challenge_header(&issued, &sig)) + Ok(crate::challenge::encode_challenge_header(&issued, &sig)) } /// Re-fetch before this, rather than racing the expiry with a POST in @@ -770,7 +770,7 @@ impl LanePayer { let header = self.header(http, cfg).await?.to_string(); let resp = http .get(format!("{}/v1/account", self.base_url.trim_end_matches('/'))) - .header(crate::metered::CHALLENGE_HEADER, header) + .header(crate::challenge::CHALLENGE_HEADER, header) .timeout(std::time::Duration::from_secs(60)) .send() .await @@ -884,7 +884,7 @@ impl LanePayer { let header = self.header(http, cfg).await?.to_string(); let resp = http .post(format!("{}/v1/pay", self.base_url.trim_end_matches('/'))) - .header(crate::metered::CHALLENGE_HEADER, header) + .header(crate::challenge::CHALLENGE_HEADER, header) .header("content-type", "application/json") .body(body) .timeout(std::time::Duration::from_secs(120)) @@ -1066,7 +1066,11 @@ mod tests { assert!(matches!(e, QuoteError::BadParams(_)), "got {e:?}"); } + // The only test here that needs the relay half, to prove the two ends + // of the header agree. Everything else about paying is testable without + // a server, which is the point of building the client half separately. #[test] + #[cfg(feature = "pusher")] fn a_challenge_round_trips_into_a_header_the_relay_accepts() { use crate::ledger::Ledger; use crate::metered::{MeterConfig, Metered}; diff --git a/src/pusher.rs b/src/pusher.rs index ed9328a..6216441 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -601,7 +601,7 @@ fn admit_metered( }; let raw = req .headers() - .get(crate::metered::CHALLENGE_HEADER) + .get(crate::challenge::CHALLENGE_HEADER) .and_then(|v| v.to_str().ok()) .unwrap_or_default(); // **Soft mode never refuses** (§7.1). A request with no challenge is an @@ -832,7 +832,7 @@ async fn pay_response(state: Arc, req: Request) -> }; let header = req .headers() - .get(crate::metered::CHALLENGE_HEADER) + .get(crate::challenge::CHALLENGE_HEADER) .and_then(|v| v.to_str().ok()) .unwrap_or_default() .to_string(); @@ -982,7 +982,7 @@ fn account_response(state: &State, headers: &hyper::HeaderMap) -> Response, } From 07ba8d5697ddc686557cf394d2d02bdb9c96d0f5 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 16:40:57 +0300 Subject: [PATCH 15/27] fix(metered): bound a relay's claimed debt by the ceiling it signed, not by the chequebook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2 justified the one-directional trust model with "relays are a curated, pinned set — four URLs, all ours". That is not true of the system, only of one client's configuration. A relay is a plain HTTP service: there is no registry, no discovery and no list to be admitted to, and any client can point `--pusher` at any URL. `PUSHER_URLS` is the dApp's default fleet, not a federation roster. The real asymmetry is **pinning, not curation** — a client verifies the signed quote and pins `(url, node_eth_address, beneficiary)` before sending a byte, while a relay gets whoever POSTs. That still points every relay-side defence at the client, so nothing downstream of §2 changes. What it does change is how much a relay's own assertions may be trusted. `LanePayer::reconcile` adopted whatever `owed` the relay reported, bounded only by the chequebook balance. Under the curated premise that was defensible. Under the real one it is not: any lane a client points at could answer an unpayable 402 by naming the entire chequebook, and `settle` would sign for it. The advertised worst case is one credit line — ~$0.0024; the actual one was the funded balance. The sound bound is the quote's `max_outstanding_plur`. Admission refuses whenever `outstanding + reserve > cap`, and §10.3 makes every per-batch cap `min(value / credit_ratio, ceiling)` — so no debt can legitimately stand above the ceiling. Unlike the per-batch line it is a constant of the signed quote and does not shrink as the batch decays, which is what §17.1 rejected the line for. The balance check stays as a looser second gate. Split the decision out as `check_reported_debt` so it is testable without a server, with regression tests for both gates. Also corrects §2, §7.3, §8.4, §15 and §17.1, which all inherited the curation premise, and replaces "governed socially: the lane is removed from PUSHER_URLS" with the four bounds that actually protect a client — self- computed bill, pinned price, capped exposure, measured outcomes. Adds docs/pusher-incentives-slides.md: a 41-slide Marp deck covering the design and the six bugs from §17. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 580 +++++++++++++++++++++++++++++++ docs/pusher-incentives.md | 113 ++++-- src/payer.rs | 113 +++++- 3 files changed, 757 insertions(+), 49 deletions(-) create mode 100644 docs/pusher-incentives-slides.md diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md new file mode 100644 index 0000000..a5ff91d --- /dev/null +++ b/docs/pusher-incentives-slides.md @@ -0,0 +1,580 @@ +--- +marp: true +theme: default +paginate: true +header: 'Paying for relay — SWAP cheques for hoverfly pushers' +--- + +# Paying for relay + +## SWAP cheques for hoverfly pushers + +Theory, and what happened when we ran it + +
+ +*Companion to `docs/pusher-incentives.md`. Stages 0–1 shipped; one metered lane in production.* + +--- + +# The problem: the relay eats a cost it did not incur + +In a **native** upload, your own machine opens the pushsync streams. Bee debits *you*: + +``` +price(po) = (32 − po) × 10 000 accounting units +``` + +Put a relay in the middle and that debt moves **wholesale to the relay** — it is the peer bee sees, so it is the peer bee charges. + +The browser client that caused the traffic pays **nothing but postage**. + +> Today this is booked as accepted risk: *"worst case = the platform's free egress for the month burned, $0 lost."* + +--- + +# What changes, precisely + +| | open (today) | metered | +|---|---|---| +| client → relay | nothing | `4.8e8` PLUR per KiB of body sent | +| relay → bee | free pseudosettle | **unchanged** — free pseudosettle | +| relay's egress | unrecovered | recovered above the cashout threshold | + +Relay→bee settlement stays free because the relay is **session- and RTT-bound, not credit-bound**: bee grants 4.5e6 accounting units/s per peer, ≈ 2 400 chunks/s across a 128-session pool, against **~150 chunks/s** actually measured. Buying credit buys nothing. + +> The recovered amount is small. A relay pushing 100 GB of egress a month is moving ~27 GiB of payload, which at $0.02/GiB is **$0.54**. + +--- + +# Part I — Theory + +--- + +# Everything follows from one asymmetry + +> **The client chose its relay. The relay did not choose its client.** + +A relay is a plain HTTP service. Anyone can run one; there is no registry, no discovery, no allowlist to be admitted to. What creates the asymmetry is **pinning, not curation**: before it sends a byte, a client verifies the signed quote and pins `(url, node_eth_address, beneficiary)`. The relay has no equivalent — a client is whoever POSTs. + +**So every defence the relay has points at the client**, and the relay's goal is: *a client cannot obtain service without paying, and cannot lie about what it owes.* + +The client's protections are different in kind — arithmetic and exposure limits, not authentication. Next slide. + +--- + +# What protects the client, then + +Not cryptography. Four bounds, none of which need the relay to be trustworthy: + +- **It bills itself.** `owed` is computed from bytes the client sent, so a relay over-reporting `kib_admitted` is immediately visible and attributable (§8.4). +- **It pinned the price.** `price_plur_per_kib` comes from a quote signed by the node identity, echoed verbatim in every 402. +- **Exposure is capped at the credit line** — ~$0.0024 at the global ceiling, and `remaining_value ÷ 1000` for a small batch. +- **It measures outcomes.** A lane that takes bytes and delivers poorly is deweighted by the scheduler's existing EWMA. + +> The curated-set premise was hiding a gap in the fourth: on an unpayable 402 the client **adopts the relay's own `owed`**, and that used to be bounded only by the chequebook balance. It is now bounded by `max_outstanding_plur` from the quote the relay signed — the credit it granted is the most it can claim. + +--- + +# What the asymmetry corrected + +Three rounds of adversarial review. The most important finding was structural, not local: + +> The design was building **two-sided cryptographic verification** for a **one-sided trust relationship.** + +Roughly half of it defended the client against the relay — against lanes we run ourselves. Three concrete costs: + +- **Billing unit was a third party's signature** (a pushsync receipt), so it was forgeable and needed a staking-registry anchor +- **Unbounded residual** — the invoice was set-valued, with no bound on retained receipts +- **A kill criterion nobody had measured**: the staked fraction of receipt signers, which could have vetoed the whole thing + +--- + +# What we borrow from SWAP + +| Borrowed | Why | +|---|---| +| `ERC20SimpleSwap` + canonical factory | Audited, deployed, in production. Nothing to write. | +| EIP-712 cheque | `Cheque(chequebook, beneficiary, cumulativePayout)` | +| Cumulative-payout monotonicity | Loss-tolerant *and* replay-proof | +| Funding check | …but `liquidBalanceFor(us)`, **not** `balance()` — bee's version is unsound | +| Reservation against concurrent issuance | Needed on **both** sides | +| Payee-only role | The beneficiary is a plain **EOA**. A payee needs no contract. | + +Gnosis factory: `0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420` + +--- + +# What we drop + +| Dropped | Reason | +|---|---| +| swap libp2p stream, `Handshake`, `EmitCheque` | We're on HTTP already | +| priceoracle, `exchange`, `deduction` | Relay quotes PLUR directly | +| accounting-unit indirection | One unit. No conversion. | +| ghost balances, tolerance, trust ramp | No analogue in request/response | +| `StakeRegistry` snapshot | Nothing left to anchor — see next section | + +> **Both counterparties are hoverfly.** Bee is not a party to the payment. We borrow SWAP's *contracts and cheque format*, not its protocol. + +--- + +# Identity: the account is the batch owner + +The relay's existing auth already establishes on-chain identity for every push: the stamp signature must recover to the address `PostageStamp` reports as the batch's owner. + +> **Account = the batch-owner EOA.** +> A cheque is valid for that account **iff** its chequebook's on-chain `issuer()` equals the same EOA. +> **Credit is keyed one level finer — on the *batch*.** + +No session tokens. No registration. No extra protocol message. + +In a browser: the session key already owns the batch, so cheques sign with **zero wallet prompts**. + +--- + +# The central decision: bill bytes admitted + +> **`owed = (kib_admitted − kib_dedup) × price_plur_per_kib`** + +That is the whole billing rule. One property matters more than everything else in the design: + +> **The client cannot lie about it — the client produced the bytes and the relay counted them.** + +- No third-party attestation to forge +- No signature to replay +- No chain state to disagree about +- No relay assertion taken on faith + +Both numbers are known to both parties *before* any push work happens. + +--- + +# What that replaced + +The earlier draft billed per **verified pushsync receipt**. To make a *third party's* signature into a billing input, it needed: + +- a staking-registry anchor (to stop trivial forgery) +- a set-valued invoice (to stop self-replay) +- a sampled retrieval audit (to catch fabrication) +- a staked-set log sweep on both sides +- a shared predicate module, byte-identical on both sides, **with no adjudicator if it drifted** + +Changing the unit **deleted the entire apparatus and every attack against it.** + +> Receipts are still forwarded — as *telemetry*, feeding lane weighting. They do not enter the invoice. + +--- + +# Bytes, not successful pushes + +Bytes are what the relay **spends money on**. + +Egress is incurred on *attempts*: the 3-way peer race and the shallow retries happen whether or not a chunk lands. + +Billing successes would mean the relay eats the cost of every failure. + +Instead — two mechanisms, each doing what it is good at: + +| Concern | Mechanism | +|---|---| +| Relay recovers what it spends | Bill attempts | +| Client protects itself from a lane that spends without succeeding | **Deweight it in the scheduler** (already exists, already works) | + +--- + +# Dedup hits are billed at zero + +A frame served from the recent-ack cache does no push work, so its bytes are subtracted. + +This is the **one** place a relay assertion enters the bill — the relay claims *"this was a dedup hit"*. + +It is safe for a structural reason: + +> The claim only ever **lowers** the amount owed. + +A relay has no incentive to make it falsely, and a client that disagrees is disagreeing in its own favour. + +--- + +# Cumulative cheques + +A cheque is a running total for one `(chequebook, beneficiary)` pair. Three properties fall out: + +- **Loss-tolerant** — a dropped `/v1/pay` costs nothing; the next cheque supersedes it. No retry state machine. +- **Replay-proof** — strict monotonicity means a re-presented cheque credits zero. +- **Gas-amortizing by construction** — the relay cashes only the *latest* cumulative, so gas is paid once per **account**, not per cheque. + +**Per-chunk cheques are rejected:** ~137 B per 4 KiB chunk, an EIP-712 signature on the hot path, and cumulative payouts are *serial* per `(issuer, beneficiary)` pair — which forces a total order on chunks within a lane, removing the concurrent multi-POST pipelining the scheduler depends on. + +--- + +# One subtlety that bricks a real deployment + +A cumulative is per `(chequebook, beneficiary)`. A **lane** is a URL. + +One operator running four lane URLs behind one beneficiary EOA is the obvious deployment. + +If the client tracks cumulatives **per lane**, that configuration bricks: + +``` +lane 1 issues cumulative 10 +lane 2, counting from its own zero, issues 8 +relay applies ErrChequeNotIncreasing → rejected, forever +``` + +> **Key the client's cumulative store on `(chequebook, beneficiary)`** — not on lane, not on overlay. + +Detecting the sharing is free: the beneficiary is in the signed quote. + +--- + +# Wire protocol + +Push frames are **unchanged**. Payment is out-of-band — it must not sit on the hot path, and a payment failure must not fail a push. + +| Endpoint | Shape | +|---|---| +| `GET /v1/status` | signed `payment` block; `mode: open \| metered` | +| `GET /v1/challenge` | `{nonce, expires_ms, max_outstanding_plur}` — stateless MAC | +| `GET /v1/account` | authenticated. `owed`, `reserved`, `outstanding`, `kib_admitted`, … | +| `POST /v1/pay` | body = `SignedCheque` JSON | +| `POST /v1/push` | `402 Payment Required` when over cap | + +`/v1/account` is authenticated because unauthenticated it is a per-identity **volume oracle** over on-chain-enumerable batch owners. + +--- + +# Admission: why a challenge at all + +The naive claim: *"402 is easy — `/v1/push` commits its status before processing any chunk."* + +True, and that is exactly the **problem**: at that moment the relay does not yet know **whose account to check**. The account only exists after `stamp::validate` and `resolve_owner` — both inside the spawned task. + +Hoisting them means up to **512 ecrecovers (~40 ms) plus an RPC round-trip synchronously in front of every response** — the precise unauthenticated amplification surface we spend a whole section defending. + +--- + +# The challenge is a capability + +Standing is resolved once when the challenge is **issued**. The credit line is baked into the nonce. `/v1/push` admission then reads **no chain state at all**. + +``` +GET /v1/challenge?account=A&batch=B + → resolve standing(B) (cached per batch, TTL) + → require owner(B) == A, else 403 + → cap = credit_line(standing(B)) + → nonce = HMAC(relay_secret, preimage(A, B, origin, expiry, cap)) +``` + +Admission becomes: verify MAC (constant-time) → verify origin → verify client signature → `reserve = ceil(len/1024) × price`; if `outstanding + reserve > cap` → **402, before reading the body**. + +**Stateless.** No server-side nonce table, so a free `GET /v1/challenge` cannot exhaust memory. + +--- + +# Two ways the binding silently becomes a no-op + +**1. The preimage is fixed-width and domain-tagged, not a concatenation.** + +`origin` is variable-length, so a bare concatenation makes `("host.a","bc")` and `("host.ab","c")` share a preimage — one nonce valid for two hostnames. + +**2. `origin` must be *configured*, not derived.** + +The obvious implementation compares the challenge's `origin` against the `Host` header. That is a **no-op** — `Host` is supplied by the same client supplying the challenge. + +> An attacker replaying a victim's signature at relay B just sends `Host: relay-b.example`. The comparison passes. The cross-relay replay is restored **while the doc claims it is closed.** + +--- + +# Deriving the credit line instead of asserting it + +The tempting argument: *"an account is a batch owner, a live batch costs real BZZ, so the margin is three orders of magnitude."* **False.** + +The relay checks **liveness**, and liveness is satisfied by the cheapest batch the contract accepts — minimum depth, minimum validity — a fraction of a cent. At a flat credit line the real Sybil margin is of order **1×**. + +> **`max_outstanding(A,B) = min(remaining_value_plur(B) ÷ credit_ratio, max_outstanding_plur)`** +> with `credit_ratio = 1000` + +The margin is now **1000× by construction, independent of batch size.** There is no cheap corner of the parameter space, because the *ratio* is the invariant. + +--- + +# Parameters, and the invariant that must hold + +> **`min_cheque_plur ≤ settle_every_plur < max_outstanding_plur`** +> A client that is 402'd must always be able to clear it with a cheque for exactly what it owes. + +| parameter | PLUR | in payload | +|---|---:|---:| +| `price_plur_per_kib` | 4.8e8 | 1 KiB | +| `min_cheque_plur` | 3.9e12 | ~8 MiB | +| `settle_every_plur` | 1.56e13 | ~32 MiB | +| `max_outstanding_plur` | 6.22e13 | ~127 MiB (*ceiling*, not the cap) | + +An early draft published `min_cheque` **87× larger** than `settle_every`. Every metered account would have bricked: accrue → cross → sign → **rejected as dust** → accrue → 402 → the only clearing cheque is 21× what is owed. **No exit.** + +--- + +# Reservation: bee's `reserve` was needed after all + +A monotone debit counter is **not** sufficient. `/v1/push` deliberately does not serialize, so N concurrent POSTs each read `outstanding` before any of them debits. + +A *polite* client at the relay's own advertised `inflight_max` of 8 overshoots the cap on its own. + +Fix: reserve `ceil(Content-Length / 1024) × price` **atomically at admission**, release the remainder at completion. + +> **But `reserved` must not be persisted.** A reservation belongs to an in-flight POST, and no in-flight POST survives a restart — there is no task left to release it. + +Persist `owed`, `last_cumulative`, the chequebook binding. **Reconstruct `reserved` as zero at boot.** + +--- + +# Pricing: the cost basis + +| | per 4 KiB chunk relayed | +|---|---| +| Delivery on the wire | ≈ 4.4 KiB | +| Peer race (`CHUNK_PEER_PARALLELISM = 3`) | **×3** | +| Shallow retries at pool 128 | **×1.15** | +| **Egress per chunk** | **≈ 15 KiB** | +| **Egress per GiB of payload** | **≈ 3.7 GiB** | + +Suggested price **$0.02/GiB** — ~5× a VPS's raw bandwidth cost, ~18× cheaper than AWS egress. + +> **`price_plur_per_kib ≈ 4.8 × 10⁸`** (1 BZZ = 10¹⁶ PLUR) + +Flat per KiB, deliberately: any curve steeper than flat re-introduces a per-item number for the two sides to disagree about. + +--- + +# On per-GB-billed hosts, metering loses money + +At 3.7 GiB of real egress per GiB of payload: + +| | per GiB of payload | +|---|---:| +| AWS egress at $0.09/GB | **−$0.33** | +| Revenue at $0.02/GiB | **+$0.02** | +| Net | **−$0.31** | + +So metered mode only clears cost on **flat-rate or included bandwidth**. A relay on per-GB egress should run `open` and absorb the quota. + +That is the same host class already required for durable storage (§11.4), so the two constraints select the same machines. + +--- + +# Revenue per account vs. cashout gas + +Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ **$0.0005** on Gnosis. + +| account's lifetime traffic | revenue | gas | gas as % | +|---|---:|---:|---:| +| 71 MB (one browser upload) | $0.0014 | $0.0005 | **36 %** | +| 5 GiB (cashout threshold) | $0.10 | $0.0005 | **0.5 %** | + +Because cheques are cumulative, gas is paid once per **account**, not per cheque — so the ratio improves with every return visit, and accounts below the threshold are written off unclaimed. + +A relay whose traffic is entirely one-shot uploads should run `open`. + +--- + +# Attack surface — the three that matter + +**Stamp replay becomes billing griefing** *(introduced)* +Swarm stamps are public, and a relay holds every stamp it ever relayed. Replay a victim's stamps at a metered relay and the work bills to the *victim*. Cost to attacker: zero. +→ Closed by the **account-signed** challenge + one batch per POST. + +**The withdraw race** *(inherited)* +Chequebooks deploy with a hard-deposit timeout of zero, so the balance stays liquid. The funding check is true **at acceptance time, not at cashout time.** Bee has the identical exposure. + +**Relay state loss is an unbounded free-service loop** *(introduced)* +An ephemeral filesystem turns one signature into unlimited free service. +→ Durable storage is a **requirement**, not a recommendation. + +--- + +# Part II — Practice + +--- + +# Modes, and the optionality rule + +| relay mode | client has a chequebook | client does not | +|---|---|---| +| `open` | used, nothing billed | used, nothing billed | +| `metered`, soft | used, billed, settles | **used, billed, served anyway** | +| `metered`, hard | used, billed, settles | **lane retired at startup** | + +Retiring matters because a hard lane answers an unchallenged push with **401**, and only a 402 is exempt from **lane health**. Scheduling one anyway costs each chunk one of its `max_attempts` retries, per chunk, to rediscover what `/v1/status` already stated. + +Both drivers drop it up front — native when no `--chequebook` is configured, browser unconditionally. + +--- + +# Soft mode is an instrument, not a migration path + +Soft mode meters, reports and accepts cheques, but **never answers 402**. + +**It still requires the challenge.** An earlier draft implied unchallenged requests should be served — which would make metering bypassable *by omitting a header*. That is not a degraded mode; it is no mode at all. + +> What soft mode drops is enforcement of the cap, **not authentication**. + +A relay flipping to `--meter` therefore *does* break clients that predate the protocol. Acceptable: the only dApp using these lanes ships alongside them. + +--- + +# Six bugs that only a *running* relay could find + +None is reachable from a single upload against a fresh relay. All six survived the full test suite **and** the Stage 1 round-trip. + +Reaching them needed, simultaneously: + +- a relay whose ledger **persists across client runs** +- enough concurrency to have several POSTs on the wire at once +- a batch **spent down far enough that its credit line binds** + +> §17.3 is the one to generalise from: it is not a coding error but an **invariant checked against the wrong quantity**, and it only becomes reachable once a real batch's value has decayed below ~0.39 BZZ. + +--- + +# The six + +| # | Bug | +|---|---| +| 17.1 | Debt the relay carried **across sessions** could not be paid | +| 17.2 | The headroom guard admitted a *frame*, then sent a *batch* | +| 17.3 | §10.1's invariant checked against the wrong quantity | +| 17.4 | A lane refused for **bytes in flight** was parked for good | +| 17.5 | A broken response stream made **every later cheque bounce** | +| 17.6 | The first POST of a run was sized **before the debt was known** | + +--- + +# 17.1 — a slow-motion deadlock + +The dust floor guarantees a run ends owing something: the residual below `min_cheque_plur` is left unpaid, because a cheque for it would be refused. + +The relay is **right** to keep counting it — forgiving it would make *"stay under the floor"* a way to be served free. + +But the client's books are per-process. The next run starts believing it owes **nothing**, and the relay's `owed` only ever grows. Once the carry crosses the cap, the first POST is refused — and the refusal is **unpayable**, because the cheque is computed from the client's own `owed`, which is zero. + +> Observed live: a second upload failing **151/151** against a relay carrying 290,400,000,000 PLUR. + +Fix: **ask rather than remember** — reconcile against `GET /v1/account`. + +--- + +# 17.1 — three ways to get the fix wrong + +Each of these was wrong in a draft: + +- **Use `owed`, not the 402 body's `outstanding_plur`.** `reserve()` adds the reservation *before* computing what it reports, so the body includes the request just refused. Adopting it over-pays by exactly that body — and the next cheque bounces as an **overpayment**. + +- **Exclude reservations on our side too.** Bytes still in flight are already held in `pending`, and would be billed twice when those POSTs land. *Under*-counting is safe and self-correcting; *over*-counting is a refused cheque. + +- **Bound by the quote's signed ceiling** — not the per-batch line, and not the balance. The line shrinks as the batch decays, so it falls below debt properly incurred earlier, and rejecting on that basis **preserves the deadlock**. The balance is far too loose: it lets any lane a client points at name the whole chequebook. Admission refuses above the cap and every cap is `min(value ÷ ratio, ceiling)`, so above the ceiling the debt cannot exist. + +--- + +# 17.3 — the invariant was checked against the wrong quantity + +`Params::validate` checked `min_cheque ≤ settle_every < max_outstanding` against **`max_outstanding_plur`** — the global ceiling. + +But the line that actually binds is **per batch**: + +``` +min(remaining_value / credit_ratio, ceiling) +``` + +Below ~0.39 BZZ of batch value, the configured dust floor **exceeds everything the account can owe**. It accrues to its cap and can never write an acceptable cheque. + +> Permanent refusal. Nothing broken. No error anywhere. + +Thresholds are now resolved via `Params::effective(cap)` on **both** sides. + +--- + +# Results — after all six fixes + +Hard-mode relay, uploads 128 KiB → 4 MiB: + +| payload | frames acked | 402s | stuck | rejected cheques | +|--------:|-------------:|-----:|------:|-----------------:| +| 128 KiB | 43/43 | 0 | 0 | 0 | +| 512 KiB | 151/151 | 0 | 0 | 0 | +| 1 MiB | 290/290 | 2 | 0 | 0 | +| 4 MiB | 1122/1122 | 4 | 0 | 0 | + +The remaining 402s are the **intended** kind: the line genuinely fills, the client pays or waits, the lane resumes. + +--- + +# Results — over public HTTPS, with §17.6 in place + +Repeated through a reverse proxy, with the client learning its carried debt *before* sizing anything: + +| run | payload | frames acked | 402s | rejected cheques | +|----:|--------:|-------------:|-----:|-----------------:| +| 1 | 2 MiB | 567/567 | 0 | 0 | +| 2 | 2 MiB | 567/567 | 0 | 0 | +| 3 | 2 MiB | 567/567 | 0 | 0 | + +Each run settles to **`owed: 0`** on the relay, so the next carries nothing. + +> That is the intended steady state: **402 is the recovery path, not the mechanism.** + +--- + +# It settles on-chain + +The loop closes end to end on Gnosis mainnet: + +- Relay accrues `owed`, client signs a **cumulative** cheque, relay verifies and credits +- Relay holds the latest cumulative — currently **21,366,720,000,000 PLUR** +- Cashed from a **separate machine**, because `cashChequeBeneficiary` must be sent *by* the beneficiary + +Observed cashout: **status 1, gasUsed 75,378** — well under the 300,000 budget. + +> The relay box never holds spendable key material. It needs the beneficiary's **address** only. The property that makes today's pusher safe survives metering intact. + +--- + +# What is actually deployed + +**Four `open` lanes** (free tiers, ephemeral disks — they *must* run open) plus **one hard-metered lane**, `pusher.browserbzz.link`, at 4.8e8 PLUR/KiB. + +The browser dApp lists all five and **skips the metered one automatically**, because it stamps but never settles. + +A native client with `--chequebook` uses all five. + +> Payment is a property of a relay, not of the fleet. + +--- + +# Net effect on the design + +**Removed** by changing the billing unit from receipts to bytes — five mechanisms, and every attack against them: staking-registry anchor, set-valued invoice, sampled retrieval audit, two-sided staked-set log sweep, shared predicate module. + +**Added** — three, all cheap: one MAC per challenge, one `eth_call` per batch per TTL, one atomic reservation per POST. + +**Properties that now hold by construction rather than by assumption:** + +- Sybil margin is exactly `credit_ratio` = 1000×, at any batch size +- Credit decays as the batch is spent down, with no expiry logic +- The relay holds no key that can move money + +**Found by running it:** six bugs no test suite reached; five needed a ledger outliving the client. + +--- + +# Still open + +- **Stage 0 data gates Stage 2 pricing.** `accounts_reaching_cashout` is the number that decides whether metering funds anything at all. +- **One account = one batch owner = one chequebook.** A client uploading under several owners needs a chequebook per owner. +- **The withdraw race is inherited and unfixed.** Bee has it too; it is inherent to SimpleSwap-as-deployed. +- **A malicious relay is out of scope.** If the federation opens to unvetted operators, that assumption breaks first. + +
+ +*Full design: `docs/pusher-incentives.md` — §8 for the billing unit, §10.3 for the Sybil bound, §17 for the bugs.* diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 7d35ea7..1c1f4ef 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -16,13 +16,12 @@ cheque format*, not its protocol. **This is a rewrite.** Three rounds of adversarial review went into the previous version, and the third round's most important finding was structural rather than local: the doc was building *two-sided* -cryptographic verification for a *one-sided* trust relationship. Relays -are a curated, pinned set (`PUSHER_URLS`, `apps/upload/src/config.ts:18-25` -— four URLs, all ours); clients are anonymous. Roughly half the old -document defended the client against the relay, which meant defending our -own infrastructure from itself, and it paid for that with a forgeable -billing unit, an unbounded residual, and a ship-blocking measurement -nobody had taken. +cryptographic verification for a *one-sided* trust relationship. A client +**chooses and pins** its relay before sending a byte; a relay gets whoever +POSTs. Roughly half the old document defended the client against the +relay — a party it had already vetted — and it paid for that with a +forgeable billing unit, an unbounded residual, and a ship-blocking +measurement nobody had taken. This version points every defence in one direction — **the relay is what gets protected, from the client** — and picks a billing unit the client @@ -63,19 +62,44 @@ profit. Every design decision below follows from one asymmetry. -> **Relays are known. Clients are anonymous.** +> **The client chose its relay. The relay did not choose its client.** > -> A relay is an entry in a hardcoded list, run by a named operator, pinned -> by URL over HTTPS. A client is whoever POSTs. Defences point *from* the -> relay *at* the client. The security goal is: **a client cannot obtain -> relay service without paying for it, and cannot lie to the relay about -> what it owes.** - -The inverse — a relay defrauding a client — is explicitly **out of scope -for this iteration**. Not because it is impossible, but because it is -governed socially rather than cryptographically: a lane that misbehaves is -removed from `PUSHER_URLS`. If the federation ever opens to unvetted -operators that becomes a real problem and §14 records what it would take. +> A client verifies the signed quote and pins +> `(url, node_eth_address, beneficiary)` (§7.3) before it sends a byte. A +> relay is whatever URL it was pointed at, and a client is whoever POSTs. +> Defences point *from* the relay *at* the client. The security goal is: +> **a client cannot obtain relay service without paying for it, and cannot +> lie to the relay about what it owes.** + +**The asymmetry is pinning, not curation, and the difference is not +cosmetic.** A relay is a plain HTTP service: there is no registry, no +discovery mechanism, and no list anyone has to be admitted to. Anyone can +run `hoverfly pusher`, and any client can point `--pusher` at any URL. +`PUSHER_URLS` is one client's default fleet — the dApp's — not a +federation roster. + +So "relays are known" is only true of a relay a given client has already +chosen. The property the design can actually lean on is that **a client +only ever pays a relay it configured and pinned**, which is a decision it +made with the signed quote in hand. + +The inverse — a relay defrauding a client — is still **out of scope for +cryptographic treatment**, but it does not follow that the client is +unprotected. Four bounds hold without the relay being trustworthy: + +- **The client computes its own bill** from bytes it sent (§8.4), so an + over-reported `kib_admitted` is immediately visible and attributable. +- **The price is pinned** from a signed quote and echoed verbatim in every + 402 (§7.3), so it cannot move under the client. +- **Exposure per lane is capped at the credit line** — `max_outstanding` + at most, and `remaining_value ÷ credit_ratio` for a small batch. +- **Outcomes are measured.** A lane that takes bytes and delivers poorly is + deweighted by the scheduler's existing EWMA. + +The one place a relay's own assertion could exceed that cap is §17.1's +reconcile, where the client adopts the relay's `owed`. It is bounded by +the ceiling in the relay's *signed quote* precisely so that this list stays +true; see `LanePayer::check_reported_debt`. Three consequences worth stating so they don't get re-litigated: @@ -516,8 +540,9 @@ recovered address and the pinned overlay are values in different spaces. The signed block therefore carries `node_eth_address` and `overlay_nonce`, so any client can recompute `overlay` and check it against what `/v1/status` advertises, and clients pin -**`(url, node_eth_address, beneficiary)`**. `PUSHER_URLS` is already a -hardcoded list, so extending each entry costs nothing. +**`(url, node_eth_address, beneficiary)`**. A client already names its +relays somewhere — `PUSHER_URLS` in the dApp, `--pusher` on the CLI — so +carrying two more fields alongside a URL it already had costs nothing. ```jsonc // GET /v1/status → new field (the whole object is covered by `sig`) @@ -660,8 +685,11 @@ many bytes it sent. There is nothing to dispute. A relay over-reporting `kib_admitted` is therefore immediately visible and attributable. The client's response is to withhold the next cheque and -deweight the lane, which is §2's social enforcement rather than a protocol -mechanism, and is adequate for a pinned lane set. +deweight the lane. That is not social enforcement — it needs no operator, +no list and no reputation — it is simply the client declining to sign for +a number it can check itself. What it costs the client to detect the +disagreement is bounded by the credit line, since nothing above that can +be admitted before the next settlement. ## 9. Pricing @@ -1415,10 +1443,12 @@ Carried knowingly: - **§11.5** — griefing by aiming at badly-covered arcs still forces the relay to spend more than the attacker pays, even though the attacker now pays. Bounded by attempt caps, not eliminated. -- **A relay can take payment and drop chunks.** Out of scope by §2 and - governed socially: lanes are pinned in `PUSHER_URLS` and a - misbehaving one is removed. This is adequate for a curated set and - **not** adequate for an open one. +- **A relay can take payment and drop chunks.** Out of scope for + cryptographic treatment by §2. What bounds it is that the client pinned + the lane, caps its exposure at one credit line, and deweights a lane + whose acks do not arrive — so the loss is at most `max_outstanding` and + is detected within one settlement window. Adequate for a lane a client + chose; **not** a proof of delivery, which pushsync cannot give us. Open: @@ -1613,15 +1643,26 @@ which number, each of which was wrong in a draft: still in flight are already held in `pending_plur` and would be billed twice when those POSTs land. Under-counting is safe and self-correcting; over-counting is a refused cheque. -- **Bounded by the chequebook balance, not the credit line.** Debt reaches - the line by construction, and an operator who lowers the line leaves - legitimately-incurred debt above it. Rejecting on that basis refuses to - pay a real bill and preserves the deadlock. The funded balance is what - actually bounds exposure, and `settle` already enforces it exactly - across all lanes (§8.3). - -This makes the client trust a curated relay's arithmetic about its own -receivable, which §2 already grants. The alternative — persisting the +- **Bounded by the quote's ceiling, not by the per-batch credit line — and + then by the chequebook balance.** The line is the wrong bound: it shrinks + as the batch is spent down, so it falls below debt properly incurred when + the batch was worth more, and rejecting on that basis refuses a real bill + and preserves the deadlock. `max_outstanding_plur` from the *signed* quote + is the right one: admission refuses above the per-batch cap, and every cap + is `min(value / ratio, ceiling)`, so a figure above the ceiling describes + debt that cannot have been incurred. The balance check stays as a second, + looser gate — `settle` enforces it exactly across all lanes (§8.3), and + stating it here turns a confusing failure at signing time into a legible + one at reconcile time. + +This makes the client trust the relay's arithmetic about its own +receivable, but only within the credit that relay granted — which is what +pinning buys (§2). The first version bounded on the chequebook balance +alone, which would let any lane a client pointed at name the whole balance +and be signed for it. That is not something §2 grants: a relay is pinned, +not vetted, and there is no list to be thrown off. + +The alternative — persisting the residual client-side — keeps better books but has no recovery when that state is lost, and a client permanently locked out of a relay with no way to clear it is the worse failure. diff --git a/src/payer.rs b/src/payer.rs index 640e5c7..ca99087 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -805,22 +805,55 @@ impl LanePayer { cfg: &PaymentConfig, ) -> Result { let owed = self.relay_owed(http, cfg).await?; - // Bounded by the chequebook's balance, deliberately *not* by the - // credit line. The line caps what may be newly admitted, not what - // may stand: debt reaches the line by construction, and an operator - // who lowers the line leaves legitimately-incurred debt above it. - // Rejecting on that basis refuses to pay a real bill and keeps the - // deadlock this method exists to break. What genuinely bounds our - // exposure to an inflated figure is the funded balance, which - // `settle` enforces exactly across all lanes (§8.3); this is the - // same ceiling stated early for a legible error. - if owed > cfg.balance_plur { + self.check_reported_debt(owed, cfg.balance_plur)?; + Ok(self.account.adopt_relay_debt(owed)) + } + + /// Is a debt figure the relay reports one it could legitimately hold, + /// and one we could pay? + /// + /// Split out of [`Self::reconcile`] because it is the only part of the + /// reconcile that *decides* anything, and it should be testable without + /// standing up a server. + /// + /// Two ceilings, and the tighter one binds. + /// + /// The first is the quote's global `max_outstanding_plur`. The relay + /// refuses admission whenever `outstanding + reserve > cap`, and §10.3 + /// makes every per-batch cap `min(value / ratio, ceiling)` — never above + /// the ceiling. So no debt can *legitimately* stand above it, whatever + /// our own books say. + /// + /// Deliberately not the per-batch credit line, which §17.1 rejected for + /// a reason that still holds: the line shrinks as the batch is spent + /// down, so it can fall below debt properly incurred when the batch was + /// worth more, and refusing on that basis preserves the very deadlock + /// this reconcile exists to break. The ceiling is a constant of the + /// signed quote and does not move. + /// + /// Why bound at all, when the relay is the one keeping the ledger: a + /// relay is not a curated identity. It is an HTTP service the client + /// chose and pinned (§7.3) — anyone can run one, and there is no + /// registry to be admitted to. Pinning buys the right to trust its + /// arithmetic *within the credit it granted*. Trusting it past that puts + /// the entire chequebook behind one JSON field. + fn check_reported_debt(&self, owed: u128, balance_plur: u128) -> Result<(), String> { + let ceiling = self.quote.params.max_outstanding_plur; + if owed > ceiling { return Err(format!( - "relay claims {owed} owed, more than the chequebook's {} balance", - cfg.balance_plur + "relay claims {owed} owed, above the {ceiling} ceiling it signed \ + — it cannot have admitted that much" )); } - Ok(self.account.adopt_relay_debt(owed)) + // Then what we can actually pay. `settle` enforces this exactly + // across all lanes (§8.3); stating it here turns a confusing failure + // at signing time into a legible one at reconcile time. + if owed > balance_plur { + return Err(format!( + "relay claims {owed} owed, more than the chequebook's {balance_plur} balance" + )); + } + Ok(()) } /// Settle if there is enough owed to be worth a cheque. @@ -1290,6 +1323,60 @@ mod tests { /// `min_cheque_plur`, so it cannot be settled (§10.2) and the headroom /// the guard waited for could never come back — 60 of 76 chunks were /// left unacked against a live relay. + /// A relay's reported debt is trusted only up to the ceiling it signed. + /// + /// Reconcile adopts the relay's own figure (§17.1), which is sound + /// *within the credit it granted*: admission refuses above the cap, and + /// every per-batch cap is `min(value / ratio, ceiling)`. Above the + /// ceiling there is no story in which the debt was legitimately + /// incurred, so the figure is refused rather than signed for. + /// + /// This matters because a relay is not a curated identity — it is an + /// HTTP service the client pinned. Bounding only by the chequebook + /// balance, as an earlier version did, let any lane a client pointed at + /// name the whole balance and be paid it. + #[test] + fn a_relay_cannot_claim_more_debt_than_the_ceiling_it_signed() { + let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let cap = q.params.max_outstanding_plur; + let payer = LanePayer::new("http://lane".into(), q, 0); + // A chequebook far richer than the credit line, which is the normal + // case and exactly what made the old bound useless. + let balance = cap * 1000; + + payer + .check_reported_debt(cap, balance) + .expect("debt exactly at the ceiling is legitimate"); + payer + .check_reported_debt(cap - 1, balance) + .expect("and anything under it"); + + let e = payer + .check_reported_debt(cap + 1, balance) + .expect_err("one PLUR above the ceiling cannot have been admitted"); + assert!(e.contains("ceiling"), "got {e}"); + + let e = payer + .check_reported_debt(balance, balance) + .expect_err("and the whole chequebook certainly cannot"); + assert!(e.contains("ceiling"), "got {e}"); + } + + /// The balance still bounds, for a chequebook too thin to cover a + /// legitimate debt — caught here rather than as a confusing failure at + /// signing time. + #[test] + fn debt_within_the_ceiling_but_over_the_balance_is_refused() { + let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let cap = q.params.max_outstanding_plur; + let payer = LanePayer::new("http://lane".into(), q, 0); + + let e = payer + .check_reported_debt(cap, cap / 2) + .expect_err("we cannot sign for more than the chequebook holds"); + assert!(e.contains("balance"), "got {e}"); + } + #[test] fn a_line_barely_wider_than_one_post_still_makes_progress() { let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); From 6d7198c7524eaa08e499e3ff18e1c69794eae9bd Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 16:51:40 +0300 Subject: [PATCH 16/27] docs(slides): rewrite the bullets in plain language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bullets read like excerpts from the design doc, because that is what they were. Slides get glanced at while someone is talking, so a phrase like "gas-amortizing by construction" or "the invoice was set-valued" costs the reader the sentence the speaker is currently saying. Every bullet now leads with what happens, in ordinary words, and names the mechanism second. All the numbers stay — this is a legibility pass, not a vagueness one. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 76 ++++++++++++++++---------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index a5ff91d..3fb9e50 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -67,10 +67,10 @@ The client's protections are different in kind — arithmetic and exposure limit Not cryptography. Four bounds, none of which need the relay to be trustworthy: -- **It bills itself.** `owed` is computed from bytes the client sent, so a relay over-reporting `kib_admitted` is immediately visible and attributable (§8.4). -- **It pinned the price.** `price_plur_per_kib` comes from a quote signed by the node identity, echoed verbatim in every 402. -- **Exposure is capped at the credit line** — ~$0.0024 at the global ceiling, and `remaining_value ÷ 1000` for a small batch. -- **It measures outcomes.** A lane that takes bytes and delivers poorly is deweighted by the scheduler's existing EWMA. +- **It works out the bill itself.** The client adds up the bytes it sent. If the relay reports a bigger number, the client sees it immediately — and knows the relay is the one that is wrong (§8.4). +- **The price is fixed in advance.** It arrives in a quote the relay signs, and the same signed quote comes back with every refusal — so the price cannot move mid-upload. +- **The most it can lose is one credit limit** — about $0.0024 at the maximum, and far less on a small batch, which gets a thousandth of whatever it is still worth. +- **It watches whether chunks actually arrive.** A relay that takes bytes and delivers badly gets sent less work, using the running average that already picks between relays today. > The curated-set premise was hiding a gap in the fourth: on an unpayable 402 the client **adopts the relay's own `owed`**, and that used to be bounded only by the chequebook balance. It is now bounded by `max_outstanding_plur` from the quote the relay signed — the credit it granted is the most it can claim. @@ -84,9 +84,9 @@ Three rounds of adversarial review. The most important finding was structural, n Roughly half of it defended the client against the relay — against lanes we run ourselves. Three concrete costs: -- **Billing unit was a third party's signature** (a pushsync receipt), so it was forgeable and needed a staking-registry anchor -- **Unbounded residual** — the invoice was set-valued, with no bound on retained receipts -- **A kill criterion nobody had measured**: the staked fraction of receipt signers, which could have vetoed the whole thing +- **The bill was built on someone else's signature** — a receipt from a bee node. Those are easy to fake, so every one had to be checked against the on-chain list of staked nodes. +- **The bill was a list, not a number**, and nothing limited how long that list could get. +- **One unmeasured number could have killed the project**: how many receipt signers are actually staked. Nobody had checked. --- @@ -141,10 +141,10 @@ That is the whole billing rule. One property matters more than everything else i > **The client cannot lie about it — the client produced the bytes and the relay counted them.** -- No third-party attestation to forge -- No signature to replay -- No chain state to disagree about -- No relay assertion taken on faith +- Nothing to forge — no outside signature is part of the bill +- Nothing to replay — the relay counts arrivals, not tokens +- Nothing to look up on chain +- Nothing the client has to take on the relay's word Both numbers are known to both parties *before* any push work happens. @@ -154,11 +154,11 @@ Both numbers are known to both parties *before* any push work happens. The earlier draft billed per **verified pushsync receipt**. To make a *third party's* signature into a billing input, it needed: -- a staking-registry anchor (to stop trivial forgery) -- a set-valued invoice (to stop self-replay) -- a sampled retrieval audit (to catch fabrication) -- a staked-set log sweep on both sides -- a shared predicate module, byte-identical on both sides, **with no adjudicator if it drifted** +- check every receipt signer against the on-chain staking list, or faking one is trivial +- make the bill a list instead of a number, or the same receipt gets submitted twice +- spot-check by fetching chunks back, to catch receipts for work never done +- sweep the staking logs on both sides, and keep them in step +- share one piece of code that decides whether a receipt counts — and **if the two copies ever disagree, nobody can settle it** Changing the unit **deleted the entire apparatus and every attack against it.** @@ -201,9 +201,9 @@ A relay has no incentive to make it falsely, and a client that disagrees is disa A cheque is a running total for one `(chequebook, beneficiary)` pair. Three properties fall out: -- **Loss-tolerant** — a dropped `/v1/pay` costs nothing; the next cheque supersedes it. No retry state machine. -- **Replay-proof** — strict monotonicity means a re-presented cheque credits zero. -- **Gas-amortizing by construction** — the relay cashes only the *latest* cumulative, so gas is paid once per **account**, not per cheque. +- **Losing one costs nothing.** Every cheque is a running total, so if a payment fails the next one covers it anyway. No retry logic needed. +- **Old cheques are worthless.** Each must be larger than the last, so sending an old one again pays nothing. +- **Gas is paid once per customer, not once per cheque.** The relay only ever cashes the newest total, so every earlier cheque costs nothing to collect. **Per-chunk cheques are rejected:** ~137 B per 4 KiB chunk, an EIP-712 signature on the hot path, and cumulative payouts are *serial* per `(issuer, beneficiary)` pair — which forces a total order on chunks within a lane, removing the concurrent multi-POST pipelining the scheduler depends on. @@ -430,9 +430,9 @@ None is reachable from a single upload against a fresh relay. All six survived t Reaching them needed, simultaneously: -- a relay whose ledger **persists across client runs** -- enough concurrency to have several POSTs on the wire at once -- a batch **spent down far enough that its credit line binds** +- a relay that **remembers what you owe between runs**, so debt carries over +- several uploads in flight at the same time, not one after another +- a batch **used up far enough that the credit limit is what stops you**, rather than anything else > §17.3 is the one to generalise from: it is not a coding error but an **invariant checked against the wrong quantity**, and it only becomes reachable once a real batch's value has decayed below ~0.39 BZZ. @@ -469,11 +469,11 @@ Fix: **ask rather than remember** — reconcile against `GET /v1/account`. Each of these was wrong in a draft: -- **Use `owed`, not the 402 body's `outstanding_plur`.** `reserve()` adds the reservation *before* computing what it reports, so the body includes the request just refused. Adopting it over-pays by exactly that body — and the next cheque bounces as an **overpayment**. +- **Ask what you owe. Don't read it off the refusal.** The refusal already counts the request it is turning down, so paying that number overpays by exactly one request — and the next cheque bounces for being too big. -- **Exclude reservations on our side too.** Bytes still in flight are already held in `pending`, and would be billed twice when those POSTs land. *Under*-counting is safe and self-correcting; *over*-counting is a refused cheque. +- **Don't count bytes still on the wire.** The client is already tracking those, so counting them again bills them twice. Guessing low is safe and fixes itself on the next round; guessing high gets the cheque rejected. -- **Bound by the quote's signed ceiling** — not the per-batch line, and not the balance. The line shrinks as the batch decays, so it falls below debt properly incurred earlier, and rejecting on that basis **preserves the deadlock**. The balance is far too loose: it lets any lane a client points at name the whole chequebook. Admission refuses above the cap and every cap is `min(value ÷ ratio, ceiling)`, so above the ceiling the debt cannot exist. +- **Cap it at the limit the relay signed up to.** Not the batch's current limit — that falls as the batch is used up, so it can drop below a bill you honestly ran up, and refusing to pay that **keeps you stuck**. And not the chequebook balance either: that lets any relay you point at ask for everything you have. --- @@ -530,11 +530,11 @@ Each run settles to **`owed: 0`** on the relay, so the next carries nothing. The loop closes end to end on Gnosis mainnet: -- Relay accrues `owed`, client signs a **cumulative** cheque, relay verifies and credits -- Relay holds the latest cumulative — currently **21,366,720,000,000 PLUR** -- Cashed from a **separate machine**, because `cashChequeBeneficiary` must be sent *by* the beneficiary +- The relay counts what you owe. You sign a cheque for the running total. The relay checks it and marks you paid. +- The relay is holding a cheque for **21,366,720,000,000 PLUR** right now. +- Cashing it happens on a **different machine**. Only the payee can cash a cheque, and the relay must never hold that key. -Observed cashout: **status 1, gasUsed 75,378** — well under the 300,000 budget. +It worked: the transaction succeeded and used **75,378 gas**, against a 300,000 budget. > The relay box never holds spendable key material. It needs the beneficiary's **address** only. The property that makes today's pusher safe survives metering intact. @@ -554,14 +554,14 @@ A native client with `--chequebook` uses all five. # Net effect on the design -**Removed** by changing the billing unit from receipts to bytes — five mechanisms, and every attack against them: staking-registry anchor, set-valued invoice, sampled retrieval audit, two-sided staked-set log sweep, shared predicate module. +**Removed:** five mechanisms and every attack on them, just by billing bytes instead of receipts — the staking check, the list-shaped bill, the spot-check audit, the log sweep on both sides, and the shared receipt-checking code. -**Added** — three, all cheap: one MAC per challenge, one `eth_call` per batch per TTL, one atomic reservation per POST. +**Added:** three, all cheap — one signature check when a client asks permission, one chain lookup per batch every half hour, and one amount set aside per upload. -**Properties that now hold by construction rather than by assumption:** +**Now guaranteed rather than hoped for:** -- Sybil margin is exactly `credit_ratio` = 1000×, at any batch size -- Credit decays as the batch is spent down, with no expiry logic +- An attacker gets credit worth a thousandth of what they actually funded, whatever size batch they buy +- Credit shrinks by itself as a batch is used up, with no expiry code to get wrong - The relay holds no key that can move money **Found by running it:** six bugs no test suite reached; five needed a ledger outliving the client. @@ -570,10 +570,10 @@ A native client with `--chequebook` uses all five. # Still open -- **Stage 0 data gates Stage 2 pricing.** `accounts_reaching_cashout` is the number that decides whether metering funds anything at all. -- **One account = one batch owner = one chequebook.** A client uploading under several owners needs a chequebook per owner. -- **The withdraw race is inherited and unfixed.** Bee has it too; it is inherent to SimpleSwap-as-deployed. -- **A malicious relay is out of scope.** If the federation opens to unvetted operators, that assumption breaks first. +- **We don't yet know if this pays for itself.** The number to watch is how many accounts ever build up enough debt to be worth cashing. If it stays at zero, metering funds nothing. +- **One chequebook per batch owner.** If you upload using batches owned by different addresses, each one needs its own chequebook. +- **The empty-the-chequebook problem has no fix.** Bee has it too — it comes from how the chequebook contract is deployed, not from anything here. +- **Nothing stops a relay taking your money and dropping chunks.** You lose at most one credit limit and then stop using it — that bounds the damage, but it is not a guarantee of service.
From 19acaccb12edf9b404f655a1601ed54759220e9a Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 16:54:54 +0300 Subject: [PATCH 17/27] docs(slides): plainer title and headings Subtitle is now what the deck is about rather than a tagline. Two headings along with it: the problem slide says the relay pays for traffic it did not cause, and "What changes, precisely" drops the self-congratulation. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 3fb9e50..2d3ec09 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -2,14 +2,12 @@ marp: true theme: default paginate: true -header: 'Paying for relay — SWAP cheques for hoverfly pushers' +header: 'Paying for relay — an incentive layer for hoverfly pushers' --- # Paying for relay -## SWAP cheques for hoverfly pushers - -Theory, and what happened when we ran it +## An incentive layer for hoverfly pushers, reusing parts of SWAP
@@ -17,7 +15,7 @@ Theory, and what happened when we ran it --- -# The problem: the relay eats a cost it did not incur +# The problem: the relay pays for traffic it did not cause In a **native** upload, your own machine opens the pushsync streams. Bee debits *you*: @@ -33,7 +31,7 @@ The browser client that caused the traffic pays **nothing but postage**. --- -# What changes, precisely +# What changes | | open (today) | metered | |---|---|---| From 964148f18d6d1011971514cc7992b338b358f180 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 17:05:52 +0300 Subject: [PATCH 18/27] docs(slides): plain-language sweep over headings, callouts and prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bullets got fixed last pass; the headings and body text had the same problem. Slogan headings became statements ("Derive the credit line, don't assert it" -> "Credit scales with what the batch is worth"), and the prose lost the shorthand that only makes sense if you already know the design: "session- and RTT-bound", "per-identity volume oracle", "the precise unauthenticated amplification surface", "monotone debit counter". Every figure stays. The point was that the compressed phrasing was hiding the mechanism, not carrying it. Adds docs/deck2md.py and makes the markdown a generated file. Keeping the two decks in step by hand meant applying every copy edit twice, which is how they drift — the HTML is now the source, and the markdown is derived from it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/deck2md.py | 109 +++++++++ docs/pusher-incentives-slides.md | 404 ++++++++++++------------------- 2 files changed, 261 insertions(+), 252 deletions(-) create mode 100644 docs/deck2md.py diff --git a/docs/deck2md.py b/docs/deck2md.py new file mode 100644 index 0000000..28eb7a8 --- /dev/null +++ b/docs/deck2md.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Regenerate the Marp markdown deck from the HTML deck. + +The two carry the same content, and keeping them in step by hand meant every +copy edit had to be applied twice — which is exactly how they drift. The HTML +is the source; this derives the markdown from it. +""" +import html +import re +import sys + +SRC, DST = sys.argv[1], sys.argv[2] +doc = open(SRC).read() + +FRONT = """--- +marp: true +theme: default +paginate: true +header: 'Paying for relay — an incentive layer for hoverfly pushers' +--- +""" + + +def inline(s: str) -> str: + """HTML inline markup -> markdown.""" + s = re.sub(r"(.*?)", lambda m: "`" + re.sub(r"<[^>]+>", "", m.group(1)) + "`", s, flags=re.S) + s = re.sub(r"(.*?)", r"**\1**", s, flags=re.S) + s = re.sub(r"(.*?)", r"*\1*", s, flags=re.S) + s = re.sub(r"", "\n", s) + s = re.sub(r"<[^>]+>", "", s) # spans (ok/bad/q/hz) carry colour only + s = html.unescape(s) + return re.sub(r"[ \t]*\n[ \t]*", " ", s).strip() + + +def cells(row: str, tag: str) -> list: + # A literal pipe inside a cell would open a new column. + return [inline(c).replace("|", r"\|") + for c in re.findall(rf"<{tag}[^>]*>(.*?)", row, re.S)] + + +def table(block: str) -> str: + rows = re.findall(r"(.*?)", block, re.S) + if not rows: + return "" + head = cells(rows[0], "th") or cells(rows[0], "td") + body = [cells(r, "td") for r in (rows[1:] if cells(rows[0], "th") else rows)] + # Right-align any column the HTML marked with class="n". + aligns = ["---:" if 'class="n"' in c else "---" + for c in re.findall(r"]*)>", rows[0], re.S)] + out = ["| " + " | ".join(head) + " |", + "|" + "|".join(aligns[:len(head)] or ["---"] * len(head)) + "|"] + for b in body: + out.append("| " + " | ".join(b) + " |") + return "\n".join(out) + + +slides = [] +for sec in re.findall(r'
(.*?)
', doc, re.S): + kind, body = sec + out = [] + + if "title" in kind: + h1 = re.search(r"

(.*?)

", body, re.S) + lede = re.search(r'

(.*?)

', body, re.S) + meta = re.search(r'
(.*?)
', body, re.S) + out.append("# " + inline(h1.group(1))) + if lede: + out.append("## " + inline(lede.group(1)).rstrip(".")) + if meta: + out.append("*" + inline(meta.group(1)).replace(" ", " · ") + "*") + slides.append("\n\n".join(out)) + continue + + if "part" in kind: + kicker = re.search(r'
(.*?)
', body, re.S) + h2 = re.search(r"

(.*?)

", body, re.S) + label = inline(kicker.group(1)) if kicker else "" + slides.append(f"# {label.title()} — {inline(h2.group(1))}") + continue + + h2 = re.search(r"

(.*?)

", body, re.S) + if h2: + out.append("# " + inline(h2.group(1))) + + # Walk the slide's blocks in document order. + pattern = (r'
(.*?)
\s*(?=<)' + r'|
\s*(.*?
)\s*
' + r'|
(.*?)
' + r'|
(.*?)
') + for m in re.finditer(pattern, body, re.S): + bodydiv, tbl, pre, claim = m.groups() + if bodydiv is not None: + for el in re.finditer(r"]*>(.*?)

|
    (.*?)
", bodydiv, re.S): + para, ul = el.groups() + if para is not None: + out.append(inline(para)) + else: + out.append("\n".join( + "- " + inline(li) for li in re.findall(r"
  • (.*?)
    • ", ul, re.S))) + elif tbl is not None: + out.append(table(tbl)) + elif pre is not None: + out.append("```\n" + html.unescape(re.sub(r"<[^>]+>", "", pre)).strip() + "\n```") + elif claim is not None: + out.append("> " + inline(claim)) + slides.append("\n\n".join(x for x in out if x.strip())) + +open(DST, "w").write(FRONT + "\n" + "\n\n---\n\n".join(slides) + "\n") +print(f"{len(slides)} slides -> {DST}") diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 2d3ec09..1ce1f3f 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -9,55 +9,49 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' ## An incentive layer for hoverfly pushers, reusing parts of SWAP -
    - -*Companion to `docs/pusher-incentives.md`. Stages 0–1 shipped; one metered lane in production.* +*docs/pusher-incentives.md · Stages 0–1 shipped · one metered lane in production* --- -# The problem: the relay pays for traffic it did not cause +# The relay pays for traffic it did not cause -In a **native** upload, your own machine opens the pushsync streams. Bee debits *you*: +In a **native** upload your own machine opens the pushsync streams, and bee debits *you*: ``` -price(po) = (32 − po) × 10 000 accounting units +price(po) = (32 − po) × 10 000 accounting units ``` -Put a relay in the middle and that debt moves **wholesale to the relay** — it is the peer bee sees, so it is the peer bee charges. - -The browser client that caused the traffic pays **nothing but postage**. +Put a relay in the middle and that debt moves **wholesale to the relay** — it is the peer bee sees, so it is the peer bee charges. The browser client that caused the traffic pays **nothing but postage**. -> Today this is booked as accepted risk: *"worst case = the platform's free egress for the month burned, $0 lost."* +> Booked today as accepted risk: *“worst case = the platform's free egress for the month burned, $0 lost.”* --- # What changes -| | open (today) | metered | +| | open (today) | metered | |---|---|---| -| client → relay | nothing | `4.8e8` PLUR per KiB of body sent | -| relay → bee | free pseudosettle | **unchanged** — free pseudosettle | +| client → relay | nothing | 4.8e8 PLUR per KiB of body sent | +| relay → bee | free pseudosettle | unchanged — free pseudosettle | | relay's egress | unrecovered | recovered above the cashout threshold | -Relay→bee settlement stays free because the relay is **session- and RTT-bound, not credit-bound**: bee grants 4.5e6 accounting units/s per peer, ≈ 2 400 chunks/s across a 128-session pool, against **~150 chunks/s** actually measured. Buying credit buys nothing. +The relay still pays bee nothing, because credit was never what limited it. Bee hands out enough free credit for **~2,400 chunks a second** across a 128-connection pool, and the relay only manages **~150**. It is limited by connections and round trips, so buying credit would buy nothing. -> The recovered amount is small. A relay pushing 100 GB of egress a month is moving ~27 GiB of payload, which at $0.02/GiB is **$0.54**. +> The recovered amount is small. A relay pushing 100 GB of egress a month moves ~27 GiB of payload, which at $0.02/GiB is **$0.54**. --- -# Part I — Theory +# Part One — Theory --- -# Everything follows from one asymmetry +# Who has to trust whom -> **The client chose its relay. The relay did not choose its client.** +> The client chose its relay. The relay did not choose its client. -A relay is a plain HTTP service. Anyone can run one; there is no registry, no discovery, no allowlist to be admitted to. What creates the asymmetry is **pinning, not curation**: before it sends a byte, a client verifies the signed quote and pins `(url, node_eth_address, beneficiary)`. The relay has no equivalent — a client is whoever POSTs. +A relay is just an HTTP service. Anyone can run one, and there is no registry or list to get onto. The asymmetry comes from **the client picking**: before sending anything it checks the relay's signed quote and remembers who that relay is. The relay gets no such choice — a client is whoever shows up. -**So every defence the relay has points at the client**, and the relay's goal is: *a client cannot obtain service without paying, and cannot lie about what it owes.* - -The client's protections are different in kind — arithmetic and exposure limits, not authentication. Next slide. +So every defence *the relay* has points at the client: *a client cannot obtain service without paying, and cannot lie about what it owes.* --- @@ -65,22 +59,20 @@ The client's protections are different in kind — arithmetic and exposure limit Not cryptography. Four bounds, none of which need the relay to be trustworthy: -- **It works out the bill itself.** The client adds up the bytes it sent. If the relay reports a bigger number, the client sees it immediately — and knows the relay is the one that is wrong (§8.4). +- **It works out the bill itself.** The client adds up the bytes it sent. If the relay reports a bigger number, the client sees it immediately — and knows the relay is the one that is wrong. - **The price is fixed in advance.** It arrives in a quote the relay signs, and the same signed quote comes back with every refusal — so the price cannot move mid-upload. - **The most it can lose is one credit limit** — about $0.0024 at the maximum, and far less on a small batch, which gets a thousandth of whatever it is still worth. - **It watches whether chunks actually arrive.** A relay that takes bytes and delivers badly gets sent less work, using the running average that already picks between relays today. -> The curated-set premise was hiding a gap in the fourth: on an unpayable 402 the client **adopts the relay's own `owed`**, and that used to be bounded only by the chequebook balance. It is now bounded by `max_outstanding_plur` from the quote the relay signed — the credit it granted is the most it can claim. +> That last bound had a hole in it. When a client cannot pay a refusal, it asks the relay what it owes and believes the answer. That used to be capped only by the chequebook balance, so any relay could ask for everything. It is now capped by the limit in the quote the relay signed: the credit it granted is the most it can claim. --- -# What the asymmetry corrected - -Three rounds of adversarial review. The most important finding was structural, not local: +# What the previous design got wrong > The design was building **two-sided cryptographic verification** for a **one-sided trust relationship.** -Roughly half of it defended the client against the relay — against lanes we run ourselves. Three concrete costs: +About half of it protected the client from the relay — from relays we run ourselves. That cost three real things: - **The bill was built on someone else's signature** — a receipt from a bee node. Those are easy to fake, so every one had to be checked against the on-chain list of staked nodes. - **The bill was a list, not a number**, and nothing limited how long that list could get. @@ -92,14 +84,12 @@ Roughly half of it defended the client against the relay — against lanes we ru | Borrowed | Why | |---|---| -| `ERC20SimpleSwap` + canonical factory | Audited, deployed, in production. Nothing to write. | -| EIP-712 cheque | `Cheque(chequebook, beneficiary, cumulativePayout)` | +| ERC20SimpleSwap + canonical factory | Audited, deployed, in production. Nothing to write. | +| EIP-712 cheque | Cheque(chequebook, beneficiary, cumulativePayout) | | Cumulative-payout monotonicity | Loss-tolerant *and* replay-proof | -| Funding check | …but `liquidBalanceFor(us)`, **not** `balance()` — bee's version is unsound | -| Reservation against concurrent issuance | Needed on **both** sides | -| Payee-only role | The beneficiary is a plain **EOA**. A payee needs no contract. | - -Gnosis factory: `0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420` +| Funding check | …but liquidBalanceFor(us), not balance() — bee's version is unsound | +| Reservation vs. concurrent issuance | Needed on *both* sides | +| Payee-only role | The beneficiary is a plain EOA. A payee needs no contract. | --- @@ -107,42 +97,36 @@ Gnosis factory: `0xc2d5a532cf69aa9a1378737d8ccdef884b6e7420` | Dropped | Reason | |---|---| -| swap libp2p stream, `Handshake`, `EmitCheque` | We're on HTTP already | -| priceoracle, `exchange`, `deduction` | Relay quotes PLUR directly | +| swap libp2p stream, Handshake, EmitCheque | We're on HTTP already | +| priceoracle, exchange, deduction | Relay quotes PLUR directly | | accounting-unit indirection | One unit. No conversion. | | ghost balances, tolerance, trust ramp | No analogue in request/response | -| `StakeRegistry` snapshot | Nothing left to anchor — see next section | +| StakeRegistry snapshot | Nothing left to anchor — see next | -> **Both counterparties are hoverfly.** Bee is not a party to the payment. We borrow SWAP's *contracts and cheque format*, not its protocol. +> Both sides here are hoverfly. Bee is not involved in the payment at all — we reuse SWAP's *contracts and cheque format*, not the protocol it speaks over the network. --- -# Identity: the account is the batch owner - -The relay's existing auth already establishes on-chain identity for every push: the stamp signature must recover to the address `PostageStamp` reports as the batch's owner. +# The account is the batch owner -> **Account = the batch-owner EOA.** -> A cheque is valid for that account **iff** its chequebook's on-chain `issuer()` equals the same EOA. -> **Credit is keyed one level finer — on the *batch*.** +Relays already know who is uploading. Every chunk carries a stamp, and its signature has to match the address the postage contract lists as the batch's owner. -No session tokens. No registration. No extra protocol message. +> Account = the batch-owner EOA. A cheque is valid for it **iff** its chequebook's on-chain `issuer()` is the same EOA. Credit is keyed one level finer — on the **batch**. -In a browser: the session key already owns the batch, so cheques sign with **zero wallet prompts**. +So there is nothing to add: no logins, no sign-up, no extra message. In a browser the key that owns the batch is already loaded, so cheques get signed with **no wallet pop-ups at all**. --- -# The central decision: bill bytes admitted - -> **`owed = (kib_admitted − kib_dedup) × price_plur_per_kib`** +# Bill bytes admitted -That is the whole billing rule. One property matters more than everything else in the design: +``` +owed = (kib_admitted − kib_dedup) × price_plur_per_kib +``` -> **The client cannot lie about it — the client produced the bytes and the relay counted them.** +> The client cannot lie about it — the client produced the bytes and the relay counted them. - Nothing to forge — no outside signature is part of the bill -- Nothing to replay — the relay counts arrivals, not tokens -- Nothing to look up on chain -- Nothing the client has to take on the relay's word +- Nothing to argue about — no chain lookup, and nothing the client takes on the relay's word Both numbers are known to both parties *before* any push work happens. @@ -150,70 +134,49 @@ Both numbers are known to both parties *before* any push work happens. # What that replaced -The earlier draft billed per **verified pushsync receipt**. To make a *third party's* signature into a billing input, it needed: +The earlier draft billed per **verified pushsync receipt**. To make a *third party's* signature into a billing input it needed: - check every receipt signer against the on-chain staking list, or faking one is trivial - make the bill a list instead of a number, or the same receipt gets submitted twice - spot-check by fetching chunks back, to catch receipts for work never done -- sweep the staking logs on both sides, and keep them in step -- share one piece of code that decides whether a receipt counts — and **if the two copies ever disagree, nobody can settle it** +- share one piece of code that decides whether a receipt counts — and if the two copies ever disagree, nobody can settle it -Changing the unit **deleted the entire apparatus and every attack against it.** - -> Receipts are still forwarded — as *telemetry*, feeding lane weighting. They do not enter the invoice. +> Billing bytes instead of receipts removed all five, and every attack on them. --- # Bytes, not successful pushes -Bytes are what the relay **spends money on**. - -Egress is incurred on *attempts*: the 3-way peer race and the shallow retries happen whether or not a chunk lands. - -Billing successes would mean the relay eats the cost of every failure. - -Instead — two mechanisms, each doing what it is good at: +Bytes are what actually costs the relay money. It sends every chunk to three peers at once and retries the ones that go nowhere, and that traffic goes out whether or not the chunk ends up stored. Charging only for successes would leave the relay paying for every failure. | Concern | Mechanism | |---|---| | Relay recovers what it spends | Bill attempts | -| Client protects itself from a lane that spends without succeeding | **Deweight it in the scheduler** (already exists, already works) | +| Client protects itself from a lane that spends without succeeding | Deweight it in the scheduler — already exists, already works | --- # Dedup hits are billed at zero -A frame served from the recent-ack cache does no push work, so its bytes are subtracted. - -This is the **one** place a relay assertion enters the bill — the relay claims *"this was a dedup hit"*. +If the relay already pushed the same chunk moments ago, it does no work the second time, so those bytes come off the bill. This is the **only** part of the bill that rests on the relay's own word. -It is safe for a structural reason: - -> The claim only ever **lowers** the amount owed. - -A relay has no incentive to make it falsely, and a client that disagrees is disagreeing in its own favour. +> It is safe for a simple reason: the claim only ever **lowers** the bill. A relay gains nothing by lying, and a client that disputes it is arguing in its own favour. --- # Cumulative cheques -A cheque is a running total for one `(chequebook, beneficiary)` pair. Three properties fall out: - - **Losing one costs nothing.** Every cheque is a running total, so if a payment fails the next one covers it anyway. No retry logic needed. - **Old cheques are worthless.** Each must be larger than the last, so sending an old one again pays nothing. - **Gas is paid once per customer, not once per cheque.** The relay only ever cashes the newest total, so every earlier cheque costs nothing to collect. -**Per-chunk cheques are rejected:** ~137 B per 4 KiB chunk, an EIP-712 signature on the hot path, and cumulative payouts are *serial* per `(issuer, beneficiary)` pair — which forces a total order on chunks within a lane, removing the concurrent multi-POST pipelining the scheduler depends on. +> A cheque per chunk was rejected: 137 bytes and a signature on every 4 KiB, and — worse — running totals have to go in order, so chunks would have to be sent one at a time. That would remove the parallel uploads the scheduler relies on for speed. --- -# One subtlety that bricks a real deployment - -A cumulative is per `(chequebook, beneficiary)`. A **lane** is a URL. +# Cheques are per payee, not per relay -One operator running four lane URLs behind one beneficiary EOA is the obvious deployment. - -If the client tracks cumulatives **per lane**, that configuration bricks: +A running total belongs to a **payee**, but a relay is a **URL**. One operator running four relay URLs that all pay into the same account is the obvious way to deploy. ``` lane 1 issues cumulative 10 @@ -221,41 +184,35 @@ lane 2, counting from its own zero, issues 8 relay applies ErrChequeNotIncreasing → rejected, forever ``` -> **Key the client's cumulative store on `(chequebook, beneficiary)`** — not on lane, not on overlay. - -Detecting the sharing is free: the beneficiary is in the signed quote. +> So the client must track running totals per *payee*, not per relay URL. Spotting that two relays share one is free — the payee's address is in the signed quote, before the first upload. --- -# Wire protocol - -Push frames are **unchanged**. Payment is out-of-band — it must not sit on the hot path, and a payment failure must not fail a push. +# Payment happens outside the upload | Endpoint | Shape | |---|---| -| `GET /v1/status` | signed `payment` block; `mode: open \| metered` | -| `GET /v1/challenge` | `{nonce, expires_ms, max_outstanding_plur}` — stateless MAC | -| `GET /v1/account` | authenticated. `owed`, `reserved`, `outstanding`, `kib_admitted`, … | -| `POST /v1/pay` | body = `SignedCheque` JSON | -| `POST /v1/push` | `402 Payment Required` when over cap | +| GET /v1/status | signed *payment* block; mode: open \| metered | +| GET /v1/challenge | {nonce, expires_ms, max_outstanding_plur} — stateless MAC | +| GET /v1/account | authenticated. owed, reserved, outstanding, kib_admitted … | +| POST /v1/pay | body = SignedCheque JSON | +| POST /v1/push | 402 Payment Required when over cap | -`/v1/account` is authenticated because unauthenticated it is a per-identity **volume oracle** over on-chain-enumerable batch owners. +The upload format is **unchanged**. Payment stays off the upload path, so a payment problem never breaks an upload. The balance endpoint needs a login, because otherwise anyone could look up how much any account has uploaded. --- -# Admission: why a challenge at all +# Why a challenge at all -The naive claim: *"402 is easy — `/v1/push` commits its status before processing any chunk."* +The tempting answer is that refusing is easy, because the relay picks its response code before it looks at any chunk. That is true, and it is exactly the **problem** — at that moment it does not yet know **whose account to check**. -True, and that is exactly the **problem**: at that moment the relay does not yet know **whose account to check**. The account only exists after `stamp::validate` and `resolve_owner` — both inside the spawned task. +Working out who is paying means checking the stamp and looking up the batch owner, and both happen later — after the response has already gone out. -Hoisting them means up to **512 ecrecovers (~40 ms) plus an RPC round-trip synchronously in front of every response** — the precise unauthenticated amplification surface we spend a whole section defending. +> Doing those checks first means up to 512 signature recoveries (~40 ms) and a chain lookup **before the relay can answer at all** — cheap for an attacker to trigger, expensive for the relay to serve. Exactly the shape the design spends a section trying to avoid. --- -# The challenge is a capability - -Standing is resolved once when the challenge is **issued**. The credit line is baked into the nonce. `/v1/push` admission then reads **no chain state at all**. +# The permission slip proves the checks were already done ``` GET /v1/challenge?account=A&batch=B @@ -265,84 +222,71 @@ GET /v1/challenge?account=A&batch=B → nonce = HMAC(relay_secret, preimage(A, B, origin, expiry, cap)) ``` -Admission becomes: verify MAC (constant-time) → verify origin → verify client signature → `reserve = ceil(len/1024) × price`; if `outstanding + reserve > cap` → **402, before reading the body**. +The chain lookups happen once, when the slip is issued, and the credit limit is sealed into it. After that an upload needs **no chain lookups at all**: check the slip, check it was issued for this relay, check the signature, set the money aside — and if that would go over the limit, **refuse before reading the upload**. -**Stateless.** No server-side nonce table, so a free `GET /v1/challenge` cannot exhaust memory. +The relay keeps no list of the slips it has issued, so handing them out for free cannot fill up its memory. --- -# Two ways the binding silently becomes a no-op - -**1. The preimage is fixed-width and domain-tagged, not a concatenation.** +# Two ways to make the check do nothing -`origin` is variable-length, so a bare concatenation makes `("host.a","bc")` and `("host.ab","c")` share a preimage — one nonce valid for two hostnames. +**1. Glue the fields together carelessly and two different inputs look identical.** The hostname varies in length, so `("host.a","bc")` and `("host.ab","c")` run together into the same bytes — one slip that works for two different hostnames. -**2. `origin` must be *configured*, not derived.** +**2. The relay must know its own name from its config, not from the request.** The obvious version compares the slip against the `Host` header — which does nothing, because the attacker sends both. -The obvious implementation compares the challenge's `origin` against the `Host` header. That is a **no-op** — `Host` is supplied by the same client supplying the challenge. - -> An attacker replaying a victim's signature at relay B just sends `Host: relay-b.example`. The comparison passes. The cross-relay replay is restored **while the doc claims it is closed.** +> An attacker reusing a victim's signature at relay B just sends `Host: relay-b.example`. The check passes, and the attack works again — **while the code looks like it is preventing it.** --- -# Deriving the credit line instead of asserting it - -The tempting argument: *"an account is a batch owner, a live batch costs real BZZ, so the margin is three orders of magnitude."* **False.** +# Credit scales with what the batch is worth -The relay checks **liveness**, and liveness is satisfied by the cheapest batch the contract accepts — minimum depth, minimum validity — a fraction of a cent. At a flat credit line the real Sybil margin is of order **1×**. +It is tempting to assume a batch owner has real money at stake. They do not have to: the relay only checks that a batch is *alive*, and the cheapest batch the contract accepts costs a fraction of a cent. With one flat credit limit, an attacker gets back roughly what they paid. -> **`max_outstanding(A,B) = min(remaining_value_plur(B) ÷ credit_ratio, max_outstanding_plur)`** -> with `credit_ratio = 1000` +``` +max_outstanding(A,B) = min(remaining_value_plur(B) ÷ credit_ratio, + max_outstanding_plur) credit_ratio = 1000 +``` -The margin is now **1000× by construction, independent of batch size.** There is no cheap corner of the parameter space, because the *ratio* is the invariant. +> An attacker now gets **a thousandth of what they funded, whatever they buy.** There is no cheap way in, because what is fixed is the *ratio* — not an amount someone can undercut. --- -# Parameters, and the invariant that must hold +# The rule the three thresholds must satisfy -> **`min_cheque_plur ≤ settle_every_plur < max_outstanding_plur`** -> A client that is 402'd must always be able to clear it with a cheque for exactly what it owes. +> min_cheque_plur ≤ settle_every_plur < max_outstanding_plur -| parameter | PLUR | in payload | +| Parameter | PLUR | In payload | |---|---:|---:| -| `price_plur_per_kib` | 4.8e8 | 1 KiB | -| `min_cheque_plur` | 3.9e12 | ~8 MiB | -| `settle_every_plur` | 1.56e13 | ~32 MiB | -| `max_outstanding_plur` | 6.22e13 | ~127 MiB (*ceiling*, not the cap) | +| price_plur_per_kib | 4.8e8 | 1 KiB | +| min_cheque_plur | 3.9e12 | ~8 MiB | +| settle_every_plur | 1.56e13 | ~32 MiB | +| max_outstanding_plur | 6.22e13 | ~127 MiB | -An early draft published `min_cheque` **87× larger** than `settle_every`. Every metered account would have bricked: accrue → cross → sign → **rejected as dust** → accrue → 402 → the only clearing cheque is 21× what is owed. **No exit.** +An early draft set the minimum cheque **87× larger** than the point where you are meant to pay. Every account would have jammed: run up a bill, try to pay, get told the cheque is too small, keep running it up, get refused — and the only cheque that would clear is 21× what you owe. No way out. --- -# Reservation: bee's `reserve` was needed after all - -A monotone debit counter is **not** sufficient. `/v1/push` deliberately does not serialize, so N concurrent POSTs each read `outstanding` before any of them debits. +# Counting is not enough when uploads overlap -A *polite* client at the relay's own advertised `inflight_max` of 8 overshoots the cap on its own. +Just adding up what is owed is **not** enough. Uploads run in parallel on purpose, so several can each check the balance before any of them has been charged. Even a well-behaved client sending the 8 the relay itself recommends will blow past the limit. -Fix: reserve `ceil(Content-Length / 1024) × price` **atomically at admission**, release the remainder at completion. +The fix: set the money aside up front, based on the size the client declared, and give back whatever was not used when the upload finishes. -> **But `reserved` must not be persisted.** A reservation belongs to an in-flight POST, and no in-flight POST survives a restart — there is no task left to release it. - -Persist `owed`, `last_cumulative`, the chequebook binding. **Reconstruct `reserved` as zero at boot.** +> But the set-aside amounts must not be saved to disk. Each one belongs to an upload in progress, and no upload survives a restart — so nothing would ever release it. Save what is owed and the running total; start the set-aside amounts at zero. --- -# Pricing: the cost basis +# The cost basis -| | per 4 KiB chunk relayed | -|---|---| +| Per 4 KiB chunk relayed | | +|---|---:| | Delivery on the wire | ≈ 4.4 KiB | -| Peer race (`CHUNK_PEER_PARALLELISM = 3`) | **×3** | -| Shallow retries at pool 128 | **×1.15** | -| **Egress per chunk** | **≈ 15 KiB** | -| **Egress per GiB of payload** | **≈ 3.7 GiB** | - -Suggested price **$0.02/GiB** — ~5× a VPS's raw bandwidth cost, ~18× cheaper than AWS egress. +| Sent to three peers at once | × 3 | +| Retries for chunks that go nowhere | × 1.15 | +| **Egress per chunk** | ≈ 15 KiB | +| **Egress per GiB of payload** | ≈ 3.7 GiB | -> **`price_plur_per_kib ≈ 4.8 × 10⁸`** (1 BZZ = 10¹⁶ PLUR) - -Flat per KiB, deliberately: any curve steeper than flat re-introduces a per-item number for the two sides to disagree about. +> $0.02 per GiB works out to `4.8 × 10⁸` PLUR per KiB. The rate is flat on purpose: anything more complicated puts a per-chunk number back into the bill, and that is one more thing the two sides can disagree about. --- @@ -350,15 +294,13 @@ Flat per KiB, deliberately: any curve steeper than flat re-introduces a per-item At 3.7 GiB of real egress per GiB of payload: -| | per GiB of payload | +| | Per GiB of payload | |---|---:| -| AWS egress at $0.09/GB | **−$0.33** | -| Revenue at $0.02/GiB | **+$0.02** | -| Net | **−$0.31** | - -So metered mode only clears cost on **flat-rate or included bandwidth**. A relay on per-GB egress should run `open` and absorb the quota. +| AWS egress at $0.09/GB | −$0.33 | +| Revenue at $0.02/GiB | +$0.02 | +| **Net** | **−$0.31** | -That is the same host class already required for durable storage (§11.4), so the two constraints select the same machines. +> Charging only covers costs on hosts with **flat-rate or included bandwidth**. On per-GB hosting, run it free and absorb the quota instead. That is the same kind of host already needed for a disk that survives restarts, so both requirements point at the same machines. --- @@ -366,185 +308,151 @@ That is the same host class already required for durable storage (§11.4), so th Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ **$0.0005** on Gnosis. -| account's lifetime traffic | revenue | gas | gas as % | +| Account's lifetime traffic | Revenue | Gas | Gas as % | |---|---:|---:|---:| -| 71 MB (one browser upload) | $0.0014 | $0.0005 | **36 %** | -| 5 GiB (cashout threshold) | $0.10 | $0.0005 | **0.5 %** | - -Because cheques are cumulative, gas is paid once per **account**, not per cheque — so the ratio improves with every return visit, and accounts below the threshold are written off unclaimed. +| 71 MB — one browser upload | $0.0014 | $0.0005 | 36 % | +| 5 GiB — the cashout threshold | $0.10 | $0.0005 | 0.5 % | -A relay whose traffic is entirely one-shot uploads should run `open`. +> Gas is paid once per **customer**, not per cheque, so the ratio gets better every time someone comes back. Anyone who never reaches the threshold is written off. A relay whose users all upload once and leave should not charge at all. --- -# Attack surface — the three that matter +# Three attacks worth knowing about -**Stamp replay becomes billing griefing** *(introduced)* -Swarm stamps are public, and a relay holds every stamp it ever relayed. Replay a victim's stamps at a metered relay and the work bills to the *victim*. Cost to attacker: zero. -→ Closed by the **account-signed** challenge + one batch per POST. - -**The withdraw race** *(inherited)* -Chequebooks deploy with a hard-deposit timeout of zero, so the balance stays liquid. The funding check is true **at acceptance time, not at cashout time.** Bee has the identical exposure. - -**Relay state loss is an unbounded free-service loop** *(introduced)* -An ephemeral filesystem turns one signature into unlimited free service. -→ Durable storage is a **requirement**, not a recommendation. +- **Someone else's stamps, billed to them** *(new)* — stamps are public, and a relay has a copy of every one it has forwarded. Send a victim's stamps to a paid relay and the victim's account picks up the bill, at no cost to the attacker. Fixed: the client must sign the challenge with the batch owner's key, and every chunk in a request must belong to the batch named in it. +- **The customer can empty the chequebook after you accept** *(inherited)* — the owner can withdraw at any time, so "this cheque is funded" is true when you take it, not when you cash it. Bee has exactly the same exposure. +- **A relay that forgets serves for free forever** *(new)* — if the ledger does not survive a restart, one signature buys unlimited service. So a paid relay needs a real disk. That is a requirement, not advice. --- -# Part II — Practice +# Part Two — Practice --- -# Modes, and the optionality rule +# Paying is optional, per lane -| relay mode | client has a chequebook | client does not | +| Relay mode | Client has a chequebook | Client does not | |---|---|---| -| `open` | used, nothing billed | used, nothing billed | -| `metered`, soft | used, billed, settles | **used, billed, served anyway** | -| `metered`, hard | used, billed, settles | **lane retired at startup** | +| open | used, nothing billed | used, nothing billed | +| metered, soft | used, billed, settles | used, billed, served anyway | +| metered, hard | used, billed, settles | lane retired at startup | -Retiring matters because a hard lane answers an unchallenged push with **401**, and only a 402 is exempt from **lane health**. Scheduling one anyway costs each chunk one of its `max_attempts` retries, per chunk, to rediscover what `/v1/status` already stated. - -Both drivers drop it up front — native when no `--chequebook` is configured, browser unconditionally. +> A paid relay rejects an upload with no permission slip, and that rejection counts against the relay's health score — only a genuine "you owe too much" is excused. So sending work there anyway burns one retry on every single chunk, to rediscover something the relay already said up front. --- -# Soft mode is an instrument, not a migration path +# What soft mode does and does not drop Soft mode meters, reports and accepts cheques, but **never answers 402**. -**It still requires the challenge.** An earlier draft implied unchallenged requests should be served — which would make metering bypassable *by omitting a header*. That is not a degraded mode; it is no mode at all. +**It still requires the permission slip.** An earlier draft suggested serving requests that arrive without one — which would let anyone skip paying by leaving out a header. > What soft mode drops is enforcement of the cap, **not authentication**. -A relay flipping to `--meter` therefore *does* break clients that predate the protocol. Acceptable: the only dApp using these lanes ships alongside them. +A relay flipping to `--meter` therefore does break clients that predate the protocol. Acceptable: the only dApp using these lanes ships alongside them. --- -# Six bugs that only a *running* relay could find - -None is reachable from a single upload against a fresh relay. All six survived the full test suite **and** the Stage 1 round-trip. +# Six bugs only a running relay could find -Reaching them needed, simultaneously: +None is reachable from a single upload against a fresh relay. All six survived the full test suite **and** the Stage 1 round-trip. Reaching them needed, simultaneously: - a relay that **remembers what you owe between runs**, so debt carries over - several uploads in flight at the same time, not one after another - a batch **used up far enough that the credit limit is what stops you**, rather than anything else -> §17.3 is the one to generalise from: it is not a coding error but an **invariant checked against the wrong quantity**, and it only becomes reachable once a real batch's value has decayed below ~0.39 BZZ. +> The one to learn from is §17.3. It is not a coding mistake — it is a **rule checked against the wrong number**, and it only shows up once a real batch has been worn down below about 0.39 BZZ. --- -# The six +# The six bugs -| # | Bug | -|---|---| +| § | Bug | +|---:|---| | 17.1 | Debt the relay carried **across sessions** could not be paid | | 17.2 | The headroom guard admitted a *frame*, then sent a *batch* | -| 17.3 | §10.1's invariant checked against the wrong quantity | +| 17.3 | The §10.1 invariant checked against the wrong quantity | | 17.4 | A lane refused for **bytes in flight** was parked for good | | 17.5 | A broken response stream made **every later cheque bounce** | | 17.6 | The first POST of a run was sized **before the debt was known** | --- -# 17.1 — a slow-motion deadlock - -The dust floor guarantees a run ends owing something: the residual below `min_cheque_plur` is left unpaid, because a cheque for it would be refused. +# Debt carried over, and could not be paid -The relay is **right** to keep counting it — forgiving it would make *"stay under the floor"* a way to be served free. +Because there is a minimum cheque size, every upload ends owing a little too small to pay. The relay is **right** to keep counting it — writing it off would make staying under the minimum a way to be served for free. -But the client's books are per-process. The next run starts believing it owes **nothing**, and the relay's `owed` only ever grows. Once the carry crosses the cap, the first POST is refused — and the refusal is **unpayable**, because the cheque is computed from the client's own `owed`, which is zero. +But the client forgets when it exits. The next run starts thinking it owes **nothing**, while the relay's total keeps growing. Eventually the very first upload is refused — and the client **cannot pay**, because it writes cheques from its own figure, which is zero. > Observed live: a second upload failing **151/151** against a relay carrying 290,400,000,000 PLUR. -Fix: **ask rather than remember** — reconcile against `GET /v1/account`. - --- -# 17.1 — three ways to get the fix wrong - -Each of these was wrong in a draft: +# Three ways to get the fix wrong - **Ask what you owe. Don't read it off the refusal.** The refusal already counts the request it is turning down, so paying that number overpays by exactly one request — and the next cheque bounces for being too big. - - **Don't count bytes still on the wire.** The client is already tracking those, so counting them again bills them twice. Guessing low is safe and fixes itself on the next round; guessing high gets the cheque rejected. - -- **Cap it at the limit the relay signed up to.** Not the batch's current limit — that falls as the batch is used up, so it can drop below a bill you honestly ran up, and refusing to pay that **keeps you stuck**. And not the chequebook balance either: that lets any relay you point at ask for everything you have. +- **Cap it at the limit the relay signed up to.** Not the batch's current limit — that falls as the batch is used up, so it can drop below a bill you honestly ran up, and refusing to pay that keeps you stuck. And not the chequebook balance either: that lets any relay you point at ask for everything you have. --- -# 17.3 — the invariant was checked against the wrong quantity - -`Params::validate` checked `min_cheque ≤ settle_every < max_outstanding` against **`max_outstanding_plur`** — the global ceiling. +# The invariant was checked against the wrong quantity -But the line that actually binds is **per batch**: +The check compared the three thresholds against the highest limit the relay ever grants. But the limit that actually applies is **per batch**, and it is usually much smaller: ``` min(remaining_value / credit_ratio, ceiling) ``` -Below ~0.39 BZZ of batch value, the configured dust floor **exceeds everything the account can owe**. It accrues to its cap and can never write an acceptable cheque. +Once a batch is worth less than about 0.39 BZZ, the minimum cheque is **bigger than anything that account is allowed to owe**. It runs up to its limit and can never write a cheque the relay will take. -> Permanent refusal. Nothing broken. No error anywhere. - -Thresholds are now resolved via `Params::effective(cap)` on **both** sides. +> The account is refused from then on. Nothing is broken and nothing logs an error — it just stops working. Both sides now compare against the limit that actually applies. --- -# Results — after all six fixes - -Hard-mode relay, uploads 128 KiB → 4 MiB: +# After all six fixes -| payload | frames acked | 402s | stuck | rejected cheques | -|--------:|-------------:|-----:|------:|-----------------:| +| Payload | Frames acked | 402s | Stuck | Rejected cheques | +|---|---:|---:|---:|---:| | 128 KiB | 43/43 | 0 | 0 | 0 | | 512 KiB | 151/151 | 0 | 0 | 0 | | 1 MiB | 290/290 | 2 | 0 | 0 | | 4 MiB | 1122/1122 | 4 | 0 | 0 | -The remaining 402s are the **intended** kind: the line genuinely fills, the client pays or waits, the lane resumes. +> The remaining 402s are the *intended* kind: the line genuinely fills, the client pays or waits, the lane resumes. --- -# Results — over public HTTPS, with §17.6 in place - -Repeated through a reverse proxy, with the client learning its carried debt *before* sizing anything: +# Over public HTTPS, with §17.6 in place -| run | payload | frames acked | 402s | rejected cheques | -|----:|--------:|-------------:|-----:|-----------------:| +| Run | Payload | Frames acked | 402s | Rejected cheques | +|---:|---|---:|---:|---:| | 1 | 2 MiB | 567/567 | 0 | 0 | | 2 | 2 MiB | 567/567 | 0 | 0 | | 3 | 2 MiB | 567/567 | 0 | 0 | -Each run settles to **`owed: 0`** on the relay, so the next carries nothing. +Each run settles to `owed: 0` on the relay, so the next carries nothing. -> That is the intended steady state: **402 is the recovery path, not the mechanism.** +> That is how it is meant to run: **a refusal is the fallback, not the normal path.** --- # It settles on-chain -The loop closes end to end on Gnosis mainnet: - - The relay counts what you owe. You sign a cheque for the running total. The relay checks it and marks you paid. - The relay is holding a cheque for **21,366,720,000,000 PLUR** right now. - Cashing it happens on a **different machine**. Only the payee can cash a cheque, and the relay must never hold that key. +- It worked: the transaction succeeded and used 75,378 gas, against a 300,000 budget. -It worked: the transaction succeeded and used **75,378 gas**, against a 300,000 budget. - -> The relay box never holds spendable key material. It needs the beneficiary's **address** only. The property that makes today's pusher safe survives metering intact. +> The relay never holds a key that can spend anything. It only needs the payee's **address**. So charging for relay does not make a relay box worth breaking into. --- # What is actually deployed -**Four `open` lanes** (free tiers, ephemeral disks — they *must* run open) plus **one hard-metered lane**, `pusher.browserbzz.link`, at 4.8e8 PLUR/KiB. +**Four free relays** — on hosts whose disks are wiped on restart, so they have to stay free — plus **one paid relay** that enforces payment. -The browser dApp lists all five and **skips the metered one automatically**, because it stamps but never settles. - -A native client with `--chequebook` uses all five. +The browser app lists all five and **skips the paid one automatically**, because it can sign chunks but not cheques. A command-line client with a chequebook uses all five. > Payment is a property of a relay, not of the fleet. @@ -552,17 +460,11 @@ A native client with `--chequebook` uses all five. # Net effect on the design -**Removed:** five mechanisms and every attack on them, just by billing bytes instead of receipts — the staking check, the list-shaped bill, the spot-check audit, the log sweep on both sides, and the shared receipt-checking code. - -**Added:** three, all cheap — one signature check when a client asks permission, one chain lookup per batch every half hour, and one amount set aside per upload. +- **Removed:** five mechanisms and every attack on them, just by billing bytes instead of receipts — the staking check, the list-shaped bill, the spot-check audit, the log sweep on both sides, and the shared receipt-checking code. +- **Added:** three, all cheap — one signature check when a client asks permission, one chain lookup per batch every half hour, and one amount set aside per upload. +- **Now guaranteed rather than hoped for:** an attacker gets credit worth a thousandth of what they actually funded, whatever size batch they buy; credit shrinks by itself as a batch is used up, with no expiry code to get wrong; and the relay holds no key that can move money. -**Now guaranteed rather than hoped for:** - -- An attacker gets credit worth a thousandth of what they actually funded, whatever size batch they buy -- Credit shrinks by itself as a batch is used up, with no expiry code to get wrong -- The relay holds no key that can move money - -**Found by running it:** six bugs no test suite reached; five needed a ledger outliving the client. +> Found by running it: six bugs no test suite reached — five needed a ledger that outlives the client. --- @@ -573,6 +475,4 @@ A native client with `--chequebook` uses all five. - **The empty-the-chequebook problem has no fix.** Bee has it too — it comes from how the chequebook contract is deployed, not from anything here. - **Nothing stops a relay taking your money and dropping chunks.** You lose at most one credit limit and then stop using it — that bounds the damage, but it is not a guarantee of service. -
    - -*Full design: `docs/pusher-incentives.md` — §8 for the billing unit, §10.3 for the Sybil bound, §17 for the bugs.* +Full design: `docs/pusher-incentives.md` — §8 billing unit, §10.3 Sybil bound, §17 the bugs. From 4def682a83fc9cf5a46fe7f824492b806c00c2a9 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 17:24:34 +0300 Subject: [PATCH 19/27] docs(deck): markdown is the source, HTML is built from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Had the pipeline backwards. md4c's md2html is installed, so the markdown can be the source and the HTML generated from it — which is the right direction anyway: the markdown is what gets copy-edited, and it is what marp-cli renders to PDF unchanged. docs/deck/build.py splits the deck on slide breaks, hands each one to md2html --github, and wraps the result in the deck shell (CSS, nav, print rules). md2html does the inline markup, GFM tables and column alignment; the script only assembles. Four HTML-comment directives carry what markdown has no syntax for — title card, section divider, eyebrow label, hazard callout — and every markdown renderer ignores them. Blockquotes become the accent callout, so a plain reading of the markdown and the rendered deck emphasise the same lines. Drops docs/deck2md.py, which went the other way. Output verified against the hand-written HTML: identical slide, callout, eyebrow and table counts, and 99.4% token-identical text (the difference is slide numbers, which the old file filled in via JS). Co-Authored-By: Claude Opus 5 (1M context) --- docs/deck/build.py | 141 ++++++++++++++++ docs/deck/shell.bar.html | 7 + docs/deck/shell.css.html | 274 +++++++++++++++++++++++++++++++ docs/deck/shell.js.html | 44 +++++ docs/deck2md.py | 109 ------------ docs/pusher-incentives-slides.md | 108 +++++++++++- 6 files changed, 572 insertions(+), 111 deletions(-) create mode 100644 docs/deck/build.py create mode 100644 docs/deck/shell.bar.html create mode 100644 docs/deck/shell.css.html create mode 100644 docs/deck/shell.js.html delete mode 100644 docs/deck2md.py diff --git a/docs/deck/build.py b/docs/deck/build.py new file mode 100644 index 0000000..1b359d7 --- /dev/null +++ b/docs/deck/build.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build the slide deck HTML from the Marp markdown. + + python3 docs/deck/build.py docs/pusher-incentives-slides.md -o /tmp/deck.html + +The markdown is the source. It stays a valid Marp deck — `marp-cli` renders it +to PDF unchanged — and this script produces the self-contained HTML version +with keyboard navigation. Inline markdown, GFM tables and column alignment are +handled by `md2html` (md4c), not by anything here. + +Four directives, all HTML comments, so every markdown renderer ignores them: + + this slide is the title card + section divider, with that kicker + the small label above the heading + the next blockquote is the warning colour + +Blockquotes become the accent callout. Tables are wrapped so wide ones scroll +inside the slide rather than pushing the page sideways. +""" +import argparse +import html +import pathlib +import re +import shutil +import subprocess +import sys + +HERE = pathlib.Path(__file__).parent + + +def render(md: str) -> str: + """Markdown fragment -> HTML, via md2html.""" + if not md.strip(): + return "" + out = subprocess.run( + ["md2html", "--github"], input=md, capture_output=True, text=True, check=True + ).stdout + # Wide content scrolls in its own box; the slide itself never does. + out = re.sub(r"(.*?
    )", r'
    \1
    ', out, flags=re.S) + return out.strip() + + +def slide(body: str, index: int) -> str: + """One markdown slide -> one
    .""" + title = "" in body + part = re.search(r"", body) + eyebrow = re.search(r"", body) + body = re.sub(r"", "", body) + + # `` tags the blockquote that follows it. md2html drops + # comments, so the flag rides through as a sentinel inside the quote. + body = re.sub(r"\s*\n+>", "> @@HZ@@", body) + out = render(body) + + # Blockquote is the deck's callout. Unwrap its paragraph so the callout is + # one styled block rather than a quote wrapping a paragraph. + def callout(m: re.Match) -> str: + inner = re.sub(r"", "", m.group(1)).strip() + cls = "claim" + if inner.startswith("@@HZ@@"): + cls, inner = "claim hz", inner[len("@@HZ@@"):].strip() + return f'
    {inner}
    ' + + out = re.sub(r"
    (.*?)
    ", callout, out, flags=re.S) + + num = f'
    {index:02d}
    ' + + if title: + h1 = re.search(r"

    (.*?)

    ", out, re.S) + h2 = re.search(r"

    (.*?)

    ", out, re.S) + meta = re.search(r"

    (.*?)

    ", out, re.S) + return ( + '
    \n' + '
    \n' + f"

    {h1.group(1) if h1 else ''}

    \n" + f'

    {h2.group(1) if h2 else ""}.

    \n' + f'
    {meta.group(1) if meta else ""}
    \n' + "
    " + ) + + if part: + h1 = re.search(r"

    (.*?)

    ", out, re.S) + return ( + '
    \n' + f'
    {html.escape(part.group(1))}
    \n' + f"

    {h1.group(1) if h1 else ''}

    \n" + f" {num}\n" + "
    " + ) + + # A slide's `#` heading is an

    visually —

    is reserved for the deck. + out = re.sub(r"

    (.*?)

    ", r"

    \1

    ", out, flags=re.S) + eb = ( + f'
    {eyebrow.group(1)}
    \n ' + if eyebrow + else "" + ) + # Everything that is not a heading, table, code block or callout is prose. + out = re.sub( + r"((?:<(?:p|ul|ol)>.*?\s*)+)", + lambda m: f'
    {m.group(1).strip()}
    ', + out, + flags=re.S, + ) + return f'
    \n {eb}{out}\n {num}\n
    ' + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("source", type=pathlib.Path) + ap.add_argument("-o", "--output", type=pathlib.Path, required=True) + args = ap.parse_args() + + if not shutil.which("md2html"): + print("md2html not found (md4c). Install it, or render with marp-cli.", file=sys.stderr) + return 1 + + text = args.source.read_text() + # Drop the Marp front matter; it configures marp-cli, not this. + text = re.sub(r"\A---\n.*?\n---\n", "", text, flags=re.S) + title = "Paying for relay — an incentive layer for hoverfly pushers" + + slides = [s for s in re.split(r"\n---\n", text) if s.strip()] + sections = "\n\n ".join(slide(s, i + 1) for i, s in enumerate(slides)) + + args.output.write_text( + f"{title}\n\n" + + (HERE / "shell.css.html").read_text() + + f'\n
    \n\n {sections}\n\n
    \n\n' + + (HERE / "shell.bar.html").read_text() + + "\n" + + (HERE / "shell.js.html").read_text() + + "\n" + ) + print(f"{len(slides)} slides -> {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/deck/shell.bar.html b/docs/deck/shell.bar.html new file mode 100644 index 0000000..38d1235 --- /dev/null +++ b/docs/deck/shell.bar.html @@ -0,0 +1,7 @@ +
    + + +
    + +
    + diff --git a/docs/deck/shell.css.html b/docs/deck/shell.css.html new file mode 100644 index 0000000..50cca9f --- /dev/null +++ b/docs/deck/shell.css.html @@ -0,0 +1,274 @@ + \ No newline at end of file diff --git a/docs/deck/shell.js.html b/docs/deck/shell.js.html new file mode 100644 index 0000000..13688ca --- /dev/null +++ b/docs/deck/shell.js.html @@ -0,0 +1,44 @@ + \ No newline at end of file diff --git a/docs/deck2md.py b/docs/deck2md.py deleted file mode 100644 index 28eb7a8..0000000 --- a/docs/deck2md.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -"""Regenerate the Marp markdown deck from the HTML deck. - -The two carry the same content, and keeping them in step by hand meant every -copy edit had to be applied twice — which is exactly how they drift. The HTML -is the source; this derives the markdown from it. -""" -import html -import re -import sys - -SRC, DST = sys.argv[1], sys.argv[2] -doc = open(SRC).read() - -FRONT = """--- -marp: true -theme: default -paginate: true -header: 'Paying for relay — an incentive layer for hoverfly pushers' ---- -""" - - -def inline(s: str) -> str: - """HTML inline markup -> markdown.""" - s = re.sub(r"(.*?)", lambda m: "`" + re.sub(r"<[^>]+>", "", m.group(1)) + "`", s, flags=re.S) - s = re.sub(r"(.*?)", r"**\1**", s, flags=re.S) - s = re.sub(r"(.*?)", r"*\1*", s, flags=re.S) - s = re.sub(r"", "\n", s) - s = re.sub(r"<[^>]+>", "", s) # spans (ok/bad/q/hz) carry colour only - s = html.unescape(s) - return re.sub(r"[ \t]*\n[ \t]*", " ", s).strip() - - -def cells(row: str, tag: str) -> list: - # A literal pipe inside a cell would open a new column. - return [inline(c).replace("|", r"\|") - for c in re.findall(rf"<{tag}[^>]*>(.*?)", row, re.S)] - - -def table(block: str) -> str: - rows = re.findall(r"(.*?)", block, re.S) - if not rows: - return "" - head = cells(rows[0], "th") or cells(rows[0], "td") - body = [cells(r, "td") for r in (rows[1:] if cells(rows[0], "th") else rows)] - # Right-align any column the HTML marked with class="n". - aligns = ["---:" if 'class="n"' in c else "---" - for c in re.findall(r"]*)>", rows[0], re.S)] - out = ["| " + " | ".join(head) + " |", - "|" + "|".join(aligns[:len(head)] or ["---"] * len(head)) + "|"] - for b in body: - out.append("| " + " | ".join(b) + " |") - return "\n".join(out) - - -slides = [] -for sec in re.findall(r'
    (.*?)
    ', doc, re.S): - kind, body = sec - out = [] - - if "title" in kind: - h1 = re.search(r"

    (.*?)

    ", body, re.S) - lede = re.search(r'

    (.*?)

    ', body, re.S) - meta = re.search(r'
    (.*?)
    ', body, re.S) - out.append("# " + inline(h1.group(1))) - if lede: - out.append("## " + inline(lede.group(1)).rstrip(".")) - if meta: - out.append("*" + inline(meta.group(1)).replace(" ", " · ") + "*") - slides.append("\n\n".join(out)) - continue - - if "part" in kind: - kicker = re.search(r'
    (.*?)
    ', body, re.S) - h2 = re.search(r"

    (.*?)

    ", body, re.S) - label = inline(kicker.group(1)) if kicker else "" - slides.append(f"# {label.title()} — {inline(h2.group(1))}") - continue - - h2 = re.search(r"

    (.*?)

    ", body, re.S) - if h2: - out.append("# " + inline(h2.group(1))) - - # Walk the slide's blocks in document order. - pattern = (r'
    (.*?)
    \s*(?=<)' - r'|
    \s*(.*?
    )\s*
    ' - r'|
    (.*?)
    ' - r'|
    (.*?)
    ') - for m in re.finditer(pattern, body, re.S): - bodydiv, tbl, pre, claim = m.groups() - if bodydiv is not None: - for el in re.finditer(r"]*>(.*?)

    |
      (.*?)
    ", bodydiv, re.S): - para, ul = el.groups() - if para is not None: - out.append(inline(para)) - else: - out.append("\n".join( - "- " + inline(li) for li in re.findall(r"
  • (.*?)
    • ", ul, re.S))) - elif tbl is not None: - out.append(table(tbl)) - elif pre is not None: - out.append("```\n" + html.unescape(re.sub(r"<[^>]+>", "", pre)).strip() + "\n```") - elif claim is not None: - out.append("> " + inline(claim)) - slides.append("\n\n".join(x for x in out if x.strip())) - -open(DST, "w").write(FRONT + "\n" + "\n\n---\n\n".join(slides) + "\n") -print(f"{len(slides)} slides -> {DST}") diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 1ce1f3f..26c4edf 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -5,6 +5,14 @@ paginate: true header: 'Paying for relay — an incentive layer for hoverfly pushers' --- +, part:, eyebrow:, hazard) are documented + in docs/deck/build.py. --> + + + # Paying for relay ## An incentive layer for hoverfly pushers, reusing parts of SWAP @@ -13,6 +21,8 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' --- + + # The relay pays for traffic it did not cause In a **native** upload your own machine opens the pushsync streams, and bee debits *you*: @@ -23,10 +33,14 @@ price(po) = (32 − po) × 10 000 accounting units Put a relay in the middle and that debt moves **wholesale to the relay** — it is the peer bee sees, so it is the peer bee charges. The browser client that caused the traffic pays **nothing but postage**. + + > Booked today as accepted risk: *“worst case = the platform's free egress for the month burned, $0 lost.”* --- + + # What changes | | open (today) | metered | @@ -41,10 +55,14 @@ The relay still pays bee nothing, because credit was never what limited it. Bee --- -# Part One — Theory + + +# Theory --- + + # Who has to trust whom > The client chose its relay. The relay did not choose its client. @@ -55,6 +73,8 @@ So every defence *the relay* has points at the client: *a client cannot obtain s --- + + # What protects the client, then Not cryptography. Four bounds, none of which need the relay to be trustworthy: @@ -68,8 +88,12 @@ Not cryptography. Four bounds, none of which need the relay to be trustworthy: --- + + # What the previous design got wrong + + > The design was building **two-sided cryptographic verification** for a **one-sided trust relationship.** About half of it protected the client from the relay — from relays we run ourselves. That cost three real things: @@ -80,6 +104,8 @@ About half of it protected the client from the relay — from relays we run ours --- + + # What we borrow from SWAP | Borrowed | Why | @@ -93,6 +119,8 @@ About half of it protected the client from the relay — from relays we run ours --- + + # What we drop | Dropped | Reason | @@ -107,6 +135,8 @@ About half of it protected the client from the relay — from relays we run ours --- + + # The account is the batch owner Relays already know who is uploading. Every chunk carries a stamp, and its signature has to match the address the postage contract lists as the batch's owner. @@ -117,6 +147,8 @@ So there is nothing to add: no logins, no sign-up, no extra message. In a browse --- + + # Bill bytes admitted ``` @@ -132,6 +164,8 @@ Both numbers are known to both parties *before* any push work happens. --- + + # What that replaced The earlier draft billed per **verified pushsync receipt**. To make a *third party's* signature into a billing input it needed: @@ -145,6 +179,8 @@ The earlier draft billed per **verified pushsync receipt**. To make a *third par --- + + # Bytes, not successful pushes Bytes are what actually costs the relay money. It sends every chunk to three peers at once and retries the ones that go nowhere, and that traffic goes out whether or not the chunk ends up stored. Charging only for successes would leave the relay paying for every failure. @@ -156,6 +192,8 @@ Bytes are what actually costs the relay money. It sends every chunk to three pee --- + + # Dedup hits are billed at zero If the relay already pushed the same chunk moments ago, it does no work the second time, so those bytes come off the bill. This is the **only** part of the bill that rests on the relay's own word. @@ -164,16 +202,22 @@ If the relay already pushed the same chunk moments ago, it does no work the seco --- + + # Cumulative cheques - **Losing one costs nothing.** Every cheque is a running total, so if a payment fails the next one covers it anyway. No retry logic needed. - **Old cheques are worthless.** Each must be larger than the last, so sending an old one again pays nothing. - **Gas is paid once per customer, not once per cheque.** The relay only ever cashes the newest total, so every earlier cheque costs nothing to collect. + + > A cheque per chunk was rejected: 137 bytes and a signature on every 4 KiB, and — worse — running totals have to go in order, so chunks would have to be sent one at a time. That would remove the parallel uploads the scheduler relies on for speed. --- + + # Cheques are per payee, not per relay A running total belongs to a **payee**, but a relay is a **URL**. One operator running four relay URLs that all pay into the same account is the obvious way to deploy. @@ -188,6 +232,8 @@ relay applies ErrChequeNotIncreasing → rejected, forever --- + + # Payment happens outside the upload | Endpoint | Shape | @@ -202,16 +248,22 @@ The upload format is **unchanged**. Payment stays off the upload path, so a paym --- + + # Why a challenge at all The tempting answer is that refusing is easy, because the relay picks its response code before it looks at any chunk. That is true, and it is exactly the **problem** — at that moment it does not yet know **whose account to check**. Working out who is paying means checking the stamp and looking up the batch owner, and both happen later — after the response has already gone out. + + > Doing those checks first means up to 512 signature recoveries (~40 ms) and a chain lookup **before the relay can answer at all** — cheap for an attacker to trigger, expensive for the relay to serve. Exactly the shape the design spends a section trying to avoid. --- + + # The permission slip proves the checks were already done ``` @@ -228,16 +280,22 @@ The relay keeps no list of the slips it has issued, so handing them out for free --- + + # Two ways to make the check do nothing **1. Glue the fields together carelessly and two different inputs look identical.** The hostname varies in length, so `("host.a","bc")` and `("host.ab","c")` run together into the same bytes — one slip that works for two different hostnames. **2. The relay must know its own name from its config, not from the request.** The obvious version compares the slip against the `Host` header — which does nothing, because the attacker sends both. + + > An attacker reusing a victim's signature at relay B just sends `Host: relay-b.example`. The check passes, and the attack works again — **while the code looks like it is preventing it.** --- + + # Credit scales with what the batch is worth It is tempting to assume a batch owner has real money at stake. They do not have to: the relay only checks that a batch is *alive*, and the cheapest batch the contract accepts costs a fraction of a cent. With one flat credit limit, an attacker gets back roughly what they paid. @@ -251,6 +309,8 @@ max_outstanding(A,B) = min(remaining_value_plur(B) ÷ credit_ratio, --- + + # The rule the three thresholds must satisfy > min_cheque_plur ≤ settle_every_plur < max_outstanding_plur @@ -266,16 +326,22 @@ An early draft set the minimum cheque **87× larger** than the point where you a --- + + # Counting is not enough when uploads overlap Just adding up what is owed is **not** enough. Uploads run in parallel on purpose, so several can each check the balance before any of them has been charged. Even a well-behaved client sending the 8 the relay itself recommends will blow past the limit. The fix: set the money aside up front, based on the size the client declared, and give back whatever was not used when the upload finishes. + + > But the set-aside amounts must not be saved to disk. Each one belongs to an upload in progress, and no upload survives a restart — so nothing would ever release it. Save what is owed and the running total; start the set-aside amounts at zero. --- + + # The cost basis | Per 4 KiB chunk relayed | | @@ -290,6 +356,8 @@ The fix: set the money aside up front, based on the size the client declared, an --- + + # On per-GB-billed hosts, metering loses money At 3.7 GiB of real egress per GiB of payload: @@ -304,6 +372,8 @@ At 3.7 GiB of real egress per GiB of payload: --- + + # Revenue per account vs. cashout gas Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ **$0.0005** on Gnosis. @@ -317,6 +387,8 @@ Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ * --- + + # Three attacks worth knowing about - **Someone else's stamps, billed to them** *(new)* — stamps are public, and a relay has a copy of every one it has forwarded. Send a victim's stamps to a paid relay and the victim's account picks up the bill, at no cost to the attacker. Fixed: the client must sign the challenge with the batch owner's key, and every chunk in a request must belong to the batch named in it. @@ -325,10 +397,14 @@ Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ * --- -# Part Two — Practice + + +# Practice --- + + # Paying is optional, per lane | Relay mode | Client has a chequebook | Client does not | @@ -341,6 +417,8 @@ Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ * --- + + # What soft mode does and does not drop Soft mode meters, reports and accepts cheques, but **never answers 402**. @@ -353,6 +431,8 @@ A relay flipping to `--meter` therefore does break clients that predate the prot --- + + # Six bugs only a running relay could find None is reachable from a single upload against a fresh relay. All six survived the full test suite **and** the Stage 1 round-trip. Reaching them needed, simultaneously: @@ -365,6 +445,8 @@ None is reachable from a single upload against a fresh relay. All six survived t --- + + # The six bugs | § | Bug | @@ -378,16 +460,22 @@ None is reachable from a single upload against a fresh relay. All six survived t --- + + # Debt carried over, and could not be paid Because there is a minimum cheque size, every upload ends owing a little too small to pay. The relay is **right** to keep counting it — writing it off would make staying under the minimum a way to be served for free. But the client forgets when it exits. The next run starts thinking it owes **nothing**, while the relay's total keeps growing. Eventually the very first upload is refused — and the client **cannot pay**, because it writes cheques from its own figure, which is zero. + + > Observed live: a second upload failing **151/151** against a relay carrying 290,400,000,000 PLUR. --- + + # Three ways to get the fix wrong - **Ask what you owe. Don't read it off the refusal.** The refusal already counts the request it is turning down, so paying that number overpays by exactly one request — and the next cheque bounces for being too big. @@ -396,6 +484,8 @@ But the client forgets when it exits. The next run starts thinking it owes **not --- + + # The invariant was checked against the wrong quantity The check compared the three thresholds against the highest limit the relay ever grants. But the limit that actually applies is **per batch**, and it is usually much smaller: @@ -406,10 +496,14 @@ min(remaining_value / credit_ratio, ceiling) Once a batch is worth less than about 0.39 BZZ, the minimum cheque is **bigger than anything that account is allowed to owe**. It runs up to its limit and can never write a cheque the relay will take. + + > The account is refused from then on. Nothing is broken and nothing logs an error — it just stops working. Both sides now compare against the limit that actually applies. --- + + # After all six fixes | Payload | Frames acked | 402s | Stuck | Rejected cheques | @@ -423,6 +517,8 @@ Once a batch is worth less than about 0.39 BZZ, the minimum cheque is **bigger t --- + + # Over public HTTPS, with §17.6 in place | Run | Payload | Frames acked | 402s | Rejected cheques | @@ -437,6 +533,8 @@ Each run settles to `owed: 0` on the relay, so the next carries nothing. --- + + # It settles on-chain - The relay counts what you owe. You sign a cheque for the running total. The relay checks it and marks you paid. @@ -448,6 +546,8 @@ Each run settles to `owed: 0` on the relay, so the next carries nothing. --- + + # What is actually deployed **Four free relays** — on hosts whose disks are wiped on restart, so they have to stay free — plus **one paid relay** that enforces payment. @@ -458,6 +558,8 @@ The browser app lists all five and **skips the paid one automatically**, because --- + + # Net effect on the design - **Removed:** five mechanisms and every attack on them, just by billing bytes instead of receipts — the staking check, the list-shaped bill, the spot-check audit, the log sweep on both sides, and the shared receipt-checking code. @@ -468,6 +570,8 @@ The browser app lists all five and **skips the paid one automatically**, because --- + + # Still open - **We don't yet know if this pays for itself.** The number to watch is how many accounts ever build up enough debt to be worth cashing. If it stays at zero, metering funds nothing. From 6dfb558c8b411d03044b719defbada4dd62b583a Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 17:58:50 +0300 Subject: [PATCH 20/27] docs(deck): cut 41 slides to 10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructured rather than trimmed. The deck now follows the argument's spine: problem, who trusts whom, the billing unit, admission, credit, settlement, economics, deployment, the bugs, results. Cut whole slides that were interesting but not load-bearing — the SWAP borrow/drop tables, the endpoint list, the soft-mode rollout, dedup-at-zero, the per-payee cheque gotcha — and folded the rest into the slide that needed them. Every figure that carried an argument survived; the ones that only carried detail did not. Also adds per-slide type auto-fit. The stage is a fixed 16:9 box with overflow hidden, and slides now vary a lot in density, so a dense one could silently lose its last line. Type scales through a --fit multiplier that the JS lowers until the slide fits; slides with room to spare are untouched. The 41-slide version is in this branch's history if the long form is ever wanted. Co-Authored-By: Claude Opus 5 (1M context) --- docs/deck/shell.css.html | 14 +- docs/deck/shell.js.html | 14 + docs/pusher-incentives-slides.md | 550 +++++-------------------------- 3 files changed, 105 insertions(+), 473 deletions(-) diff --git a/docs/deck/shell.css.html b/docs/deck/shell.css.html index 50cca9f..a276266 100644 --- a/docs/deck/shell.css.html +++ b/docs/deck/shell.css.html @@ -76,7 +76,7 @@ padding: 5.2cqw 6cqw 6.4cqw; display: none; flex-direction: column; - gap: 2.2cqh; + gap: calc(2.2cqh * var(--fit, 1)); overflow: hidden; } .slide.on { display: flex; } @@ -120,7 +120,7 @@ text-wrap: balance; } h2 { - font-size: 3.1cqw; + font-size: calc(3.1cqw * var(--fit, 1)); line-height: 1.1; margin: 0; font-weight: 600; @@ -131,7 +131,7 @@ h2 .q { color: var(--accent); } h2 .hz { color: var(--hazard); } - .body { font-size: 1.82cqw; line-height: 1.5; } + .body { font-size: calc(1.82cqw * var(--fit, 1)); line-height: 1.5; } p { margin: 0; max-width: 62ch; } p + p { margin-top: 1.2cqh; } @@ -139,7 +139,7 @@ .lede { font-size: 2.15cqw; line-height: 1.4; color: var(--ink); max-width: 54ch; } .sub { color: var(--ink-2); } - ul { margin: 0; padding-left: 1.4em; display: flex; flex-direction: column; gap: 1.1cqh; max-width: 66ch; } + ul { margin: 0; padding-left: 1.4em; display: flex; flex-direction: column; gap: calc(1.1cqh * var(--fit, 1)); max-width: 66ch; } li::marker { color: var(--accent); } strong { font-weight: 600; } @@ -157,7 +157,7 @@ .claim { border-left: 3px solid var(--accent); padding: .3cqw 0 .3cqw 1.6cqw; - font-size: 2.05cqw; + font-size: calc(2.05cqw * var(--fit, 1)); line-height: 1.38; max-width: 58ch; } @@ -168,7 +168,7 @@ table { border-collapse: collapse; font-family: var(--mono); - font-size: 1.42cqw; + font-size: calc(1.42cqw * var(--fit, 1)); width: 100%; font-variant-numeric: tabular-nums; } @@ -194,7 +194,7 @@ pre { font-family: var(--mono); - font-size: 1.42cqw; + font-size: calc(1.42cqw * var(--fit, 1)); line-height: 1.5; margin: 0; padding: 1.2cqw 1.5cqw; diff --git a/docs/deck/shell.js.html b/docs/deck/shell.js.html index 13688ca..ac7c398 100644 --- a/docs/deck/shell.js.html +++ b/docs/deck/shell.js.html @@ -13,6 +13,18 @@ fill.style.width = ((i + 1) / slides.length * 100) + '%'; counter.textContent = (i + 1) + ' / ' + slides.length; if (location.hash !== '#' + (i + 1)) history.replaceState(null, '', '#' + (i + 1)); + fit(slides[i]); + } + + // Slides carry very different amounts of content, and the stage is a fixed + // 16:9 box that clips. Rather than let a dense slide silently lose its last + // line, shrink that slide's type until it fits. Slides with room to spare + // are untouched, so the designed sizes hold wherever they can. + function fit(s) { + s.style.setProperty('--fit', '1'); + for (let k = 1; k > 0.62 && s.scrollHeight > s.clientHeight + 1; k -= 0.02) { + s.style.setProperty('--fit', String(k)); + } } document.getElementById('next').addEventListener('click', () => show(i + 1)); @@ -39,6 +51,8 @@ x0 = null; }, {passive: true}); + addEventListener('resize', () => fit(slides[i])); + const start = parseInt(location.hash.slice(1), 10); show(Number.isFinite(start) && start > 0 ? start - 1 : 0); \ No newline at end of file diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 26c4edf..1ee1743 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -17,566 +17,184 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' ## An incentive layer for hoverfly pushers, reusing parts of SWAP -*docs/pusher-incentives.md · Stages 0–1 shipped · one metered lane in production* +*docs/pusher-incentives.md · Stages 0–1 shipped · one paid relay in production* --- - + # The relay pays for traffic it did not cause -In a **native** upload your own machine opens the pushsync streams, and bee debits *you*: +When you upload directly, bee charges **you** for every chunk. Put a relay in the middle and that cost moves to the relay — it is the peer bee sees, so it is the peer bee bills. The client pays only for postage. -``` -price(po) = (32 − po) × 10 000 accounting units -``` - -Put a relay in the middle and that debt moves **wholesale to the relay** — it is the peer bee sees, so it is the peer bee charges. The browser client that caused the traffic pays **nothing but postage**. - - - -> Booked today as accepted risk: *“worst case = the platform's free egress for the month burned, $0 lost.”* - ---- - - - -# What changes - -| | open (today) | metered | +| | free (today) | paid | |---|---|---| -| client → relay | nothing | 4.8e8 PLUR per KiB of body sent | -| relay → bee | free pseudosettle | unchanged — free pseudosettle | -| relay's egress | unrecovered | recovered above the cashout threshold | +| client → relay | nothing | 4.8e8 PLUR per KiB sent | +| relay → bee | nothing | **unchanged** — still nothing | +| relay's bandwidth | unrecovered | recovered above the cashout threshold | -The relay still pays bee nothing, because credit was never what limited it. Bee hands out enough free credit for **~2,400 chunks a second** across a 128-connection pool, and the relay only manages **~150**. It is limited by connections and round trips, so buying credit would buy nothing. +The relay still pays bee nothing, because credit was never what limited it: bee allows about **2,400 chunks a second** and the relay manages **~150**. It runs out of connections long before it runs out of credit. -> The recovered amount is small. A relay pushing 100 GB of egress a month moves ~27 GiB of payload, which at $0.02/GiB is **$0.54**. +> The sums are small. A relay pushing 100 GB a month moves ~27 GiB of payload, which at $0.02/GiB is **$0.54**. This works on repeat and bulk traffic, or not at all. --- - - -# Theory - ---- - - + # Who has to trust whom -> The client chose its relay. The relay did not choose its client. - -A relay is just an HTTP service. Anyone can run one, and there is no registry or list to get onto. The asymmetry comes from **the client picking**: before sending anything it checks the relay's signed quote and remembers who that relay is. The relay gets no such choice — a client is whoever shows up. - -So every defence *the relay* has points at the client: *a client cannot obtain service without paying, and cannot lie about what it owes.* - ---- - - - -# What protects the client, then +A relay is just an HTTP service. Anyone can run one, and there is no registry or list to get onto. The asymmetry is that **the client picks**: before sending anything it checks the relay's signed quote and remembers who that relay is. The relay gets no such choice — a client is whoever shows up. -Not cryptography. Four bounds, none of which need the relay to be trustworthy: +> So every defence the relay has points at the client: it cannot get service without paying, and cannot lie about what it owes. -- **It works out the bill itself.** The client adds up the bytes it sent. If the relay reports a bigger number, the client sees it immediately — and knows the relay is the one that is wrong. -- **The price is fixed in advance.** It arrives in a quote the relay signs, and the same signed quote comes back with every refusal — so the price cannot move mid-upload. -- **The most it can lose is one credit limit** — about $0.0024 at the maximum, and far less on a small batch, which gets a thousandth of whatever it is still worth. -- **It watches whether chunks actually arrive.** A relay that takes bytes and delivers badly gets sent less work, using the running average that already picks between relays today. +The client's protections are a different kind of thing — arithmetic and limits, not cryptography: -> That last bound had a hole in it. When a client cannot pay a refusal, it asks the relay what it owes and believes the answer. That used to be capped only by the chequebook balance, so any relay could ask for everything. It is now capped by the limit in the quote the relay signed: the credit it granted is the most it can claim. +- **It works out the bill itself**, so a relay reporting a bigger number is caught immediately +- **The price is fixed in advance**, in a quote the relay signed +- **The most it can lose is one credit limit** — about $0.0024, and less on a small batch +- **It watches whether chunks arrive**, and sends less work to relays that deliver badly --- - + -# What the previous design got wrong - - - -> The design was building **two-sided cryptographic verification** for a **one-sided trust relationship.** - -About half of it protected the client from the relay — from relays we run ourselves. That cost three real things: - -- **The bill was built on someone else's signature** — a receipt from a bee node. Those are easy to fake, so every one had to be checked against the on-chain list of staked nodes. -- **The bill was a list, not a number**, and nothing limited how long that list could get. -- **One unmeasured number could have killed the project**: how many receipt signers are actually staked. Nobody had checked. - ---- - - - -# What we borrow from SWAP - -| Borrowed | Why | -|---|---| -| ERC20SimpleSwap + canonical factory | Audited, deployed, in production. Nothing to write. | -| EIP-712 cheque | Cheque(chequebook, beneficiary, cumulativePayout) | -| Cumulative-payout monotonicity | Loss-tolerant *and* replay-proof | -| Funding check | …but liquidBalanceFor(us), not balance() — bee's version is unsound | -| Reservation vs. concurrent issuance | Needed on *both* sides | -| Payee-only role | The beneficiary is a plain EOA. A payee needs no contract. | - ---- - - - -# What we drop - -| Dropped | Reason | -|---|---| -| swap libp2p stream, Handshake, EmitCheque | We're on HTTP already | -| priceoracle, exchange, deduction | Relay quotes PLUR directly | -| accounting-unit indirection | One unit. No conversion. | -| ghost balances, tolerance, trust ramp | No analogue in request/response | -| StakeRegistry snapshot | Nothing left to anchor — see next | - -> Both sides here are hoverfly. Bee is not involved in the payment at all — we reuse SWAP's *contracts and cheque format*, not the protocol it speaks over the network. - ---- - - - -# The account is the batch owner - -Relays already know who is uploading. Every chunk carries a stamp, and its signature has to match the address the postage contract lists as the batch's owner. - -> Account = the batch-owner EOA. A cheque is valid for it **iff** its chequebook's on-chain `issuer()` is the same EOA. Credit is keyed one level finer — on the **batch**. - -So there is nothing to add: no logins, no sign-up, no extra message. In a browser the key that owns the batch is already loaded, so cheques get signed with **no wallet pop-ups at all**. - ---- - - - -# Bill bytes admitted +# Bill bytes, not receipts ``` -owed = (kib_admitted − kib_dedup) × price_plur_per_kib +owed = (KiB admitted − KiB already cached) × price per KiB ``` -> The client cannot lie about it — the client produced the bytes and the relay counted them. - -- Nothing to forge — no outside signature is part of the bill -- Nothing to argue about — no chain lookup, and nothing the client takes on the relay's word - -Both numbers are known to both parties *before* any push work happens. - ---- - - - -# What that replaced - -The earlier draft billed per **verified pushsync receipt**. To make a *third party's* signature into a billing input it needed: - -- check every receipt signer against the on-chain staking list, or faking one is trivial -- make the bill a list instead of a number, or the same receipt gets submitted twice -- spot-check by fetching chunks back, to catch receipts for work never done -- share one piece of code that decides whether a receipt counts — and if the two copies ever disagree, nobody can settle it - -> Billing bytes instead of receipts removed all five, and every attack on them. - ---- - - +> The client cannot lie about it, because the client produced the bytes and the relay counted them. -# Bytes, not successful pushes +An earlier design billed per delivery receipt — a signature from a **third party**. Making that safe needed five mechanisms: check every signer against the on-chain staking list, turn the bill into a list so receipts cannot be reused, spot-check by fetching chunks back, sweep the staking logs on both sides, and share one piece of receipt-checking code that must never disagree between them. -Bytes are what actually costs the relay money. It sends every chunk to three peers at once and retries the ones that go nowhere, and that traffic goes out whether or not the chunk ends up stored. Charging only for successes would leave the relay paying for every failure. - -| Concern | Mechanism | -|---|---| -| Relay recovers what it spends | Bill attempts | -| Client protects itself from a lane that spends without succeeding | Deweight it in the scheduler — already exists, already works | +Changing the unit removed all five, and every attack on them. Bytes are also what the relay actually spends money on: it sends each chunk to three peers at once and retries failures, and that traffic goes out whether or not the chunk sticks. --- - + -# Dedup hits are billed at zero +# Getting in: a permission slip -If the relay already pushed the same chunk moments ago, it does no work the second time, so those bytes come off the bill. This is the **only** part of the bill that rests on the relay's own word. - -> It is safe for a simple reason: the claim only ever **lowers** the bill. A relay gains nothing by lying, and a client that disputes it is arguing in its own favour. - ---- - - - -# Cumulative cheques - -- **Losing one costs nothing.** Every cheque is a running total, so if a payment fails the next one covers it anyway. No retry logic needed. -- **Old cheques are worthless.** Each must be larger than the last, so sending an old one again pays nothing. -- **Gas is paid once per customer, not once per cheque.** The relay only ever cashes the newest total, so every earlier cheque costs nothing to collect. +The relay has to decide whether to accept an upload **before** it has read it — and at that moment it does not yet know whose account to check. Working that out means checking a stamp and looking up a batch owner on chain. +> Doing all that first would mean up to 512 signature recoveries and a chain lookup **before the relay can answer at all** — cheap for an attacker to trigger, expensive to serve. -> A cheque per chunk was rejected: 137 bytes and a signature on every 4 KiB, and — worse — running totals have to go in order, so chunks would have to be sent one at a time. That would remove the parallel uploads the scheduler relies on for speed. - ---- - - - -# Cheques are per payee, not per relay +So the chain lookups happen **once**, when a slip is issued, and the credit limit is sealed into it. After that an upload needs no chain lookups: check the slip, check the signature, set the money aside — and refuse before reading the body if that would go over the limit. -A running total belongs to a **payee**, but a relay is a **URL**. One operator running four relay URLs that all pay into the same account is the obvious way to deploy. - -``` -lane 1 issues cumulative 10 -lane 2, counting from its own zero, issues 8 -relay applies ErrChequeNotIncreasing → rejected, forever -``` - -> So the client must track running totals per *payee*, not per relay URL. Spotting that two relays share one is free — the payee's address is in the signed quote, before the first upload. +The slip must be signed by the batch owner. That is what stops an attacker replaying **someone else's public stamps** and having the bill land on them. --- - - -# Payment happens outside the upload + -| Endpoint | Shape | -|---|---| -| GET /v1/status | signed *payment* block; mode: open \| metered | -| GET /v1/challenge | {nonce, expires_ms, max_outstanding_plur} — stateless MAC | -| GET /v1/account | authenticated. owed, reserved, outstanding, kib_admitted … | -| POST /v1/pay | body = SignedCheque JSON | -| POST /v1/push | 402 Payment Required when over cap | +# How much a client may owe -The upload format is **unchanged**. Payment stays off the upload path, so a payment problem never breaks an upload. The balance endpoint needs a login, because otherwise anyone could look up how much any account has uploaded. - ---- - - - -# Why a challenge at all - -The tempting answer is that refusing is easy, because the relay picks its response code before it looks at any chunk. That is true, and it is exactly the **problem** — at that moment it does not yet know **whose account to check**. - -Working out who is paying means checking the stamp and looking up the batch owner, and both happen later — after the response has already gone out. - - - -> Doing those checks first means up to 512 signature recoveries (~40 ms) and a chain lookup **before the relay can answer at all** — cheap for an attacker to trigger, expensive for the relay to serve. Exactly the shape the design spends a section trying to avoid. - ---- - - - -# The permission slip proves the checks were already done +It is tempting to assume a batch owner has real money at stake. They need not: the relay only checks that a batch is alive, and the cheapest live batch costs a fraction of a cent. So the limit is tied to what the batch is actually worth: ``` -GET /v1/challenge?account=A&batch=B - → resolve standing(B) (cached per batch, TTL) - → require owner(B) == A, else 403 - → cap = credit_line(standing(B)) - → nonce = HMAC(relay_secret, preimage(A, B, origin, expiry, cap)) +credit limit = min(batch's remaining value ÷ 1000, 6.22e13 PLUR) ``` -The chain lookups happen once, when the slip is issued, and the credit limit is sealed into it. After that an upload needs **no chain lookups at all**: check the slip, check it was issued for this relay, check the signature, set the money aside — and if that would go over the limit, **refuse before reading the upload**. - -The relay keeps no list of the slips it has issued, so handing them out for free cannot fill up its memory. - ---- - - - -# Two ways to make the check do nothing - -**1. Glue the fields together carelessly and two different inputs look identical.** The hostname varies in length, so `("host.a","bc")` and `("host.ab","c")` run together into the same bytes — one slip that works for two different hostnames. - -**2. The relay must know its own name from its config, not from the request.** The obvious version compares the slip against the `Host` header — which does nothing, because the attacker sends both. - - - -> An attacker reusing a victim's signature at relay B just sends `Host: relay-b.example`. The check passes, and the attack works again — **while the code looks like it is preventing it.** - ---- - - +> An attacker gets back **a thousandth of what they funded**, whatever size batch they buy. What is fixed is the ratio, so there is no cheap corner to aim at. -# Credit scales with what the batch is worth - -It is tempting to assume a batch owner has real money at stake. They do not have to: the relay only checks that a batch is *alive*, and the cheapest batch the contract accepts costs a fraction of a cent. With one flat credit limit, an attacker gets back roughly what they paid. - -``` -max_outstanding(A,B) = min(remaining_value_plur(B) ÷ credit_ratio, - max_outstanding_plur) credit_ratio = 1000 -``` - -> An attacker now gets **a thousandth of what they funded, whatever they buy.** There is no cheap way in, because what is fixed is the *ratio* — not an amount someone can undercut. - ---- - - - -# The rule the three thresholds must satisfy - -> min_cheque_plur ≤ settle_every_plur < max_outstanding_plur - -| Parameter | PLUR | In payload | +| Setting | PLUR | In payload | |---|---:|---:| -| price_plur_per_kib | 4.8e8 | 1 KiB | -| min_cheque_plur | 3.9e12 | ~8 MiB | -| settle_every_plur | 1.56e13 | ~32 MiB | -| max_outstanding_plur | 6.22e13 | ~127 MiB | +| price per KiB | 4.8e8 | 1 KiB | +| smallest cheque accepted | 3.9e12 | ~8 MiB | +| pay when you reach | 1.56e13 | ~32 MiB | +| hard ceiling | 6.22e13 | ~127 MiB | -An early draft set the minimum cheque **87× larger** than the point where you are meant to pay. Every account would have jammed: run up a bill, try to pay, get told the cheque is too small, keep running it up, get refused — and the only cheque that would clear is 21× what you owe. No way out. +Those three must stay in that order. Otherwise an account can owe less than the smallest cheque it is allowed to write, and then it can never pay. --- - + -# Counting is not enough when uploads overlap +# Cheques are running totals -Just adding up what is owed is **not** enough. Uploads run in parallel on purpose, so several can each check the balance before any of them has been charged. Even a well-behaved client sending the 8 the relay itself recommends will blow past the limit. +Each cheque says "you have now paid me *this much in total*", not "here is a payment". Three things follow: -The fix: set the money aside up front, based on the size the client declared, and give back whatever was not used when the upload finishes. +- **Losing one costs nothing.** The next cheque covers it anyway. No retry logic. +- **Old cheques are worthless.** Each must exceed the last, so replaying one pays zero. +- **Gas is paid once per customer**, not per cheque — the relay only ever cashes the newest total. - - -> But the set-aside amounts must not be saved to disk. Each one belongs to an upload in progress, and no upload survives a restart — so nothing would ever release it. Save what is owed and the running total; start the set-aside amounts at zero. +Uploads run in parallel, so simply adding up what is owed is not enough: several can each check the balance before any of them is charged. The relay sets the money aside up front and refunds the unused part. ---- - - - -# The cost basis - -| Per 4 KiB chunk relayed | | -|---|---:| -| Delivery on the wire | ≈ 4.4 KiB | -| Sent to three peers at once | × 3 | -| Retries for chunks that go nowhere | × 1.15 | -| **Egress per chunk** | ≈ 15 KiB | -| **Egress per GiB of payload** | ≈ 3.7 GiB | - -> $0.02 per GiB works out to `4.8 × 10⁸` PLUR per KiB. The rate is flat on purpose: anything more complicated puts a per-chunk number back into the bill, and that is one more thing the two sides can disagree about. + +> But the set-aside amounts must never be written to disk. Each one belongs to an upload in progress, and no upload survives a restart — so nothing would ever release them. --- - + -# On per-GB-billed hosts, metering loses money +# What it costs, and who pays for themselves -At 3.7 GiB of real egress per GiB of payload: +Every 4 KiB chunk costs about **15 KiB** of real bandwidth — three peers at once, plus retries — so a GiB of payload costs ~3.7 GiB of traffic. At $0.02/GiB that is a real margin on flat-rate hosting and a loss anywhere else. -| | Per GiB of payload | +| | per GiB of payload | |---|---:| | AWS egress at $0.09/GB | −$0.33 | -| Revenue at $0.02/GiB | +$0.02 | -| **Net** | **−$0.31** | - -> Charging only covers costs on hosts with **flat-rate or included bandwidth**. On per-GB hosting, run it free and absorb the quota instead. That is the same kind of host already needed for a disk that survives restarts, so both requirements point at the same machines. +| revenue at $0.02/GiB | +$0.02 | +| **net** | **−$0.31** | ---- - - - -# Revenue per account vs. cashout gas - -Issuing a cheque sends no transaction. Only cashing out touches the chain: ≈ **$0.0005** on Gnosis. - -| Account's lifetime traffic | Revenue | Gas | Gas as % | +| Customer's lifetime traffic | Revenue | Gas | Gas as % | |---|---:|---:|---:| | 71 MB — one browser upload | $0.0014 | $0.0005 | 36 % | | 5 GiB — the cashout threshold | $0.10 | $0.0005 | 0.5 % | -> Gas is paid once per **customer**, not per cheque, so the ratio gets better every time someone comes back. Anyone who never reaches the threshold is written off. A relay whose users all upload once and leave should not charge at all. +> A relay on per-GB bandwidth, or one whose users upload once and never return, should stay free. --- - - -# Three attacks worth knowing about + -- **Someone else's stamps, billed to them** *(new)* — stamps are public, and a relay has a copy of every one it has forwarded. Send a victim's stamps to a paid relay and the victim's account picks up the bill, at no cost to the attacker. Fixed: the client must sign the challenge with the batch owner's key, and every chunk in a request must belong to the batch named in it. -- **The customer can empty the chequebook after you accept** *(inherited)* — the owner can withdraw at any time, so "this cheque is funded" is true when you take it, not when you cash it. Bee has exactly the same exposure. -- **A relay that forgets serves for free forever** *(new)* — if the ledger does not survive a restart, one signature buys unlimited service. So a paid relay needs a real disk. That is a requirement, not advice. +# Paying is optional, per relay ---- - - - -# Practice - ---- - - - -# Paying is optional, per lane - -| Relay mode | Client has a chequebook | Client does not | +| Relay | Client can pay | Client cannot | |---|---|---| -| open | used, nothing billed | used, nothing billed | -| metered, soft | used, billed, settles | used, billed, served anyway | -| metered, hard | used, billed, settles | lane retired at startup | - -> A paid relay rejects an upload with no permission slip, and that rejection counts against the relay's health score — only a genuine "you owe too much" is excused. So sending work there anyway burns one retry on every single chunk, to rediscover something the relay already said up front. - ---- - - - -# What soft mode does and does not drop +| free | used, nothing billed | used, nothing billed | +| paid, soft | used, billed, settles | used, billed, served anyway | +| paid, enforced | used, billed, settles | **dropped at startup** | -Soft mode meters, reports and accepts cheques, but **never answers 402**. +A paid relay rejects an upload that arrives with no permission slip, and that rejection counts against the relay's health score — only a genuine "you owe too much" is excused. Sending work there anyway burns one retry on every chunk, to rediscover what the relay already said up front. -**It still requires the permission slip.** An earlier draft suggested serving requests that arrive without one — which would let anyone skip paying by leaving out a header. - -> What soft mode drops is enforcement of the cap, **not authentication**. - -A relay flipping to `--meter` therefore does break clients that predate the protocol. Acceptable: the only dApp using these lanes ships alongside them. +> Four free relays run on hosts whose disks are wiped on restart, so they have to stay free — a relay that forgets what it is owed serves for free forever. One paid relay enforces. The browser app skips it automatically, because it can sign chunks but not cheques. --- - - -# Six bugs only a running relay could find - -None is reachable from a single upload against a fresh relay. All six survived the full test suite **and** the Stage 1 round-trip. Reaching them needed, simultaneously: + -- a relay that **remembers what you owe between runs**, so debt carries over -- several uploads in flight at the same time, not one after another -- a batch **used up far enough that the credit limit is what stops you**, rather than anything else - -> The one to learn from is §17.3. It is not a coding mistake — it is a **rule checked against the wrong number**, and it only shows up once a real batch has been worn down below about 0.39 BZZ. - ---- +# Six bugs no test suite reached - - -# The six bugs +All six survived the full test suite and a working end-to-end round trip. Reaching them needed three things **at once**: a relay that remembers debt between runs, several uploads in flight at the same time, and a batch worn down far enough that the credit limit is what stops you. | § | Bug | -|---:|---| -| 17.1 | Debt the relay carried **across sessions** could not be paid | -| 17.2 | The headroom guard admitted a *frame*, then sent a *batch* | -| 17.3 | The §10.1 invariant checked against the wrong quantity | -| 17.4 | A lane refused for **bytes in flight** was parked for good | -| 17.5 | A broken response stream made **every later cheque bounce** | -| 17.6 | The first POST of a run was sized **before the debt was known** | - ---- - - - -# Debt carried over, and could not be paid - -Because there is a minimum cheque size, every upload ends owing a little too small to pay. The relay is **right** to keep counting it — writing it off would make staying under the minimum a way to be served for free. - -But the client forgets when it exits. The next run starts thinking it owes **nothing**, while the relay's total keeps growing. Eventually the very first upload is refused — and the client **cannot pay**, because it writes cheques from its own figure, which is zero. - - - -> Observed live: a second upload failing **151/151** against a relay carrying 290,400,000,000 PLUR. - ---- - - - -# Three ways to get the fix wrong - -- **Ask what you owe. Don't read it off the refusal.** The refusal already counts the request it is turning down, so paying that number overpays by exactly one request — and the next cheque bounces for being too big. -- **Don't count bytes still on the wire.** The client is already tracking those, so counting them again bills them twice. Guessing low is safe and fixes itself on the next round; guessing high gets the cheque rejected. -- **Cap it at the limit the relay signed up to.** Not the batch's current limit — that falls as the batch is used up, so it can drop below a bill you honestly ran up, and refusing to pay that keeps you stuck. And not the chequebook balance either: that lets any relay you point at ask for everything you have. - ---- - - - -# The invariant was checked against the wrong quantity - -The check compared the three thresholds against the highest limit the relay ever grants. But the limit that actually applies is **per batch**, and it is usually much smaller: - -``` -min(remaining_value / credit_ratio, ceiling) -``` - -Once a batch is worth less than about 0.39 BZZ, the minimum cheque is **bigger than anything that account is allowed to owe**. It runs up to its limit and can never write a cheque the relay will take. - - - -> The account is refused from then on. Nothing is broken and nothing logs an error — it just stops working. Both sides now compare against the limit that actually applies. - ---- - - - -# After all six fixes - -| Payload | Frames acked | 402s | Stuck | Rejected cheques | -|---|---:|---:|---:|---:| -| 128 KiB | 43/43 | 0 | 0 | 0 | -| 512 KiB | 151/151 | 0 | 0 | 0 | -| 1 MiB | 290/290 | 2 | 0 | 0 | -| 4 MiB | 1122/1122 | 4 | 0 | 0 | +|---|---| +| 17.1 | Debt carried across runs could not be paid — each run starts believing it owes nothing | +| 17.2 | The headroom check measured one chunk, then sent a whole batch | +| 17.3 | A rule checked against the wrong number | +| 17.4 | A relay refused for bytes still in flight was parked forever | +| 17.5 | One broken response stream made every later cheque bounce | +| 17.6 | The first upload of a run was sized before the debt was known | -> The remaining 402s are the *intended* kind: the line genuinely fills, the client pays or waits, the lane resumes. +> The one to learn from is **17.3**. Not a coding mistake — a rule compared against the highest limit the relay ever grants, when the limit that applies is per batch. Below ~0.39 BZZ of batch value the smallest allowed cheque exceeds everything the account may owe. It is refused from then on, and nothing logs an error. --- - + -# Over public HTTPS, with §17.6 in place +# It works, and here is what is still open -| Run | Payload | Frames acked | 402s | Rejected cheques | -|---:|---|---:|---:|---:| +| Run | Payload | Chunks delivered | Refusals | Rejected cheques | +|---|---|---:|---:|---:| | 1 | 2 MiB | 567/567 | 0 | 0 | | 2 | 2 MiB | 567/567 | 0 | 0 | | 3 | 2 MiB | 567/567 | 0 | 0 | -Each run settles to `owed: 0` on the relay, so the next carries nothing. - -> That is how it is meant to run: **a refusal is the fallback, not the normal path.** - ---- - - - -# It settles on-chain - -- The relay counts what you owe. You sign a cheque for the running total. The relay checks it and marks you paid. -- The relay is holding a cheque for **21,366,720,000,000 PLUR** right now. -- Cashing it happens on a **different machine**. Only the payee can cash a cheque, and the relay must never hold that key. -- It worked: the transaction succeeded and used 75,378 gas, against a 300,000 budget. - -> The relay never holds a key that can spend anything. It only needs the payee's **address**. So charging for relay does not make a relay box worth breaking into. - ---- - - - -# What is actually deployed - -**Four free relays** — on hosts whose disks are wiped on restart, so they have to stay free — plus **one paid relay** that enforces payment. - -The browser app lists all five and **skips the paid one automatically**, because it can sign chunks but not cheques. A command-line client with a chequebook uses all five. - -> Payment is a property of a relay, not of the fleet. - ---- - - - -# Net effect on the design - -- **Removed:** five mechanisms and every attack on them, just by billing bytes instead of receipts — the staking check, the list-shaped bill, the spot-check audit, the log sweep on both sides, and the shared receipt-checking code. -- **Added:** three, all cheap — one signature check when a client asks permission, one chain lookup per batch every half hour, and one amount set aside per upload. -- **Now guaranteed rather than hoped for:** an attacker gets credit worth a thousandth of what they actually funded, whatever size batch they buy; credit shrinks by itself as a batch is used up, with no expiry code to get wrong; and the relay holds no key that can move money. - -> Found by running it: six bugs no test suite reached — five needed a ledger that outlives the client. - ---- - - - -# Still open - -- **We don't yet know if this pays for itself.** The number to watch is how many accounts ever build up enough debt to be worth cashing. If it stays at zero, metering funds nothing. -- **One chequebook per batch owner.** If you upload using batches owned by different addresses, each one needs its own chequebook. -- **The empty-the-chequebook problem has no fix.** Bee has it too — it comes from how the chequebook contract is deployed, not from anything here. -- **Nothing stops a relay taking your money and dropping chunks.** You lose at most one credit limit and then stop using it — that bounds the damage, but it is not a guarantee of service. +Each run settles to zero owed, so the next carries nothing. Cheques are cashed on a **different machine** — only the payee can cash one, and the relay must never hold that key. The live cashout used 75,378 gas against a 300,000 budget. -Full design: `docs/pusher-incentives.md` — §8 billing unit, §10.3 Sybil bound, §17 the bugs. +- **We do not yet know if this pays for itself.** The number to watch is how many customers ever reach the cashout threshold. +- **Nothing stops a relay taking payment and dropping chunks.** You lose at most one credit limit and stop using it — that bounds the damage, it does not guarantee service. +- **A customer can empty their chequebook after you accept a cheque.** Bee has the same problem; it comes from how the contract is deployed. From a3725a2d3b0e80971aa4926948136ff355e440a6 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Tue, 11 Aug 2026 18:30:20 +0300 Subject: [PATCH 21/27] docs(deck): sparser slides, and fix the auto-fit that was letting content crop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems, one of which was mine. The cropping: .slide is a column flexbox, and flex items shrink to fit by default. So a slide with too much content had its children squeezed rather than overflowing, which made scrollHeight equal clientHeight — and the auto-fit concluded everything fitted while the content was visibly crushed and clipped. Children are now flex: none, so the measurement is honest, and fit() also checks each child's own overflow. It re-measures after layout settles and again once fonts load, since the first pass can run against fallback metrics. The text: slides were carrying a paragraph *and* a table *and* another paragraph *and* a callout. Now the budget is a heading, one block, one callout — about 64 words a slide, down from ~110. Dropped the settings table, the second economics table, the per-bug conditions paragraph, and a good deal of connective prose. The header comment records the budget so the next edit does not quietly undo it. Also scales the table-header and paragraph-spacing sizes with --fit; they were staying large while everything around them shrank. Co-Authored-By: Claude Opus 5 (1M context) --- docs/deck/shell.css.html | 9 ++- docs/deck/shell.js.html | 14 +++- docs/pusher-incentives-slides.md | 123 ++++++++++++------------------- 3 files changed, 66 insertions(+), 80 deletions(-) diff --git a/docs/deck/shell.css.html b/docs/deck/shell.css.html index a276266..1d46cc1 100644 --- a/docs/deck/shell.css.html +++ b/docs/deck/shell.css.html @@ -80,6 +80,11 @@ overflow: hidden; } .slide.on { display: flex; } + /* Children must not shrink. A column flexbox squeezes its items to fit by + default, which makes the slide's scrollHeight equal its clientHeight even + when the content is visibly crushed — and the auto-fit below then + concludes, wrongly, that everything fits. */ + .slide > * { flex: none; } /* ── Slide furniture ───────────────────────────────────────────────── */ .eyebrow { @@ -134,7 +139,7 @@ .body { font-size: calc(1.82cqw * var(--fit, 1)); line-height: 1.5; } p { margin: 0; max-width: 62ch; } - p + p { margin-top: 1.2cqh; } + p + p { margin-top: calc(1.2cqh * var(--fit, 1)); } .lede { font-size: 2.15cqw; line-height: 1.4; color: var(--ink); max-width: 54ch; } .sub { color: var(--ink-2); } @@ -183,7 +188,7 @@ font-weight: 400; text-transform: uppercase; letter-spacing: .1em; - font-size: 1.14cqw; + font-size: calc(1.14cqw * var(--fit, 1)); border-bottom: 1px solid var(--rule); } /* md2html emits GFM column alignment as an attribute, not a class. */ diff --git a/docs/deck/shell.js.html b/docs/deck/shell.js.html index ac7c398..2789325 100644 --- a/docs/deck/shell.js.html +++ b/docs/deck/shell.js.html @@ -21,9 +21,15 @@ // line, shrink that slide's type until it fits. Slides with room to spare // are untouched, so the designed sizes hold wherever they can. function fit(s) { + if (!s) return; s.style.setProperty('--fit', '1'); - for (let k = 1; k > 0.62 && s.scrollHeight > s.clientHeight + 1; k -= 0.02) { - s.style.setProperty('--fit', String(k)); + // Measure the tallest overflow, not just the slide's own: a wide table or a + // long line can spill out of its own box while the slide still measures as + // fitting. + const over = () => s.scrollHeight > s.clientHeight + 1 + || Array.from(s.children).some(c => c.scrollHeight > c.clientHeight + 1); + for (let k = 1; k > 0.55 && over(); k -= 0.02) { + s.style.setProperty('--fit', k.toFixed(2)); } } @@ -55,4 +61,8 @@ const start = parseInt(location.hash.slice(1), 10); show(Number.isFinite(start) && start > 0 ? start - 1 : 0); + // Re-measure once layout has settled, and again when the fonts land — the + // first pass can run against fallback metrics and under-shrink. + requestAnimationFrame(() => fit(slides[i])); + if (document.fonts && document.fonts.ready) document.fonts.ready.then(() => fit(slides[i])); \ No newline at end of file diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 1ee1743..d717028 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -9,7 +9,11 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' self-contained HTML deck is built with: python3 docs/deck/build.py docs/pusher-incentives-slides.md -o deck.html Directives (, part:, eyebrow:, hazard) are documented - in docs/deck/build.py. --> + in docs/deck/build.py. + + Keep slides sparse: a heading, ONE block (a short table, up to four + bullets, or two short paragraphs), and ONE callout. The detail lives in + docs/pusher-incentives.md. --> @@ -17,7 +21,7 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' ## An incentive layer for hoverfly pushers, reusing parts of SWAP -*docs/pusher-incentives.md · Stages 0–1 shipped · one paid relay in production* +*docs/pusher-incentives.md · one paid relay in production* --- @@ -25,17 +29,14 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' # The relay pays for traffic it did not cause -When you upload directly, bee charges **you** for every chunk. Put a relay in the middle and that cost moves to the relay — it is the peer bee sees, so it is the peer bee bills. The client pays only for postage. +Upload directly and bee bills you. Put a relay in the middle and bee bills the relay — it is the peer bee sees. The client pays only postage. -| | free (today) | paid | +| | free | paid | |---|---|---| -| client → relay | nothing | 4.8e8 PLUR per KiB sent | -| relay → bee | nothing | **unchanged** — still nothing | -| relay's bandwidth | unrecovered | recovered above the cashout threshold | - -The relay still pays bee nothing, because credit was never what limited it: bee allows about **2,400 chunks a second** and the relay manages **~150**. It runs out of connections long before it runs out of credit. +| client → relay | nothing | 4.8e8 PLUR per KiB | +| relay → bee | nothing | still nothing | -> The sums are small. A relay pushing 100 GB a month moves ~27 GiB of payload, which at $0.02/GiB is **$0.54**. This works on repeat and bulk traffic, or not at all. +> 100 GB of traffic a month earns about **$0.54**. This works on repeat and bulk uploads, or not at all. --- @@ -43,16 +44,12 @@ The relay still pays bee nothing, because credit was never what limited it: bee # Who has to trust whom -A relay is just an HTTP service. Anyone can run one, and there is no registry or list to get onto. The asymmetry is that **the client picks**: before sending anything it checks the relay's signed quote and remembers who that relay is. The relay gets no such choice — a client is whoever shows up. - -> So every defence the relay has points at the client: it cannot get service without paying, and cannot lie about what it owes. +A relay is just an HTTP service — no registry, no list to get onto. -The client's protections are a different kind of thing — arithmetic and limits, not cryptography: +- The **client** picks one and checks its signed quote +- The **relay** gets whoever shows up -- **It works out the bill itself**, so a relay reporting a bigger number is caught immediately -- **The price is fixed in advance**, in a quote the relay signed -- **The most it can lose is one credit limit** — about $0.0024, and less on a small batch -- **It watches whether chunks arrive**, and sends less work to relays that deliver badly +> So the relay's defences point at the client. The client's protection is arithmetic: it counts its own bytes, and risks at most one credit limit — about **$0.0024**. --- @@ -61,14 +58,12 @@ The client's protections are a different kind of thing — arithmetic and limits # Bill bytes, not receipts ``` -owed = (KiB admitted − KiB already cached) × price per KiB +owed = KiB the relay accepted × price per KiB ``` -> The client cannot lie about it, because the client produced the bytes and the relay counted them. +> The client cannot lie about it. It produced the bytes; the relay counted them. -An earlier design billed per delivery receipt — a signature from a **third party**. Making that safe needed five mechanisms: check every signer against the on-chain staking list, turn the bill into a list so receipts cannot be reused, spot-check by fetching chunks back, sweep the staking logs on both sides, and share one piece of receipt-checking code that must never disagree between them. - -Changing the unit removed all five, and every attack on them. Bytes are also what the relay actually spends money on: it sends each chunk to three peers at once and retries failures, and that traffic goes out whether or not the chunk sticks. +Billing per delivery receipt meant trusting a **third party's signature**, which took five mechanisms to make safe. Changing the unit removed all five. --- @@ -76,14 +71,12 @@ Changing the unit removed all five, and every attack on them. Bytes are also wha # Getting in: a permission slip -The relay has to decide whether to accept an upload **before** it has read it — and at that moment it does not yet know whose account to check. Working that out means checking a stamp and looking up a batch owner on chain. +The relay must accept or refuse **before** reading an upload — but at that moment it does not know whose account to check. -> Doing all that first would mean up to 512 signature recoveries and a chain lookup **before the relay can answer at all** — cheap for an attacker to trigger, expensive to serve. - -So the chain lookups happen **once**, when a slip is issued, and the credit limit is sealed into it. After that an upload needs no chain lookups: check the slip, check the signature, set the money aside — and refuse before reading the body if that would go over the limit. +> Checking on every upload would mean 512 signature recoveries before it can answer at all. Cheap to attack, expensive to serve. -The slip must be signed by the batch owner. That is what stops an attacker replaying **someone else's public stamps** and having the bill land on them. +So the chain lookups happen **once**, when a slip is issued, and the credit limit is sealed into it. The batch owner signs it, so stolen stamps cannot bill their owner. --- @@ -91,22 +84,13 @@ The slip must be signed by the batch owner. That is what stops an attacker repla # How much a client may owe -It is tempting to assume a batch owner has real money at stake. They need not: the relay only checks that a batch is alive, and the cheapest live batch costs a fraction of a cent. So the limit is tied to what the batch is actually worth: +The cheapest live batch costs a fraction of a cent, so "owns a batch" proves nothing. The limit tracks what the batch is worth: ``` -credit limit = min(batch's remaining value ÷ 1000, 6.22e13 PLUR) +credit limit = batch's remaining value ÷ 1000 ``` -> An attacker gets back **a thousandth of what they funded**, whatever size batch they buy. What is fixed is the ratio, so there is no cheap corner to aim at. - -| Setting | PLUR | In payload | -|---|---:|---:| -| price per KiB | 4.8e8 | 1 KiB | -| smallest cheque accepted | 3.9e12 | ~8 MiB | -| pay when you reach | 1.56e13 | ~32 MiB | -| hard ceiling | 6.22e13 | ~127 MiB | - -Those three must stay in that order. Otherwise an account can owe less than the smallest cheque it is allowed to write, and then it can never pay. +> An attacker gets back **a thousandth of what they funded**, at any batch size. The ratio is what is fixed, so there is no cheap corner to aim at. --- @@ -114,37 +98,30 @@ Those three must stay in that order. Otherwise an account can owe less than the # Cheques are running totals -Each cheque says "you have now paid me *this much in total*", not "here is a payment". Three things follow: - -- **Losing one costs nothing.** The next cheque covers it anyway. No retry logic. -- **Old cheques are worthless.** Each must exceed the last, so replaying one pays zero. -- **Gas is paid once per customer**, not per cheque — the relay only ever cashes the newest total. +Each says "you have now paid me *this much in total*". -Uploads run in parallel, so simply adding up what is owed is not enough: several can each check the balance before any of them is charged. The relay sets the money aside up front and refunds the unused part. +- **Losing one costs nothing** — the next covers it +- **Old ones are worthless** — each must exceed the last +- **Gas is paid once per customer**, not per cheque -> But the set-aside amounts must never be written to disk. Each one belongs to an upload in progress, and no upload survives a restart — so nothing would ever release them. +> Money set aside for an upload in progress must never be written to disk. No upload survives a restart, so nothing would ever release it. --- -# What it costs, and who pays for themselves +# What it costs -Every 4 KiB chunk costs about **15 KiB** of real bandwidth — three peers at once, plus retries — so a GiB of payload costs ~3.7 GiB of traffic. At $0.02/GiB that is a real margin on flat-rate hosting and a loss anywhere else. +A 4 KiB chunk costs ~15 KiB of real bandwidth — three peers at once, plus retries. | | per GiB of payload | |---|---:| -| AWS egress at $0.09/GB | −$0.33 | +| AWS egress | −$0.33 | | revenue at $0.02/GiB | +$0.02 | | **net** | **−$0.31** | -| Customer's lifetime traffic | Revenue | Gas | Gas as % | -|---|---:|---:|---:| -| 71 MB — one browser upload | $0.0014 | $0.0005 | 36 % | -| 5 GiB — the cashout threshold | $0.10 | $0.0005 | 0.5 % | - -> A relay on per-GB bandwidth, or one whose users upload once and never return, should stay free. +> Only viable on flat-rate bandwidth, and only on customers who return: one 71 MB upload earns $0.0014 against $0.0005 of gas. --- @@ -154,13 +131,11 @@ Every 4 KiB chunk costs about **15 KiB** of real bandwidth — three peers at on | Relay | Client can pay | Client cannot | |---|---|---| -| free | used, nothing billed | used, nothing billed | -| paid, soft | used, billed, settles | used, billed, served anyway | -| paid, enforced | used, billed, settles | **dropped at startup** | +| free | nothing billed | nothing billed | +| paid, soft | billed, settles | billed, served anyway | +| paid, enforced | billed, settles | **dropped at startup** | -A paid relay rejects an upload that arrives with no permission slip, and that rejection counts against the relay's health score — only a genuine "you owe too much" is excused. Sending work there anyway burns one retry on every chunk, to rediscover what the relay already said up front. - -> Four free relays run on hosts whose disks are wiped on restart, so they have to stay free — a relay that forgets what it is owed serves for free forever. One paid relay enforces. The browser app skips it automatically, because it can sign chunks but not cheques. +> Four free relays run on hosts that wipe their disk on restart — and a relay that forgets what it is owed serves for free forever. The browser app skips the paid one: it can sign chunks, not cheques. --- @@ -168,33 +143,29 @@ A paid relay rejects an upload that arrives with no permission slip, and that re # Six bugs no test suite reached -All six survived the full test suite and a working end-to-end round trip. Reaching them needed three things **at once**: a relay that remembers debt between runs, several uploads in flight at the same time, and a batch worn down far enough that the credit limit is what stops you. - | § | Bug | |---|---| -| 17.1 | Debt carried across runs could not be paid — each run starts believing it owes nothing | -| 17.2 | The headroom check measured one chunk, then sent a whole batch | +| 17.1 | Carried-over debt could not be paid | +| 17.2 | Headroom measured one chunk, then sent a batch | | 17.3 | A rule checked against the wrong number | -| 17.4 | A relay refused for bytes still in flight was parked forever | -| 17.5 | One broken response stream made every later cheque bounce | -| 17.6 | The first upload of a run was sized before the debt was known | +| 17.4 | A relay refused for bytes in flight was parked forever | +| 17.5 | One broken stream bounced every later cheque | +| 17.6 | The first upload was sized before the debt was known | -> The one to learn from is **17.3**. Not a coding mistake — a rule compared against the highest limit the relay ever grants, when the limit that applies is per batch. Below ~0.39 BZZ of batch value the smallest allowed cheque exceeds everything the account may owe. It is refused from then on, and nothing logs an error. +> All six needed the same three things at once: debt surviving restarts, parallel uploads, and a nearly-spent batch. --- -# It works, and here is what is still open +# Where it stands -| Run | Payload | Chunks delivered | Refusals | Rejected cheques | +| Run | Payload | Delivered | Refusals | Rejected cheques | |---|---|---:|---:|---:| | 1 | 2 MiB | 567/567 | 0 | 0 | | 2 | 2 MiB | 567/567 | 0 | 0 | | 3 | 2 MiB | 567/567 | 0 | 0 | -Each run settles to zero owed, so the next carries nothing. Cheques are cashed on a **different machine** — only the payee can cash one, and the relay must never hold that key. The live cashout used 75,378 gas against a 300,000 budget. +Each run settles to zero owed. Cheques are cashed from a different machine — the relay must never hold that key. -- **We do not yet know if this pays for itself.** The number to watch is how many customers ever reach the cashout threshold. -- **Nothing stops a relay taking payment and dropping chunks.** You lose at most one credit limit and stop using it — that bounds the damage, it does not guarantee service. -- **A customer can empty their chequebook after you accept a cheque.** Bee has the same problem; it comes from how the contract is deployed. +> Two things still open: whether it ever pays for itself, and that nothing stops a relay taking payment and dropping chunks. From ab44500bac31db9dd8ef6bfa619010d1d5b6b3e4 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 14 Aug 2026 12:33:00 +0300 Subject: [PATCH 22/27] docs(deck): headings name the mechanism, eyebrows mark the act MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Who has to trust whom" is chatty where "Trust model" is the term, and the deck had no single rule for what a heading is: some stated a claim, some asked a question, some were a gerund plus a colon. One rule now — the heading names the mechanism and the body makes the claim about it, which is where there is room to say it precisely. The eyebrows stop restating the heading ("5 · credit" over "How much a client may owe") and mark the act instead, restoring the theory/practice split the deck was written for. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 57 ++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index d717028..a0d5cb6 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -13,7 +13,12 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' Keep slides sparse: a heading, ONE block (a short table, up to four bullets, or two short paragraphs), and ONE callout. The detail lives in - docs/pusher-incentives.md. --> + docs/pusher-incentives.md. + + Headings name the mechanism and nothing else — "Trust model", not "Who + has to trust whom". The claim about that mechanism goes in the body, + where there is room to say it precisely. Eyebrows mark the act (the + problem / theory / practice), so they never restate the heading. --> @@ -25,9 +30,9 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' --- - + -# The relay pays for traffic it did not cause +# Unpaid relay bandwidth Upload directly and bee bills you. Put a relay in the middle and bee bills the relay — it is the peer bee sees. The client pays only postage. @@ -40,11 +45,11 @@ Upload directly and bee bills you. Put a relay in the middle and bee bills the r --- - + -# Who has to trust whom +# Trust model -A relay is just an HTTP service — no registry, no list to get onto. +Trust runs one way. A relay is just an HTTP service — no registry, no list to get onto. - The **client** picks one and checks its signed quote - The **relay** gets whoever shows up @@ -53,9 +58,9 @@ A relay is just an HTTP service — no registry, no list to get onto. --- - + -# Bill bytes, not receipts +# The billing unit ``` owed = KiB the relay accepted × price per KiB @@ -63,13 +68,13 @@ owed = KiB the relay accepted × price per KiB > The client cannot lie about it. It produced the bytes; the relay counted them. -Billing per delivery receipt meant trusting a **third party's signature**, which took five mechanisms to make safe. Changing the unit removed all five. +Bytes admitted, not delivery receipts. Billing per receipt meant trusting a **third party's signature**, which took five mechanisms to make safe. Changing the unit removed all five. --- - + -# Getting in: a permission slip +# Admission control The relay must accept or refuse **before** reading an upload — but at that moment it does not know whose account to check. @@ -80,9 +85,9 @@ So the chain lookups happen **once**, when a slip is issued, and the credit limi --- - + -# How much a client may owe +# The credit limit The cheapest live batch costs a fraction of a cent, so "owns a batch" proves nothing. The limit tracks what the batch is worth: @@ -94,11 +99,11 @@ credit limit = batch's remaining value ÷ 1000 --- - + -# Cheques are running totals +# Settlement -Each says "you have now paid me *this much in total*". +A cheque is a running total: "you have now paid me *this much in total*". - **Losing one costs nothing** — the next covers it - **Old ones are worthless** — each must exceed the last @@ -109,9 +114,9 @@ Each says "you have now paid me *this much in total*". --- - + -# What it costs +# Unit economics A 4 KiB chunk costs ~15 KiB of real bandwidth — three peers at once, plus retries. @@ -125,9 +130,11 @@ A 4 KiB chunk costs ~15 KiB of real bandwidth — three peers at once, plus retr --- - + + +# Deployment -# Paying is optional, per relay +Paying is optional, and each relay sets its own mode. | Relay | Client can pay | Client cannot | |---|---|---| @@ -139,9 +146,9 @@ A 4 KiB chunk costs ~15 KiB of real bandwidth — three peers at once, plus retr --- - + -# Six bugs no test suite reached +# Bugs found in production | § | Bug | |---|---| @@ -152,13 +159,13 @@ A 4 KiB chunk costs ~15 KiB of real bandwidth — three peers at once, plus retr | 17.5 | One broken stream bounced every later cheque | | 17.6 | The first upload was sized before the debt was known | -> All six needed the same three things at once: debt surviving restarts, parallel uploads, and a nearly-spent batch. +> No test suite reached any of them. All six needed the same three things at once: debt surviving restarts, parallel uploads, and a nearly-spent batch. --- - + -# Where it stands +# Results | Run | Payload | Delivered | Refusals | Rejected cheques | |---|---|---:|---:|---:| From 3362f3092531151e0983b51638fea2dbcb266fe2 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 14 Aug 2026 12:34:45 +0300 Subject: [PATCH 23/27] docs(deck): say what the money slide meant, and cut slide 3 down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "This works on repeat and bulk uploads, or not at all" was a riddle. It meant §9.3 — one cashout costs ~$0.0005 of gas and is paid once per account, so an account that never returns is written off — and slide 8 already says that with the numbers. Slide 2 is the problem statement, so it now just sizes the problem: the egress a relay absorbs, and what metering would bill for it. Slide 3 spent a lead-in, two bullets and a callout to establish that a relay is a standalone HTTP service. Same content, one paragraph. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index a0d5cb6..057086e 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -41,7 +41,7 @@ Upload directly and bee bills you. Put a relay in the middle and bee bills the r | client → relay | nothing | 4.8e8 PLUR per KiB | | relay → bee | nothing | still nothing | -> 100 GB of traffic a month earns about **$0.54**. This works on repeat and bulk uploads, or not at all. +> A relay absorbs 70–100 GB of egress a month. Metered, that same month bills about **$0.54**. --- @@ -49,12 +49,9 @@ Upload directly and bee bills you. Put a relay in the middle and bee bills the r # Trust model -Trust runs one way. A relay is just an HTTP service — no registry, no list to get onto. +A relay is a standalone HTTP service — no registry, no list to get onto. Trust runs one way: the client checks a signed quote before sending a byte; the relay gets whoever shows up. -- The **client** picks one and checks its signed quote -- The **relay** gets whoever shows up - -> So the relay's defences point at the client. The client's protection is arithmetic: it counts its own bytes, and risks at most one credit limit — about **$0.0024**. +> So every relay-side defence points at the client, and the client needs none — it counts its own bytes and risks at most one credit limit, about **$0.0024**. --- From c1c900c98ea2bba9e8108caf2b171d8ac1496bb0 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Fri, 14 Aug 2026 19:06:43 +0300 Subject: [PATCH 24/27] docs(deck): reprice the economics against measured numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slide 8 was built on two figures that measurement contradicts. Egress amplification was modelled at 3.7x; the metered lane reports 1.151x, so three times the payload fits under a given bandwidth allowance. And gas was carried at $0.0005 per cashout, assuming 300k gas at gwei prices — the two real cashouts used 75k and 110k gas at 169 and 1292 wei, which is a fee of ~1e-10 xDAI. Six orders of magnitude. Both errors pointed the same way, so the conclusion flipped. The slide had it as a per-GiB loss that only flat-rate bandwidth could excuse. On a host whose cost is already sunk it is closer to all margin, and what actually bounds it is the bandwidth allowance: ~$10/mo under a 2 TB cap at the modelled ratio, ~$33 at the measured one. That is a ceiling, not a floor, so slide 11 now carries what has really been earned — $0.00006 cashed, one paying client, and it was me. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 057086e..bd08ac2 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -41,7 +41,7 @@ Upload directly and bee bills you. Put a relay in the middle and bee bills the r | client → relay | nothing | 4.8e8 PLUR per KiB | | relay → bee | nothing | still nothing | -> A relay absorbs 70–100 GB of egress a month. Metered, that same month bills about **$0.54**. +> A relay absorbs 70–100 GB of egress a month. Metered, that same month bills **$1.14–1.63**. --- @@ -115,15 +115,14 @@ A cheque is a running total: "you have now paid me *this much in total*". # Unit economics -A 4 KiB chunk costs ~15 KiB of real bandwidth — three peers at once, plus retries. +Relaying earns **$0.02 per GiB** admitted. On a host you already pay for, the only cost is cashout gas — 110k gas, a fraction of a microcent. -| | per GiB of payload | -|---|---:| -| AWS egress | −$0.33 | -| revenue at $0.02/GiB | +$0.02 | -| **net** | **−$0.31** | +| egress per byte relayed | fits under a 2 TB/mo cap | earns | +|---|---:|---:| +| 3.7× modelled | 503 GiB | $10 | +| 1.15× measured | 1.58 TiB | $33 | -> Only viable on flat-rate bandwidth, and only on customers who return: one 71 MB upload earns $0.0014 against $0.0005 of gas. +> So the ceiling is the bandwidth allowance, not the cost. Past the cap, egress runs $0.11 per GiB against $0.02 of revenue and it inverts. --- @@ -164,12 +163,12 @@ Paying is optional, and each relay sets its own mode. # Results -| Run | Payload | Delivered | Refusals | Rejected cheques | -|---|---|---:|---:|---:| -| 1 | 2 MiB | 567/567 | 0 | 0 | -| 2 | 2 MiB | 567/567 | 0 | 0 | -| 3 | 2 MiB | 567/567 | 0 | 0 | +Three runs of 2 MiB, 567/567 chunks delivered, no refusals, no rejected cheques, each settling to zero owed. Cheques cash from a separate machine — the relay must never hold that key. -Each run settles to zero owed. Cheques are cashed from a different machine — the relay must never hold that key. +| | to date | +|---|---:| +| billed | $0.0003 | +| cashed on-chain | $0.00006 | +| paying clients | 1, and it was me | -> Two things still open: whether it ever pays for itself, and that nothing stops a relay taking payment and dropping chunks. +> The mechanism works; nobody is paying it. The browser dApp is the only real traffic and it signs stamps, not cheques. Nothing yet stops a relay billing for chunks it drops. From aa6d5f285c13a957f71bd405e6f4051bcfe3045a Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sat, 15 Aug 2026 23:14:26 +0300 Subject: [PATCH 25/27] =?UTF-8?q?docs:=20correct=20=C2=A79.3's=20gas,=20an?= =?UTF-8?q?d=20withdraw=20the=20egress=20"measurement"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections pointing opposite ways. §9.3 carried $0.0005 per cashout, from the 300k gas *limit* at gwei-scale prices. The two real cashChequeBeneficiary calls used 75k and 110k gas at 169 and 1292 wei — about 1e-10 xDAI, six orders of magnitude cheaper. The "a one-shot user is not profitable" corollary was downstream of that number and is withdrawn: a 71 MB upload worth $0.0014 clears its own settlement fee by seven orders of magnitude. The 0.25 BZZ threshold is a batching convenience, not a break-even, and should come from live gas. The other way: `egress.attempts_per_frame` reports 1.077 against a modelled 3.45, and I took that as a measurement. It is not one. PUSH_OUTCOME_* is bumped after the await inside the racing future, and the dispatcher cancels the losers the moment it takes a receipt — their Delivery is already on the wire, so the egress is paid, but the increment never runs. ok=3756 against frames_admitted=3756 is the tell: exactly one counted success per chunk. The metric floors near 1.0 by construction. So §9.1's model stands unverified, the Stage 0 gate that was supposed to settle it is not met, and the comments in meter.rs and pusher.rs that claimed losing racers were counted are the reason this went unnoticed. Fixed those too. Counting at dispatch, beside inflight_pushes, is the fix. Slides follow the doc: the ceiling is ~$10/mo under a 2 TB cap, not the $33 the withdrawn measurement implied. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 14 ++--- docs/pusher-incentives.md | 105 ++++++++++++++++++++++--------- src/meter.rs | 22 ++++--- src/pusher.rs | 16 +++-- 4 files changed, 108 insertions(+), 49 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index bd08ac2..7edde76 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -41,7 +41,7 @@ Upload directly and bee bills you. Put a relay in the middle and bee bills the r | client → relay | nothing | 4.8e8 PLUR per KiB | | relay → bee | nothing | still nothing | -> A relay absorbs 70–100 GB of egress a month. Metered, that same month bills **$1.14–1.63**. +> A relay absorbs 70–100 GB of egress a month. Metered, that same month bills **$0.35–0.51**. --- @@ -115,14 +115,14 @@ A cheque is a running total: "you have now paid me *this much in total*". # Unit economics -Relaying earns **$0.02 per GiB** admitted. On a host you already pay for, the only cost is cashout gas — 110k gas, a fraction of a microcent. +Relaying earns **$0.02 per GiB** admitted. On a host you already pay for, the only cost is cashout gas — 110k gas, a ten-billionth of a dollar. -| egress per byte relayed | fits under a 2 TB/mo cap | earns | -|---|---:|---:| -| 3.7× modelled | 503 GiB | $10 | -| 1.15× measured | 1.58 TiB | $33 | +| | per month | +|---|---:| +| payload under a 2 TB egress cap | 503 GiB | +| billed at $0.02/GiB | **$10** | -> So the ceiling is the bandwidth allowance, not the cost. Past the cap, egress runs $0.11 per GiB against $0.02 of revenue and it inverts. +> So the ceiling is the bandwidth allowance, not the cost. Past the cap egress runs $0.36 per GiB against $0.02 of revenue, and it inverts. --- diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index 1c1f4ef..c41f030 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -53,10 +53,14 @@ Metering is the answer to both. It puts the cost back on the party that caused it, converts §6's accepted quota-drain risk into a priced one, and makes running a dedicated-IP relay a rational act rather than a donation. -Nobody gets rich. §9's economics are thin and §9.3 is explicit that a -single small upload does not cover its own cashout gas. The target is a -self-sustaining lane federation funded by repeat and bulk traffic, not -profit. +Nobody gets rich. §9's economics are thin: at $0.02/GiB and §9.1's 3.7× +egress, a relay saturating a 2 TB/month bandwidth allowance bills about +**$10**. On hardware whose cost is already sunk — a box running other +things, bandwidth already included — that is close to all margin, since +§9.3's gas turns out to be negligible and the *client* funds the +chequebook, not the relay. But the allowance is a ceiling and crossing it +inverts the economics in one step (§9.2), so what this funds is a +self-sustaining lane federation, not a business. ## 2. Trust model — read this before anything else @@ -713,6 +717,29 @@ Real cost per 4 KiB chunk relayed: | **Egress per chunk relayed** | **≈ 15 KiB** | | **Egress per GiB of payload** | **≈ 3.7 GiB** (262 144 chunks/GiB) | +**The Stage 0 counter does not check this, and currently cannot.** +`/v1/meter` → `egress.attempts_per_frame` exists to test the ×3.45 model +against reality, and the production relay reports **1.077**. That is not a +refutation of the model, it is an instrument fault. `PUSH_OUTCOME_*` is +incremented *after* the await inside the racing future +(`src/client.rs:5818-5870`), and the dispatcher cancels the losing racers +the moment it accepts a receipt — `src/client.rs:4806-4809` says so in as +many words. A cancelled racer has already written its Delivery to the +wire, so the relay pays that egress, and then the future is dropped before +it ever reaches the counter. + +The relay's own diagnostics show the shape plainly: `ok = 3756` against +`frames_admitted = 3756` — exactly one counted success per chunk, with +only 285 shallow retries and 4 errors above it. A metric that counts +completions cannot see a race it loses on purpose, so it floors near 1.0 +by construction. + +Counting at *dispatch* (where `inflight_pushes` is already incremented, +`src/client.rs:4818-4820`) rather than at completion would fix it. Until +then §9.1's model stands unverified in both directions, and **nothing here +should be repriced on `attempts_per_frame`** — which was the one job Stage +0 was supposed to do for §9.2. + ### 9.2 Price | | per GiB of payload | @@ -750,22 +777,39 @@ parties measure it directly. ### 9.3 Gas, and who is actually profitable -Cashout is `GetGasLimitWithDefault(ctx, 300_000)` (`cashout.go:145`) ≈ -**$0.0005** on Gnosis. Issuing a cheque costs nothing +Cashout's gas *limit* is `GetGasLimitWithDefault(ctx, 300_000)` +(`cashout.go:145`), but a limit is not a spend. The two real +`cashChequeBeneficiary` calls on Gnosis mainnet used **75 378** and +**109 590** gas at **169** and **1 292** wei/gas — fees of `1.3e-11` and +`1.4e-10` xDAI: + +| | gas used | gas price | fee | +|---|---:|---:|---:| +| 2026-08-07 | 75 378 | 169 wei | 1.3e-11 xDAI | +| 2026-08-08 | 109 590 | 1 292 wei | 1.4e-10 xDAI | + +An earlier draft of this section carried **$0.0005**, assuming the full +300 k limit at gwei-scale prices. Gnosis base fee during these runs was +50–1 300 wei, so that estimate was high by roughly **six orders of +magnitude**. Issuing a cheque still costs nothing (`chequebook.go:190-250` sends no transaction); only cashing out touches -the chain. Because cheques are cumulative, gas is paid **once per -account**, whenever the relay decides to cash. - -Suggested cashout threshold **0.25 BZZ ≈ $0.10 ≈ 5 GiB relayed by one -account**, which puts gas at ~0.5 % of realized revenue. - -The uncomfortable corollary: **a one-shot user is not profitable.** Design -§11's flagship metric is a 71 MB browser upload, worth -`71/1024 × $0.02 ≈ $0.0014` — about 3× the cashout gas, and only if it is -ever cashed. Accounts below the threshold that never return are written -off. Metered relay economics work on repeat and bulk accounts; cumulative -cheques are what make a returning user amortize. A relay whose traffic is -entirely one-shot browser users should run `open`. +the chain, and because cheques are cumulative that gas is paid **once per +account**. + +**This removes the gas floor, not the price floor.** At ~`1e-10` xDAI a +cashout, essentially any non-zero cheque is worth collecting. Design §11's +flagship 71 MB browser upload is worth `71/1024 × $0.02 ≈ $0.0014`, which +clears its own settlement fee by about seven orders of magnitude. The +conclusion this section used to draw — *"a one-shot user is not +profitable"* — was an artifact of the stale gas number and is withdrawn. + +What survives is an *operational* floor rather than an economic one: an +RPC round trip, a pending transaction to watch, and a chequebook that has +to stay funded. So read the 0.25 BZZ threshold as a batching convenience, +not as a break-even point — and note that Gnosis gas is not permanently +this cheap. A floor computed from `eth_gasPrice` at cashout time holds +under a price spike in a way a hardcoded constant does not, in either +direction. ## 10. Credit and settlement @@ -1285,11 +1329,13 @@ accounts**, reported as `credit.batches_below_one_full_post` and the specifically, what fraction of real users would be capped below the global ceiling, and whether §7.2's POST-sizing interaction bites in practice. -Stage 0 also settles §9.1's cost basis, which the doc currently *models* +Stage 0 was also meant to settle §9.1's cost basis, which the doc *models* rather than measures: `egress.attempts_per_frame` against the modelled -3.45, since the push path counts every per-stream attempt including losing -racers and shallow retries (`src/client.rs:5361-5363`). A materially -different number moves the price in §9.2. +3.45. **It does not, and the claim that justified it was wrong.** The +counter was believed to see every per-stream attempt including losing +racers; it is bumped after the await inside a future the dispatcher +cancels, so it sees completions only. §9.1 has the detail. This gate is +**not met** and needs the counter moved to dispatch before any repricing. *(The previous design had four gating measurements including a kill criterion — the staked fraction of receipt signers, which could have @@ -1399,9 +1445,13 @@ on-chain, and presents the ones worth collecting. Two things it forced: `paidOut(beneficiary)`. A repeated run sees `unclaimed 0` and presents nothing, which matters for a command that will be run on a timer. -`--min-amount` defaults to 0.25 BZZ (§9.3's threshold): cashing costs ~300k -gas whatever the amount, so a smaller cheque is worth less than collecting -it. Verified on Gnosis mainnet — a 145,920,000,000 PLUR cheque presented, +`--min-amount` defaults to 0.25 BZZ (§9.3's threshold). The reasoning it +was given — cashing costs ~300k gas whatever the amount, so a smaller +cheque is worth less than collecting it — does not survive §9.3's measured +fees; at ~`1e-10` xDAI the amount that is *not* worth collecting is far +below any cheque this system will produce. Keep the default as batching +convenience, and see §9.3 on deriving it from live gas. Verified on Gnosis +mainnet — a 145,920,000,000 PLUR cheque presented, `paidOut` moved by exactly that, and the beneficiary's BZZ balance moved by exactly that. Cashout reads `bounced` and `liquidBalanceFor` rather than `balance` (§11.2), and optionally offers @@ -1419,8 +1469,7 @@ an opt-in dApp flow where the wallet deploys a chequebook with `issuer = sessionKey` and funds it — two transactions on top of today's approve + `createBatch`, ≈ $0.001 of Gnosis gas, one-time per user and reused across every upload and lane. The dApp's four public lanes stay in -`open` mode throughout, so §9.3's one-shot-user problem does not bite -them. +`open` mode throughout, so nothing here is on the settlement path at all. One Stage 3 hazard Stage 1 does not have: **a session-key chequebook can strand funds permanently.** `withdraw()` is issuer-only, and the issuer diff --git a/src/meter.rs b/src/meter.rs index 2e0c206..5f7882b 100644 --- a/src/meter.rs +++ b/src/meter.rs @@ -17,12 +17,13 @@ //! which is fine; a population where most batches are in that state means //! the ratio is wrong. //! -//! It also measures §9.1's egress multiplier, which the doc currently -//! *estimates* at ×3 racing × 1.15 shallow ≈ 3.45 attempts per chunk. That -//! number sets the whole cost basis and has never been observed. The push -//! path already counts every per-stream attempt -//! (`src/client.rs:5361-5363`), so the multiplier is that total over the -//! frames this module admitted. +//! It also reports §9.1's egress multiplier, which the doc *estimates* at +//! ×3 racing × 1.15 shallow ≈ 3.45 attempts per chunk — the number that +//! sets the whole cost basis. **The reported figure is not yet a valid +//! measurement of it.** It divides completed push outcomes by frames +//! admitted, and the racing dispatcher cancels losing racers before they +//! complete, so their egress is spent but uncounted (see `stream_attempts` +//! in `src/pusher.rs`). Treat it as a floor, not an observation. //! //! **Hot path cost is one lock per POST.** A request accumulates into a //! [`PostTally`] on its own stack and merges once at completion, so N @@ -422,9 +423,12 @@ impl Meter { } } -/// §9.1's cost basis, observed rather than estimated. The doc's model is -/// ×3 peer race × 1.15 shallow retries ≈ 3.45 stream attempts per chunk; a -/// materially different number moves the price in §9.2. +/// §9.1's cost basis. The doc's model is ×3 peer race × 1.15 shallow +/// retries ≈ 3.45 stream attempts per chunk. +/// +/// `attempts` counts *completed* outcomes, so this is a lower bound rather +/// than the observation §9.2 needs — cancelled racers are missing. Do not +/// reprice on it until the counter moves to dispatch. fn egress(frames: u64, billable_kib: u64, attempts: u64) -> serde_json::Value { if frames == 0 { return json!({"frames": 0, "stream_attempts": attempts}); diff --git a/src/pusher.rs b/src/pusher.rs index 6216441..7dba70f 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -1071,11 +1071,17 @@ fn full_post_kib() -> u64 { (PUSH_BATCH_MAX * pushframe::MAX_FRAME_LEN).div_ceil(1024) as u64 } -/// Total per-stream push attempts since boot. Incentives §9.1's egress -/// multiplier is this over frames admitted — the counters are bumped per -/// stream rather than per chunk (`src/client.rs:5361-5363`), so losing -/// racers and shallow retries are both included, which is exactly the cost -/// the relay actually pays. +/// Completed per-stream push outcomes since boot. +/// +/// Incentives §9.1 reads this over frames admitted as an egress multiplier, +/// and **it undercounts**: the counters are bumped after the await inside +/// the racing future (`src/client.rs:5818-5870`), and the dispatcher +/// cancels the losing racers as soon as it takes a receipt +/// (`src/client.rs:4806-4809`). A cancelled racer has already put its +/// Delivery on the wire — the relay pays that egress — but never reaches +/// the increment. Shallow retries and errors are counted; concurrent +/// losers are not, so the ratio floors near 1.0 whenever the race is won +/// promptly. Counting at dispatch, beside `inflight_pushes`, would fix it. fn stream_attempts() -> u64 { use crate::transport::diag; use std::sync::atomic::Ordering; From 8a0ef05ec5e6fd9b9c48048cfb83f3a695a5a37a Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sat, 22 Aug 2026 13:52:44 +0300 Subject: [PATCH 26/27] docs: fact-check pass over the deck and its source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified every figure on the slides against the doc, the code, the live relay and Gnosis. The numbers hold — $0.0003 billed matches the relay's own `owed_usd`, and the $0.00006 cashed is the sum of two on-chain `cashChequeBeneficiary` transfers. What did not hold: Doc and code: - §9.3's table had the two cashouts' dates swapped. 109 590 gas at 1 292 wei is 08-07; 75 378 at 169 wei is 08-08. Added the chequebook and tx hashes so the row is checkable rather than asserted. - §9.2 priced per-GB clouds at $0.33/GiB in prose and $0.36 in the table directly above it — the prose had dropped the GB→GiB conversion. - §15 still carried "one-shot users do not cover their own cashout gas" as a residual risk, which is the exact claim §9.3 withdraws. Replaced with the risk that actually survives: a threshold pinned to a gas cost that no longer holds. - §17's intro said "all three survived the test suite" in a section about six bugs. - `meter.rs` asserted the withdrawn claim twice more, on CASHOUT_THRESHOLD_PLUR and in the module header. Slides: - The unit-economics callout blamed the inversion on going "past the cap". $0.36/GiB is AWS on-demand egress, not an overage rate; §9.2's point is that metered mode only works on bandwidth nobody bills by the gigabyte. - The problem slide read 70–100 GB as consumption. Design §6 quotes it as the free tier's allowance — a ceiling. Its table also said the relay pays bee nothing directly under a lead saying bee bills the relay; the debt is real, pseudosettle just pays it in time. - The credit-limit formula dropped the `min()` with the global ceiling, so "a thousandth at any batch size" overstated a bound that is only ever tighter. - The trust-model callout claimed the client needs no defences. §2 lists four bounds that protect it and says outright it is not unprotected. - §17.4's row inverted the actor: the client was refused, and it parked the lane. - "All six needed the same three things at once" is stronger than §17, which states the three as the union across the set. Co-Authored-By: Claude Opus 5 (1M context) --- docs/pusher-incentives-slides.md | 18 +++++++++--------- docs/pusher-incentives.md | 19 ++++++++++++++----- src/meter.rs | 11 ++++++++--- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/docs/pusher-incentives-slides.md b/docs/pusher-incentives-slides.md index 7edde76..e5666ba 100644 --- a/docs/pusher-incentives-slides.md +++ b/docs/pusher-incentives-slides.md @@ -34,14 +34,14 @@ header: 'Paying for relay — an incentive layer for hoverfly pushers' # Unpaid relay bandwidth -Upload directly and bee bills you. Put a relay in the middle and bee bills the relay — it is the peer bee sees. The client pays only postage. +Upload directly and bee debits your own node for every chunk. Put a relay in the middle and that debt moves to it — the relay is the peer bee sees. Pseudosettle pays it in time rather than money, so what a relay actually spends is bandwidth, and nothing prices that. | | free | paid | |---|---|---| | client → relay | nothing | 4.8e8 PLUR per KiB | -| relay → bee | nothing | still nothing | +| relay → bee | time, not money | unchanged | -> A relay absorbs 70–100 GB of egress a month. Metered, that same month bills **$0.35–0.51**. +> A free tier hands a relay 70–100 GB of egress a month. Burn all of it under metering and it bills **$0.35–0.51**. --- @@ -51,7 +51,7 @@ Upload directly and bee bills you. Put a relay in the middle and bee bills the r A relay is a standalone HTTP service — no registry, no list to get onto. Trust runs one way: the client checks a signed quote before sending a byte; the relay gets whoever shows up. -> So every relay-side defence points at the client, and the client needs none — it counts its own bytes and risks at most one credit limit, about **$0.0024**. +> So every defence here points from the relay at the client. The other direction gets no cryptography, only bounds: the client computes its own bill, and risks at most one credit limit per lane — about **$0.0024**. --- @@ -89,10 +89,10 @@ So the chain lookups happen **once**, when a slip is issued, and the credit limi The cheapest live batch costs a fraction of a cent, so "owns a batch" proves nothing. The limit tracks what the batch is worth: ``` -credit limit = batch's remaining value ÷ 1000 +credit limit = min(batch's remaining value ÷ 1000, a global ceiling) ``` -> An attacker gets back **a thousandth of what they funded**, at any batch size. The ratio is what is fixed, so there is no cheap corner to aim at. +> An attacker gets back **at most a thousandth of what they funded**, and less once the ceiling binds. The ratio is what is fixed, so there is no cheap corner to aim at. --- @@ -122,7 +122,7 @@ Relaying earns **$0.02 per GiB** admitted. On a host you already pay for, the on | payload under a 2 TB egress cap | 503 GiB | | billed at $0.02/GiB | **$10** | -> So the ceiling is the bandwidth allowance, not the cost. Past the cap egress runs $0.36 per GiB against $0.02 of revenue, and it inverts. +> So the ceiling is the allowance, not the cost — and only on bandwidth nobody bills by the gigabyte. Billed per GB, the same traffic costs **$0.36 per GiB** against $0.02 of revenue, and that relay should run free. --- @@ -151,11 +151,11 @@ Paying is optional, and each relay sets its own mode. | 17.1 | Carried-over debt could not be paid | | 17.2 | Headroom measured one chunk, then sent a batch | | 17.3 | A rule checked against the wrong number | -| 17.4 | A relay refused for bytes in flight was parked forever | +| 17.4 | A lane refused for bytes in flight was parked forever | | 17.5 | One broken stream bounced every later cheque | | 17.6 | The first upload was sized before the debt was known | -> No test suite reached any of them. All six needed the same three things at once: debt surviving restarts, parallel uploads, and a nearly-spent batch. +> No test suite reached any of them, and none is reachable from one upload against a fresh relay. Between them they needed debt surviving restarts, several POSTs in flight, and a batch spent down far enough to bind. --- diff --git a/docs/pusher-incentives.md b/docs/pusher-incentives.md index c41f030..644b7d3 100644 --- a/docs/pusher-incentives.md +++ b/docs/pusher-incentives.md @@ -755,7 +755,7 @@ the user already paid for postage it is a rounding error. **Read that AWS row as a constraint, not a favourable comparison.** At §9.1's 3.7 GiB of real egress per GiB of payload, per-GB-billed clouds -cost the relay ~$0.33/GiB against $0.02 of revenue. Metered mode is only +cost the relay ~$0.36/GiB against $0.02 of revenue. Metered mode is only rational on **flat-rate or included bandwidth** — the same class of host §5 already requires for durable storage, and the same class §1 says metering exists to fund. A relay on metered egress should run `open` and @@ -785,8 +785,12 @@ Cashout's gas *limit* is `GetGasLimitWithDefault(ctx, 300_000)` | | gas used | gas price | fee | |---|---:|---:|---:| -| 2026-08-07 | 75 378 | 169 wei | 1.3e-11 xDAI | -| 2026-08-08 | 109 590 | 1 292 wei | 1.4e-10 xDAI | +| 2026-08-07 | 109 590 | 1 292 wei | 1.4e-10 xDAI | +| 2026-08-08 | 75 378 | 169 wei | 1.3e-11 xDAI | + +Both are `cashChequeBeneficiary` on chequebook +`0x17c89DE40f5ec07343AB095bfDa9dE1A5c095Fc1` (txs `0xb33bd199…` and +`0x9d4a8b29…`), settling `1.46e12` PLUR in total to the relay beneficiary. An earlier draft of this section carried **$0.0005**, assuming the full 300 k limit at gwei-scale prices. Gnosis base fee during these runs was @@ -1486,7 +1490,12 @@ Carried knowingly: - **§11.2** — cheques are unsecured claims, bounded by the per-account cap. Hard deposits work but are inert as bee deploys them; secured mode costs a cold-key signature per client chequebook. -- **§9.3** — one-shot users do not cover their own cashout gas. +- **§9.3** — the cashout threshold is a hardcoded constant sized against a + gas cost that no longer holds. At today's Gnosis prices it is a batching + convenience, not a break-even, and a gas spike moves the real floor + without moving the constant. Deriving it from `eth_gasPrice` at cashout + time is the fix; until then the number is stale in whichever direction + gas has drifted. - **§11.3** — aggregate exposure across beneficiaries is invisible to the client until `total_issued` ships. - **§11.5** — griefing by aiming at badly-covered arcs still forces the @@ -1648,7 +1657,7 @@ pushed. ## 17. Found by running a metered relay: six bugs (all fixed) None of these is reachable from a single upload against a fresh relay, -which is why all three survived the test suite and the Stage 1 round-trip. +which is why all six survived the test suite and the Stage 1 round-trip. They need a relay whose ledger *persists across client runs* — the shipped configuration — enough concurrency to have several POSTs on the wire at once, and a batch that has been spent down far enough for its credit line diff --git a/src/meter.rs b/src/meter.rs index 5f7882b..8ce8315 100644 --- a/src/meter.rs +++ b/src/meter.rs @@ -9,8 +9,9 @@ //! //! 1. **Is anyone consuming enough to justify metering?** Per-account bytes //! admitted, what they would owe at §9.2's candidate price, and how many -//! accounts would ever cross the cashout threshold — below which an -//! account costs more in gas than it yields (§9.3). +//! accounts would ever cross the cashout threshold — not a profitability +//! line (§9.3's gas is negligible), but the volume at which cashing out +//! is worth an operator's round trip at all. //! 2. **Is `credit_ratio = 1000` right?** For every batch actually seen, the //! credit line §10.3 would have granted it, against the size of a full //! POST. A batch whose line is under one POST has to split its uploads, @@ -52,7 +53,11 @@ pub const PRICE_PLUR_PER_KIB: u128 = 480_000_000; pub const SETTLE_EVERY_PLUR: u128 = 15_600_000_000_000; /// ~127 MiB — the global ceiling on a credit line. pub const MAX_OUTSTANDING_PLUR: u128 = 62_200_000_000_000; -/// 0.25 BZZ ≈ 5 GiB — below this an account never repays its cashout gas. +/// 0.25 BZZ ≈ 5 GiB. Not a break-even: §9.3's measured cashout gas is ~1e-10 +/// xDAI, so any non-zero cheque repays it. This is a batching convenience — +/// how much value to let accumulate before spending an RPC round trip and a +/// pending transaction on it — and should be derived from `eth_gasPrice` +/// rather than pinned, since Gnosis gas will not stay this cheap. pub const CASHOUT_THRESHOLD_PLUR: u128 = 2_500_000_000_000_000; /// Credit line = batch remaining value ÷ this (§10.3). pub const CREDIT_RATIO: u128 = 1_000; From b6a3b525f55a0fd52b11fb842b070ecfdc001e20 Mon Sep 17 00:00:00 2001 From: v1rtl Date: Sat, 22 Aug 2026 15:52:43 +0300 Subject: [PATCH 27/27] style: cargo fmt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo fmt -- --check` is a CI step and this branch had never been run through it. No behaviour change — every hunk is rustfmt's own line breaking, and every file it touched is one the branch already added or modified. Co-Authored-By: Claude Opus 5 (1M context) --- src/batch.rs | 41 ++++++++++------- src/bin/hoverfly.rs | 92 ++++++++++++++++++++++++++++----------- src/challenge.rs | 10 ++++- src/cheques.rs | 9 ++-- src/client.rs | 43 ++++++++++-------- src/inbound_limit.rs | 15 +++++-- src/ledger.rs | 23 +++++++--- src/meter.rs | 17 +++++--- src/metered.rs | 69 ++++++++++++++++++++++++----- src/payer.rs | 54 +++++++++++++++++------ src/protocols/pushsync.rs | 3 +- src/protocols/swap.rs | 3 +- src/pusher.rs | 64 +++++++++++++++++---------- src/pushsched/tests.rs | 6 ++- src/signer.rs | 6 ++- 15 files changed, 327 insertions(+), 128 deletions(-) diff --git a/src/batch.rs b/src/batch.rs index 7015ad2..f12de64 100644 --- a/src/batch.rs +++ b/src/batch.rs @@ -1267,7 +1267,10 @@ mod chequebook_binding_tests { (paidOutCall::SELECTOR, "paidOut(address)"), (bouncedCall::SELECTOR, "bounced()"), (liquidBalanceForCall::SELECTOR, "liquidBalanceFor(address)"), - (deployedContractsCall::SELECTOR, "deployedContracts(address)"), + ( + deployedContractsCall::SELECTOR, + "deployedContracts(address)", + ), ] { let want: [u8; 32] = ::digest(sig.as_bytes()).into(); assert_eq!(got, want[..4], "selector drift for {sig}"); @@ -1280,7 +1283,10 @@ mod chequebook_binding_tests { fn only_known_chains_have_a_factory() { assert!(swap_factory_for_chain(100).is_some(), "gnosis"); assert!(swap_factory_for_chain(11155111).is_some(), "sepolia"); - assert!(swap_factory_for_chain(1).is_none(), "mainnet: no vetted factory"); + assert!( + swap_factory_for_chain(1).is_none(), + "mainnet: no vetted factory" + ); assert!(swap_factory_for_chain(31337).is_none(), "local devnet"); } } @@ -1372,10 +1378,7 @@ pub async fn deploy_chequebook( // Trust the chain, not the simulation: read the address back out of the // receipt's `SimpleSwapDeployed` log. - let deployed = rpc - .find_deployed_chequebook(tx) - .await? - .unwrap_or(predicted); + let deployed = rpc.find_deployed_chequebook(tx).await?.unwrap_or(predicted); // The relay checks this before accepting any cheque (§6), so checking it // here turns "your cheques are silently refused" into a deploy-time @@ -1419,11 +1422,14 @@ pub async fn fund_chequebook( } // Refuse to fund something that is not a chequebook: a mistyped address // sends BZZ somewhere unrecoverable. - let issuer: Address = rpc.call_view(chequebook, issuerCall {}).await.map_err(|e| { - BatchError::Rpc(format!( - "{chequebook} does not answer issuer() — is it a chequebook? ({e})" - )) - })?; + let issuer: Address = rpc + .call_view(chequebook, issuerCall {}) + .await + .map_err(|e| { + BatchError::Rpc(format!( + "{chequebook} does not answer issuer() — is it a chequebook? ({e})" + )) + })?; if issuer != from { return Err(BatchError::Rpc(format!( "chequebook {chequebook} is issued by {issuer}, not {from}: only the issuer \ @@ -1435,9 +1441,7 @@ pub async fn fund_chequebook( amount, } .abi_encode(); - let tx = rpc - .send_signed(signer, chain_id, bzz_token, &call) - .await?; + let tx = rpc.send_signed(signer, chain_id, bzz_token, &call).await?; rpc.wait_for_success(tx, receipt_timeout).await?; Ok(tx) } @@ -1446,7 +1450,10 @@ impl EthRpc { /// `eth_getCode` length, for "is there already a contract here". async fn code_len(&self, addr: Address) -> Result { let hex_str: String = self - .raw("eth_getCode", (format!("0x{}", hex::encode(addr)), "latest")) + .raw( + "eth_getCode", + (format!("0x{}", hex::encode(addr)), "latest"), + ) .await?; Ok(hex_str.trim_start_matches("0x").len() / 2) } @@ -1473,7 +1480,9 @@ impl EthRpc { .collect() }) .unwrap_or_default(); - if topics.first().map(|t| t.trim_start_matches("0x").to_lowercase()) + if topics + .first() + .map(|t| t.trim_start_matches("0x").to_lowercase()) != Some(hex::encode(topic)) { continue; diff --git a/src/bin/hoverfly.rs b/src/bin/hoverfly.rs index 46211e3..07afcd5 100644 --- a/src/bin/hoverfly.rs +++ b/src/bin/hoverfly.rs @@ -820,7 +820,11 @@ enum Commands { /// without a relay there are no cheques to cash. #[cfg(all(unix, feature = "pusher"))] Cashout { - #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] rpc_url: String, /// The **beneficiary's** private key — the EOA cheques were made /// out to, and the only address the contract will pay. @@ -975,7 +979,11 @@ enum ChequebookAction { /// Deploys with a hard-deposit timeout of 0, matching every bee /// chequebook. Deposits are therefore not locked; see §11.2. Deploy { - #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] rpc_url: String, /// Private key (hex, 32 bytes). Its address becomes the issuer. #[arg(long, value_name = "KEY")] @@ -993,7 +1001,11 @@ enum ChequebookAction { /// `issuer()`, or whose issuer is not `--key` — only the issuer can /// withdraw, so funding someone else's chequebook strands the deposit. Fund { - #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] rpc_url: String, #[arg(long, value_name = "KEY")] key: String, @@ -1011,7 +1023,11 @@ enum ChequebookAction { /// given beneficiary, what it has already paid out, and whether it has /// ever bounced. Status { - #[arg(long, default_value = "https://rpc.gnosischain.com", value_name = "URL")] + #[arg( + long, + default_value = "https://rpc.gnosischain.com", + value_name = "URL" + )] rpc_url: String, #[arg(long, value_name = "ADDR")] chequebook: String, @@ -1973,8 +1989,8 @@ async fn main() -> Result<(), Box> { // deploys or funds anything on its own. let pay_cfg = match cli.chequebook.as_ref() { Some(cb_hex) => { - let cb = parse_address_hex(cb_hex) - .map_err(|e| format!("--chequebook: {e}"))?; + let cb = + parse_address_hex(cb_hex).map_err(|e| format!("--chequebook: {e}"))?; // The relay checks funding against `liquidBalanceFor` // at accept time; we check total issuance against the // same balance before signing, so we never hand over a @@ -1985,9 +2001,11 @@ async fn main() -> Result<(), Box> { alloy_primitives::Address::ZERO, ) .await?; - let store = - hoverfly::cheques::ChequeStore::load_or_create(&cli.cheques_file, cb) - .map_err(|e| format!("loading {}: {e}", cli.cheques_file.display()))?; + let store = hoverfly::cheques::ChequeStore::load_or_create( + &cli.cheques_file, + cb, + ) + .map_err(|e| format!("loading {}: {e}", cli.cheques_file.display()))?; eprintln!( "metered: chequebook=0x{} liquid={} PLUR", hex::encode(cb), @@ -2348,8 +2366,7 @@ async fn main() -> Result<(), Box> { // is fixed by the platform. let envs = |k: &str| std::env::var(k).ok().filter(|s| !s.trim().is_empty()); let meter = meter || envs("HOVERFLY_METER").is_some_and(|v| v != "0"); - let meter_hard = - meter_hard || envs("HOVERFLY_METER_HARD").is_some_and(|v| v != "0"); + let meter_hard = meter_hard || envs("HOVERFLY_METER_HARD").is_some_and(|v| v != "0"); let origin = if origin.is_empty() { envs("HOVERFLY_METER_ORIGIN") .map(|v| v.split(',').map(|s| s.trim().to_string()).collect()) @@ -2903,11 +2920,25 @@ async fn main() -> Result<(), Box> { ) .await?; println!(); - println!("chequebook 0x{} (account 0x{})", hex::encode(cb), hex::encode(account)); - println!(" cumulative {cheque_cumulative}", cheque_cumulative = cheque.cumulative_plur); + println!( + "chequebook 0x{} (account 0x{})", + hex::encode(cb), + hex::encode(account) + ); + println!( + " cumulative {cheque_cumulative}", + cheque_cumulative = cheque.cumulative_plur + ); println!(" unclaimed {}", q.requested_plur); - println!(" payable now {}{}", q.payable_plur, - if q.would_bounce { " <- chequebook cannot cover the claim" } else { "" }); + println!( + " payable now {}{}", + q.payable_plur, + if q.would_bounce { + " <- chequebook cannot cover the claim" + } else { + "" + } + ); if q.already_bounced { println!(" NOTE: this chequebook has bounced before"); } @@ -2917,7 +2948,9 @@ async fn main() -> Result<(), Box> { continue; } if q.payable_plur < floor { - println!(" SKIP: below --min-amount ({min_amount} BZZ); gas would cost more than this collects"); + println!( + " SKIP: below --min-amount ({min_amount} BZZ); gas would cost more than this collects" + ); skipped += 1; continue; } @@ -2952,12 +2985,13 @@ async fn main() -> Result<(), Box> { } => { let signer = parse_signer(&key)?; let issuer = signer.address(); - let factory = hoverfly::batch::swap_factory_for_chain(chain_id).ok_or_else(|| { - format!( - "no vetted SimpleSwapFactory for chain {chain_id} — a factory address \ + let factory = + hoverfly::batch::swap_factory_for_chain(chain_id).ok_or_else(|| { + format!( + "no vetted SimpleSwapFactory for chain {chain_id} — a factory address \ must never be guessed, since a fake one can return a forged issuer()" - ) - })?; + ) + })?; println!("deploying chequebook: issuer 0x{}", hex::encode(issuer)); println!(" factory 0x{}", hex::encode(factory)); let out = hoverfly::batch::deploy_chequebook( @@ -3335,9 +3369,18 @@ mod chequebook_cli_tests { fn bzz_amounts_use_sixteen_decimals() { let one = parse_bzz_amount("1").expect("1 BZZ"); assert_eq!(one.to_string(), "10000000000000000"); - assert_eq!(parse_bzz_amount("0.05").expect("0.05").to_string(), "500000000000000"); - assert_eq!(parse_bzz_amount("0.0001").expect("small").to_string(), "1000000000000"); - assert_eq!(parse_bzz_amount("1.5").expect("1.5").to_string(), "15000000000000000"); + assert_eq!( + parse_bzz_amount("0.05").expect("0.05").to_string(), + "500000000000000" + ); + assert_eq!( + parse_bzz_amount("0.0001").expect("small").to_string(), + "1000000000000" + ); + assert_eq!( + parse_bzz_amount("1.5").expect("1.5").to_string(), + "15000000000000000" + ); } #[test] @@ -3347,4 +3390,3 @@ mod chequebook_cli_tests { } } } - diff --git a/src/challenge.rs b/src/challenge.rs index 80d9a7e..ba0c219 100644 --- a/src/challenge.rs +++ b/src/challenge.rs @@ -218,7 +218,10 @@ impl PresentedChallenge { let raw = hex::decode(get(k)?.trim_start_matches("0x")) .map_err(|e| format!("challenge {k} hex: {e}"))?; if raw.len() != n { - return Err(format!("challenge {k} must be {n} bytes, got {}", raw.len())); + return Err(format!( + "challenge {k} must be {n} bytes, got {}", + raw.len() + )); } Ok(raw) }; @@ -403,7 +406,10 @@ mod tests { verify(&SECRET, &f, &long, 1, &origins()), Err(ChallengeError::BadMac) ); - assert_eq!(verify(&SECRET, &f, &[], 1, &origins()), Err(ChallengeError::BadMac)); + assert_eq!( + verify(&SECRET, &f, &[], 1, &origins()), + Err(ChallengeError::BadMac) + ); } #[test] diff --git a/src/cheques.rs b/src/cheques.rs index 87571bc..d87cfd3 100644 --- a/src/cheques.rs +++ b/src/cheques.rs @@ -209,7 +209,10 @@ impl ChequeStore { /// exceeds it and bounces. Mirrors bee's `reserveTotalIssued` /// (`chequebook.go:163-178`) on the issuing side. pub fn total_issued(&self) -> u128 { - self.payouts.values().copied().fold(0u128, u128::saturating_add) + self.payouts + .values() + .copied() + .fold(0u128, u128::saturating_add) } /// Would raising `key` to `cumulative` push the total past `balance`? @@ -219,9 +222,7 @@ impl ChequeStore { /// the relay's fault and costs the lane's trust rather than the /// client's. pub fn would_exceed_balance(&self, key: &str, cumulative: u128, balance: u128) -> bool { - let others = self - .total_issued() - .saturating_sub(self.cumulative(key)); + let others = self.total_issued().saturating_sub(self.cumulative(key)); others.saturating_add(cumulative) > balance } diff --git a/src/client.rs b/src/client.rs index ef1c40e..d4914d0 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2714,14 +2714,13 @@ where // *itself* a default-weighted start, never degrade routing for the // others — which is exactly what the previous all-or-nothing overlay // collection did. - let infos: Vec = - futures::future::join_all(pusher_urls.iter().map(|u| { - // A lane quoting more than this is refused rather than paid. - // Priced off the shipped default so a lane cannot quietly - // charge an order of magnitude more than the design assumes. - fetch_lane_info(&http, u, crate::meter::PRICE_PLUR_PER_KIB * 8) - })) - .await; + let infos: Vec = futures::future::join_all(pusher_urls.iter().map(|u| { + // A lane quoting more than this is refused rather than paid. + // Priced off the shipped default so a lane cannot quietly + // charge an order of magnitude more than the design assumes. + fetch_lane_info(&http, u, crate::meter::PRICE_PLUR_PER_KIB * 8) + })) + .await; for (i, (u, info)) in pusher_urls.iter().zip(&infos).enumerate() { info!(target: "hoverfly::upload", "lane {i} {u}: pool={:?} batch_max={:?} inflight_max={:?} budget_gb={:?}", @@ -2781,7 +2780,10 @@ where // is served exactly like an `open` one. let unusable: Vec = (0..pusher_urls.len()) .filter(|&i| { - infos.get(i).is_some_and(|inf: &LaneInfo| inf.hard_enforcement) && payers[i].is_none() + infos + .get(i) + .is_some_and(|inf: &LaneInfo| inf.hard_enforcement) + && payers[i].is_none() }) .collect(); for &i in &unusable { @@ -2799,7 +2801,9 @@ where let mut lane_frame_ceiling: Vec = vec![usize::MAX; infos.len()]; if let Some(pc) = payment { for (i, payer) in payers.iter_mut().enumerate() { - let Some(payer) = payer.as_mut() else { continue }; + let Some(payer) = payer.as_mut() else { + continue; + }; if let Err(e) = payer.header(&http, pc).await { warn!(target: "hoverfly::upload", "lane {i}: no challenge ({e}); scheduling it unpaid"); @@ -2907,7 +2911,9 @@ where // attempt per bounce and fail the upload instead of pausing it. if let Some(pc) = payment { for (lane, payer) in payers.iter_mut().enumerate() { - let Some(payer) = payer.as_mut() else { continue }; + let Some(payer) = payer.as_mut() else { + continue; + }; if !payer.has_headroom() && payer.account.owed() > 0 && let Err(e) = payer.settle(&http, pc).await @@ -2940,10 +2946,7 @@ where }; let affordable = p.affordable_frames(); if affordable > 0 { - sched.set_lane_batch_max( - lane, - affordable.min(lane_frame_ceiling[lane]), - ); + sched.set_lane_batch_max(lane, affordable.min(lane_frame_ceiling[lane])); } if p.has_headroom() { anyone_can_take_work = true; @@ -3232,7 +3235,9 @@ where } if let Some(pc) = payment { for (lane, payer) in payers.iter_mut().enumerate() { - let Some(payer) = payer.as_mut() else { continue }; + let Some(payer) = payer.as_mut() else { + continue; + }; match payer.settle(&http, pc).await { Ok(Some(c)) => info!(target: "hoverfly::upload", "lane {lane}: final settlement, cumulative {c}"), @@ -3592,7 +3597,11 @@ async fn fetch_lane_info( // field from an assertion into a check. A caller that pins // does the full version. match crate::payer::PaymentQuote::verify(pay, None, 0, None, price_ceiling) { - Ok(q) => (Some(q.params.price_plur_per_kib), q.hard_enforcement, Some(q)), + Ok(q) => ( + Some(q.params.price_plur_per_kib), + q.hard_enforcement, + Some(q), + ), Err(e) => { warn!(target: "hoverfly::upload", "lane {base_url}: payment quote rejected ({e}); treating as unmetered"); diff --git a/src/inbound_limit.rs b/src/inbound_limit.rs index f356a9f..fd5c837 100644 --- a/src/inbound_limit.rs +++ b/src/inbound_limit.rs @@ -124,7 +124,10 @@ mod tests { for i in 0..5 { assert!(l.allow_at(b"a", t0), "burst request {i} must pass"); } - assert!(!l.allow_at(b"a", t0), "the 6th in the same instant must not"); + assert!( + !l.allow_at(b"a", t0), + "the 6th in the same instant must not" + ); } #[test] @@ -146,7 +149,10 @@ mod tests { let t0 = Instant::now(); assert!(l.allow_at(b"a", t0)); assert!(!l.allow_at(b"a", t0)); - assert!(l.allow_at(b"b", t0), "one key's flood must not throttle another"); + assert!( + l.allow_at(b"b", t0), + "one key's flood must not throttle another" + ); } /// The bypass this design exists to close: an attacker cycling keys must @@ -176,7 +182,10 @@ mod tests { let t0 = Instant::now(); assert!(l.allow_at(b"a", t0)); assert!(l.allow_at(b"b", t0)); - assert!(!l.allow_at(b"c", t0), "must refuse rather than evict a live limit"); + assert!( + !l.allow_at(b"c", t0), + "must refuse rather than evict a live limit" + ); } /// …but a bucket that has refilled to full is free to drop, so the map diff --git a/src/ledger.rs b/src/ledger.rs index 834a555..0ea209e 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -524,9 +524,13 @@ mod tests { )); assert!(matches!( l.credit_test(A, CB, 101), - Err(LedgerError::Overpayment { got: 101, owed: 100 }) + Err(LedgerError::Overpayment { + got: 101, + owed: 100 + }) )); - l.credit_test(A, CB, 100).expect("paying exactly what is owed is fine"); + l.credit_test(A, CB, 100) + .expect("paying exactly what is owed is fine"); assert_eq!(l.owed(&A), 0); } @@ -578,7 +582,10 @@ mod tests { } let mut l = Ledger::load_or_create(&path).expect("reload"); assert!( - matches!(l.credit_test(A, CB, 1200), Err(LedgerError::NotIncreasing { .. })), + matches!( + l.credit_test(A, CB, 1200), + Err(LedgerError::NotIncreasing { .. }) + ), "re-presenting the same cheque after a restart must credit nothing" ); let _ = std::fs::remove_file(&path); @@ -598,7 +605,10 @@ mod tests { let mut l = Ledger::load_or_create(&path).expect("reload"); l.commit(B, 0, 500); assert!( - matches!(l.credit_test(B, CB, 200), Err(LedgerError::ChequebookBound { .. })), + matches!( + l.credit_test(B, CB, 200), + Err(LedgerError::ChequebookBound { .. }) + ), "the chequebook binding must survive a restart" ); let _ = std::fs::remove_file(&path); @@ -659,7 +669,10 @@ mod leak_tests { // There is no debt to pay, so no cheque exists that could help. assert_eq!(l.owed(&A), 0); assert!( - matches!(l.credit_test(A, CB, 1), Err(LedgerError::Overpayment { .. })), + matches!( + l.credit_test(A, CB, 1), + Err(LedgerError::Overpayment { .. }) + ), "with nothing owed, a cheque cannot clear the overshoot" ); // Only releasing does. diff --git a/src/meter.rs b/src/meter.rs index 8ce8315..ad96ccd 100644 --- a/src/meter.rs +++ b/src/meter.rs @@ -525,8 +525,8 @@ mod tests { let s = m.summary(2126, 0); assert_eq!(s["kib_admitted"], (FRAME * 10).div_ceil(1024)); assert_eq!(s["kib_dedup"], (FRAME * 3).div_ceil(1024)); - let billable = s["kib_admitted"].as_u64().expect("u64") - - s["kib_dedup"].as_u64().expect("u64"); + let billable = + s["kib_admitted"].as_u64().expect("u64") - s["kib_dedup"].as_u64().expect("u64"); assert_eq!( s["owed_plur"].as_str().expect("string"), (billable as u128 * PRICE_PLUR_PER_KIB).to_string() @@ -794,7 +794,9 @@ mod param_tests { #[test] fn the_shipped_defaults_satisfy_the_invariant() { - Params::default().validate().expect("defaults must be valid"); + Params::default() + .validate() + .expect("defaults must be valid"); } /// The exact misconfiguration an early draft published: a dust floor 87× @@ -815,7 +817,8 @@ mod param_tests { max_outstanding_plur: SETTLE_EVERY_PLUR, ..Params::default() }; - p.validate().expect_err("cap must exceed the settlement window"); + p.validate() + .expect_err("cap must exceed the settlement window"); } #[test] @@ -857,7 +860,11 @@ mod param_tests { #[test] fn a_body_is_priced_by_rounding_up_once() { let p = Params::default(); - assert_eq!(p.price_bytes(1), p.price_plur_per_kib, "a partial KiB is a KiB"); + assert_eq!( + p.price_bytes(1), + p.price_plur_per_kib, + "a partial KiB is a KiB" + ); assert_eq!(p.price_bytes(1024), p.price_plur_per_kib); assert_eq!(p.price_bytes(1025), 2 * p.price_plur_per_kib); // One full frame, priced once rather than per-frame. diff --git a/src/metered.rs b/src/metered.rs index c970c8b..2560309 100644 --- a/src/metered.rs +++ b/src/metered.rs @@ -261,7 +261,11 @@ impl Metered { } pub fn shed_reservations(&self) -> bool { - self.ledger.lock().expect("ledger poisoned").live_reservations() >= MAX_LIVE_RESERVATIONS + self.ledger + .lock() + .expect("ledger poisoned") + .live_reservations() + >= MAX_LIVE_RESERVATIONS } /// Apply a cheque. Every free check runs before this is called; this is @@ -344,7 +348,11 @@ impl DeployedCache { fn get(&self, k: &[u8; 20]) -> Option { let (v, at) = self.map.get(k)?; - let ttl = if *v { DEPLOYED_OK_TTL } else { DEPLOYED_BAD_TTL }; + let ttl = if *v { + DEPLOYED_OK_TTL + } else { + DEPLOYED_BAD_TTL + }; (at.elapsed() < ttl).then_some(*v) } @@ -392,7 +400,13 @@ mod tests { let s = signer(); let account = *s.eth_address(); let issued = m - .issue(account, [5u8; 32], 6_200_000_000_000_000_000, "relay-a.example", now) + .issue( + account, + [5u8; 32], + 6_200_000_000_000_000_000, + "relay-a.example", + now, + ) .expect("issue"); let sol = crate::signer::PushChallenge { nonce: alloy_primitives::B256::from(issued.nonce), @@ -426,7 +440,13 @@ mod tests { let m = metered(); let victim = *signer().eth_address(); let issued = m - .issue(victim, [5u8; 32], 1_000_000_000_000_000, "relay-a.example", 1000) + .issue( + victim, + [5u8; 32], + 1_000_000_000_000_000, + "relay-a.example", + 1000, + ) .expect("issue"); let attacker = SwarmSigner::from_hex_with_nonce( "0x1111111111111111111111111111111111111111111111111111111111111111", @@ -457,7 +477,13 @@ mod tests { let s = signer(); let issued = a - .issue(*s.eth_address(), [5u8; 32], 1_000_000_000_000_000, "relay-a.example", 1000) + .issue( + *s.eth_address(), + [5u8; 32], + 1_000_000_000_000_000, + "relay-a.example", + 1000, + ) .expect("issue"); let sol = crate::signer::PushChallenge { nonce: alloy_primitives::B256::from(issued.nonce), @@ -480,7 +506,13 @@ mod tests { // Same issue instant, far-future presentation. let s = signer(); let issued = m - .issue(*s.eth_address(), [5u8; 32], 1_000_000_000_000_000, "relay-a.example", 1000) + .issue( + *s.eth_address(), + [5u8; 32], + 1_000_000_000_000_000, + "relay-a.example", + 1000, + ) .expect("issue"); let sol = crate::signer::PushChallenge { nonce: alloy_primitives::B256::from(issued.nonce), @@ -504,7 +536,13 @@ mod tests { let m = metered(); let s = signer(); let mut issued = m - .issue(*s.eth_address(), [5u8; 32], 100_000_000_000_000, "relay-a.example", 1000) + .issue( + *s.eth_address(), + [5u8; 32], + 100_000_000_000_000, + "relay-a.example", + 1000, + ) .expect("issue"); let honest_cap = issued.fields.cap_plur; issued.fields.cap_plur = honest_cap * 1_000_000; @@ -592,11 +630,17 @@ mod lifecycle_tests { // Only 100 frames actually got admitted. let billed = p.price_bytes(100 * 4251); - m.ledger.lock().unwrap().commit(ACCT, adm.reserved_plur, billed); + m.ledger + .lock() + .unwrap() + .commit(ACCT, adm.reserved_plur, billed); let l = m.ledger.lock().unwrap(); assert_eq!(l.reserved(&ACCT), 0, "the reservation is fully released"); assert_eq!(l.owed(&ACCT), billed, "only admitted bytes are billed"); - assert!(billed < adm.reserved_plur, "and it is less than was reserved"); + assert!( + billed < adm.reserved_plur, + "and it is less than was reserved" + ); } /// A POST that admits nothing must still hand its reservation back — @@ -626,7 +670,9 @@ mod lifecycle_tests { let second = p.price_bytes(32 * 1024 * 1024); m.ledger.lock().unwrap().commit(ACCT, 0, second); // A cumulative cheque: the *total*, not the delta. - let accepted = m.credit(ACCT, CB, first + second, [27u8; 65]).expect("second cheque"); + let accepted = m + .credit(ACCT, CB, first + second, [27u8; 65]) + .expect("second cheque"); assert_eq!(accepted, second, "only the new debt is credited"); assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); } @@ -677,7 +723,8 @@ mod lifecycle_tests { owed >= p.min_cheque_plur, "what is owed must clear the dust floor, or there is no exit" ); - m.credit(ACCT, CB, owed, [27u8; 65]).expect("a cheque for exactly what is owed"); + m.credit(ACCT, CB, owed, [27u8; 65]) + .expect("a cheque for exactly what is owed"); assert_eq!(m.ledger.lock().unwrap().owed(&ACCT), 0); assert!( !m.reserve_for_body(ACCT, 4251, cap).over_cap, diff --git a/src/payer.rs b/src/payer.rs index ca99087..50e4288 100644 --- a/src/payer.rs +++ b/src/payer.rs @@ -102,7 +102,10 @@ impl PaymentQuote { .map_err(|e| QuoteError::Signature(e.to_string()))?; let addr = |k: &'static str| -> Result<[u8; 20], QuoteError> { - let s = payment.get(k).and_then(|x| x.as_str()).ok_or(QuoteError::Field(k))?; + let s = payment + .get(k) + .and_then(|x| x.as_str()) + .ok_or(QuoteError::Field(k))?; let raw = hex::decode(s.trim_start_matches("0x")).map_err(|_| QuoteError::Field(k))?; <[u8; 20]>::try_from(raw.as_slice()).map_err(|_| QuoteError::Field(k)) }; @@ -127,8 +130,8 @@ impl PaymentQuote { .get("overlay_nonce") .and_then(|x| x.as_str()) .ok_or(QuoteError::Field("overlay_nonce"))?; - let raw = - hex::decode(s.trim_start_matches("0x")).map_err(|_| QuoteError::Field("overlay_nonce"))?; + let raw = hex::decode(s.trim_start_matches("0x")) + .map_err(|_| QuoteError::Field("overlay_nonce"))?; <[u8; 32]>::try_from(raw.as_slice()).map_err(|_| QuoteError::Field("overlay_nonce"))? }; @@ -150,7 +153,8 @@ impl PaymentQuote { } if let Some(overlay) = advertised_overlay { - let derived = crate::signer::derive_overlay(&node_eth_address, network_id, &overlay_nonce); + let derived = + crate::signer::derive_overlay(&node_eth_address, network_id, &overlay_nonce); if &derived != overlay { return Err(QuoteError::OverlayMismatch); } @@ -769,7 +773,10 @@ impl LanePayer { ) -> Result { let header = self.header(http, cfg).await?.to_string(); let resp = http - .get(format!("{}/v1/account", self.base_url.trim_end_matches('/'))) + .get(format!( + "{}/v1/account", + self.base_url.trim_end_matches('/') + )) .header(crate::challenge::CHALLENGE_HEADER, header) .timeout(std::time::Duration::from_secs(60)) .send() @@ -1076,7 +1083,10 @@ mod tests { }; let e = PaymentQuote::verify("e_json([0xBB; 20]), None, 1, Some(&pin), ceiling()) .expect_err("must refuse"); - assert!(matches!(e, QuoteError::WrongBeneficiary { .. }), "got {e:?}"); + assert!( + matches!(e, QuoteError::WrongBeneficiary { .. }), + "got {e:?}" + ); } #[test] @@ -1093,7 +1103,9 @@ mod tests { serde_json::json!((Params::default().settle_every_plur * 87).to_string()); let mut unsigned = body.clone(); unsigned.as_object_mut().unwrap().remove("sig"); - let sig = n.sign_eip191(unsigned.to_string().as_bytes()).expect("sign"); + let sig = n + .sign_eip191(unsigned.to_string().as_bytes()) + .expect("sign"); body["sig"] = serde_json::Value::String(format!("0x{}", hex::encode(sig))); let e = PaymentQuote::verify(&body, None, 1, None, ceiling()).expect_err("bricking lane"); assert!(matches!(e, QuoteError::BadParams(_)), "got {e:?}"); @@ -1121,7 +1133,13 @@ mod tests { Ledger::ephemeral(), ); let issued = m - .issue(account, [7u8; 32], 6_200_000_000_000_000_000, "relay-a.example", 1000) + .issue( + account, + [7u8; 32], + 6_200_000_000_000_000_000, + "relay-a.example", + 1000, + ) .expect("issue"); // Straight through the wire form the relay actually serves. let offered = OfferedChallenge::parse(&issued.to_json()).expect("parse"); @@ -1160,7 +1178,10 @@ mod tests { a.record_sent(40 * 1024 * 1024); a.record_answered(40 * 1024 * 1024, true); let second = a.next_cumulative().expect("cheque"); - assert!(second > first, "cumulative must increase: {second} > {first}"); + assert!( + second > first, + "cumulative must increase: {second} > {first}" + ); assert_eq!(second - first, p.price_bytes(40 * 1024 * 1024)); } @@ -1337,7 +1358,8 @@ mod tests { /// name the whole balance and be paid it. #[test] fn a_relay_cannot_claim_more_debt_than_the_ceiling_it_signed() { - let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let q = + PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); let cap = q.params.max_outstanding_plur; let payer = LanePayer::new("http://lane".into(), q, 0); // A chequebook far richer than the credit line, which is the normal @@ -1367,7 +1389,8 @@ mod tests { /// signing time. #[test] fn debt_within_the_ceiling_but_over_the_balance_is_refused() { - let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let q = + PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); let cap = q.params.max_outstanding_plur; let payer = LanePayer::new("http://lane".into(), q, 0); @@ -1379,7 +1402,8 @@ mod tests { #[test] fn a_line_barely_wider_than_one_post_still_makes_progress() { - let q = PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); + let q = + PaymentQuote::verify("e_json([3u8; 20]), None, 1, None, ceiling()).expect("quote"); let p = q.params; let mut payer = LanePayer::new("http://lane".into(), q, 0); @@ -1426,7 +1450,11 @@ mod tests { let mut a = LaneAccount::new(p, [3u8; 20]); // Fresh process: no debt, and nothing to pay with. assert_eq!(a.owed(), 0); - assert_eq!(a.next_cumulative(), None, "cannot pay what it does not know"); + assert_eq!( + a.next_cumulative(), + None, + "cannot pay what it does not know" + ); let carried = p.min_cheque_plur * 3; assert!(a.adopt_relay_debt(carried), "relay knows more than we do"); diff --git a/src/protocols/pushsync.rs b/src/protocols/pushsync.rs index 05aa9b3..303021b 100644 --- a/src/protocols/pushsync.rs +++ b/src/protocols/pushsync.rs @@ -252,7 +252,8 @@ mod tests { fn missized_addresses_are_rejected_not_panicked_on() { for bad in [vec![], vec![1u8; 31], vec![1u8; 33], vec![1u8; 64]] { let n = bad.len(); - let err = push_against(receipt_for(bad)).expect_err("missized address must be rejected"); + let err = + push_against(receipt_for(bad)).expect_err("missized address must be rejected"); assert!( matches!(&err, PushsyncError::Peer(m) if m.contains("address mismatch")), "unexpected error for {n}-byte address: {err}" diff --git a/src/protocols/swap.rs b/src/protocols/swap.rs index f0bdd57..702f4e4 100644 --- a/src/protocols/swap.rs +++ b/src/protocols/swap.rs @@ -269,8 +269,7 @@ fn json_address(v: &serde_json::Value, field: &str) -> Result<[u8; 20], SwapErro .and_then(|x| x.as_str()) .ok_or_else(|| SwapError::Json(format!("missing {field}")))?; let hex_str = s.trim_start_matches("0x").trim_start_matches("0X"); - let raw = - hex::decode(hex_str).map_err(|e| SwapError::Json(format!("{field} not hex: {e}")))?; + let raw = hex::decode(hex_str).map_err(|e| SwapError::Json(format!("{field} not hex: {e}")))?; if raw.len() != 20 { return Err(SwapError::Json(format!( "{field} must be 20 bytes, got {}", diff --git a/src/pusher.rs b/src/pusher.rs index 7dba70f..4bb0d39 100644 --- a/src/pusher.rs +++ b/src/pusher.rs @@ -479,9 +479,7 @@ pub async fn run(opts: PusherOpts) -> Result<(), Box> { // streamed probe and push responses are unaffected. let _ = hyper::server::conn::http1::Builder::new() .timer(hyper_util::rt::TokioTimer::new()) - .header_read_timeout(std::time::Duration::from_secs( - HEADER_READ_TIMEOUT_SECS, - )) + .header_read_timeout(std::time::Duration::from_secs(HEADER_READ_TIMEOUT_SECS)) .serve_connection(io, svc) .await; }); @@ -514,8 +512,14 @@ async fn handle( (&Method::POST, "/v1/probe") => probe_response(state, req.uri().query()), (&Method::POST, "/v1/tcpcheck") => tcpcheck_response(state, req.uri().query()), (&Method::POST, "/v1/push") => push_response(state, req).await, - (_, "/v1/probe") | (_, "/v1/status") | (_, "/v1/tcpcheck") | (_, "/v1/push") - | (_, "/v1/meter") | (_, "/v1/challenge") | (_, "/v1/pay") | (_, "/v1/account") => { + (_, "/v1/probe") + | (_, "/v1/status") + | (_, "/v1/tcpcheck") + | (_, "/v1/push") + | (_, "/v1/meter") + | (_, "/v1/challenge") + | (_, "/v1/pay") + | (_, "/v1/account") => { json_line_response(StatusCode::METHOD_NOT_ALLOWED, "method not allowed") } _ => json_line_response(StatusCode::NOT_FOUND, "not found"), @@ -634,7 +638,10 @@ fn admit_metered( .verify_header(raw, crate::challenge::now_unix()) .map_err(|e| Box::new(json_line_response(StatusCode::UNAUTHORIZED, &e)))?; if !m.allow_account(&verified.account) { - return Err(Box::new(json_line_response(StatusCode::TOO_MANY_REQUESTS, "slow down"))); + return Err(Box::new(json_line_response( + StatusCode::TOO_MANY_REQUESTS, + "slow down", + ))); } // The reservation ledger is attacker-influenced (one entry per batch in // standing), so shed rather than grow without bound (§7.2). @@ -732,8 +739,8 @@ fn build_metered(opts: &PusherOpts) -> Result, S } std::fs::create_dir_all(&m.state_dir) .map_err(|e| format!("--state-dir {}: {e}", m.state_dir.display()))?; - let ledger = crate::ledger::Ledger::load_or_create(m.state_dir.join("ledger.json")) - .map_err(|e| { + let ledger = + crate::ledger::Ledger::load_or_create(m.state_dir.join("ledger.json")).map_err(|e| { format!( "ledger at {}: {e} — metered mode requires durable state, because losing \ last_cumulative turns one signature into unlimited free service (§11.4)", @@ -826,7 +833,10 @@ async fn challenge_response( /// spend a single `eth_call`. Without those, every "free" check passes for /// a cheque an attacker synthesizes at zero cost and each garbage POST buys /// one `deployedContracts` call. -async fn pay_response(state: Arc, req: Request) -> Response { +async fn pay_response( + state: Arc, + req: Request, +) -> Response { let Some(m) = state.metered.as_ref() else { return json_line_response(StatusCode::NOT_FOUND, "relay is not metered"); }; @@ -847,7 +857,11 @@ async fn pay_response(state: Arc, req: Request) -> // client always has debt by the time it settles, so this costs nothing // legitimate and makes the endpoint useless to anyone who has not first // done billable work. - let owed = m.ledger.lock().expect("ledger poisoned").owed(&verified.account); + let owed = m + .ledger + .lock() + .expect("ledger poisoned") + .owed(&verified.account); if owed == 0 { return json_line_response(StatusCode::BAD_REQUEST, "nothing owed on this account"); } @@ -865,7 +879,9 @@ async fn pay_response(state: Arc, req: Request) -> } let cumulative: u128 = match u128::try_from(cheque.cumulative_payout) { Ok(v) if v <= crate::ledger::MAX_CUMULATIVE_PLUR => v, - _ => return json_line_response(StatusCode::BAD_REQUEST, "cumulative payout is implausible"), + _ => { + return json_line_response(StatusCode::BAD_REQUEST, "cumulative payout is implausible"); + } }; let have = m .ledger @@ -928,7 +944,10 @@ async fn pay_response(state: Arc, req: Request) -> ); } if cb_state.issuer.into_array() != recovered { - return json_line_response(StatusCode::BAD_REQUEST, "cheque was not signed by the issuer"); + return json_line_response( + StatusCode::BAD_REQUEST, + "cheque was not signed by the issuer", + ); } if cb_state.issuer.into_array() != verified.account { return json_line_response( @@ -1050,7 +1069,10 @@ async fn read_body_limited( StatusCode::PAYLOAD_TOO_LARGE, "body exceeds limit or read error", )), - Err(_) => Err(json_line_response(StatusCode::REQUEST_TIMEOUT, "body read timed out")), + Err(_) => Err(json_line_response( + StatusCode::REQUEST_TIMEOUT, + "body read timed out", + )), } } @@ -1130,11 +1152,12 @@ fn meter_response(state: &State, headers: &hyper::HeaderMap) -> Response { - return json_line_response( - StatusCode::REQUEST_TIMEOUT, - "body read timed out", - ); + return json_line_response(StatusCode::REQUEST_TIMEOUT, "body read timed out"); } }; let chunks = match pushframe::decode_batch(&bytes, PUSH_BATCH_MAX) { diff --git a/src/pushsched/tests.rs b/src/pushsched/tests.rs index 6c85f6a..cd4dfac 100644 --- a/src/pushsched/tests.rs +++ b/src/pushsched/tests.rs @@ -718,7 +718,11 @@ fn an_unfunded_lane_is_paused_then_restored_by_paying() { let (b, lane) = dispatch_one(&mut s, 0); s.on_batch_result(b, BatchOutcome::PaymentRequired, 0); - assert_eq!(s.unfunded_lanes(), vec![lane], "the driver must see what to pay"); + assert_eq!( + s.unfunded_lanes(), + vec![lane], + "the driver must see what to pay" + ); for _ in 0..8 { assert_ne!( s.next(20).map(|a| a.lane), diff --git a/src/signer.rs b/src/signer.rs index 2e43b8e..6fc8acb 100644 --- a/src/signer.rs +++ b/src/signer.rs @@ -707,7 +707,11 @@ mod metered_tests { let amount = alloy_primitives::U256::from(7u64); let sig = s.sign_cheque(&cb, &bn, amount, 100).expect("sign"); let other = recover_cheque_issuer(&cb, &bn, amount, 11155111, &sig).expect("recovers"); - assert_ne!(other, *s.eth_address(), "wrong chain must not recover the issuer"); + assert_ne!( + other, + *s.eth_address(), + "wrong chain must not recover the issuer" + ); } /// Changing any signed field must move the recovered address, or the